diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 5c342d14..ddfb86ec 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -5,160 +5,130 @@ "email": "daymadev89@gmail.com" }, "metadata": { - "description": "Professional Claude Code skills for GitHub operations, document conversion, diagram generation, statusline customization, Teams communication, repomix utilities, skill creation, CLI demo generation, LLM icon access, Cloudflare troubleshooting, UI design system extraction, professional presentation creation, YouTube video downloading, secure repomix packaging, ASR transcription correction, video comparison quality analysis, comprehensive QA testing infrastructure, prompt optimization with EARS methodology, session history recovery, documentation cleanup, format-controlled deep research report generation with evidence tracking, PDF generation with Chinese font support, CLAUDE.md progressive disclosure optimization, CCPM skill registry search and management, Promptfoo LLM evaluation framework, iOS app development with XcodeGen and SwiftUI, fact-checking with automated corrections, Twitter/X content fetching, intelligent macOS disk space recovery, skill quality review and improvement, GitHub contribution strategy, complete internationalization/localization setup, plugin/skill troubleshooting with diagnostic tools, evidence-based competitor analysis with source citations, Windows Remote Desktop (AVD/W365) connection quality diagnosis with transport protocol analysis and log parsing, Tailscale+proxy conflict diagnosis with SSH tunnel SOP for remote development, multi-path parallel product analysis with cross-model test-time compute scaling, real financial data collection for US equities with validation and yfinance pitfall handling, advanced Excel automation for formatted workbook generation and complex xlsm parsing, and macOS programmatic window screenshot capture workflows", - "version": "1.37.0", - "homepage": "https://github.com/daymade/claude-code-skills" + "description": "Professional Claude Code skills for GitHub operations, document conversion, diagram generation, statusline customization, Teams communication, repomix utilities, skill creation, CLI demo generation, LLM icon access, Cloudflare troubleshooting, UI design system extraction, professional presentation creation, YouTube video downloading, secure repomix packaging, ASR transcription correction, video comparison quality analysis, comprehensive QA testing infrastructure, prompt optimization with EARS methodology, session history recovery, local Claude session continuation from `.claude` artifacts, documentation cleanup, format-controlled deep research report generation with evidence tracking, PDF generation with Chinese font support, CLAUDE.md progressive disclosure optimization, CCPM skill registry search and management, Promptfoo LLM evaluation framework, iOS app development with XcodeGen and SwiftUI, fact-checking with automated corrections, Twitter/X content fetching, intelligent macOS disk space recovery, skill quality review and improvement, GitHub contribution strategy, complete internationalization/localization setup, plugin/skill troubleshooting with diagnostic tools, evidence-based competitor analysis with source citations, Windows Remote Desktop (AVD/W365) connection quality diagnosis with transport protocol analysis and log parsing, Tailscale+proxy conflict diagnosis with SSH tunnel SOP for remote development, multi-path parallel product analysis with cross-model test-time compute scaling, real financial data collection for US equities with validation and yfinance pitfall handling, advanced Excel automation for formatted workbook generation and complex xlsm parsing, macOS programmatic window screenshot capture workflows, verified Scrapling CLI installation and web extraction workflows, plugin marketplace development for converting skills repos into official Claude Code marketplaces, Tencent IMA knowledge base companion with zero-config installation across Claude Code/Codex/OpenClaw, upstream bug detection and runtime repair, personalized fan-out search with priority-based knowledge base boosting, and suite plugins that expose related skills under shared namespaces for combined installation workflows — including a dedicated Claude Code operations suite (daymade-claude-code) bundling session recovery, CLAUDE.md optimization, troubleshooting, statusline configuration, export repair, and marketplace development skills under one namespace", + "version": "1.47.0" }, "plugins": [ { - "name": "skill-creator", - "description": "Essential meta-skill for creating effective Claude Code skills with initialization scripts, validation, packaging, marketplace registration, and privacy best practices", - "source": "./", - "strict": false, - "version": "1.4.1", - "category": "developer-tools", - "keywords": [ - "skill-creation", - "claude-code", - "development", - "tooling", - "workflow", - "meta-skill", - "essential" - ], - "skills": [ - "./skill-creator" - ] - }, - { - "name": "github-ops", - "description": "Comprehensive GitHub operations using gh CLI and GitHub API for pull requests, issues, repositories, workflows, and API interactions", + "name": "asr-transcribe-to-text", + "description": "Transcribe audio and video files to text using a remote ASR service (Qwen3-ASR or OpenAI-compatible endpoint). Extracts audio from video, sends to configurable ASR endpoint, outputs clean text. Use when the user wants to transcribe recordings, convert audio/video to text, do speech-to-text, or mentions ASR, Qwen ASR, 转录, 语音转文字, 录音转文字, or has a meeting recording, lecture, interview, or screen recording to transcribe.", "source": "./", "strict": false, "version": "1.0.0", - "category": "developer-tools", + "category": "productivity", "keywords": [ - "github", - "gh-cli", - "pull-request", - "issues", - "workflows", - "api" + "asr", + "transcription", + "speech-to-text", + "qwen", + "audio", + "video", + "vllm", + "ffmpeg" ], "skills": [ - "./github-ops" + "./asr-transcribe-to-text" ] }, { - "name": "markdown-tools", - "description": "Convert documents (PDFs, Word, PowerPoint) to high-quality markdown with multi-tool orchestration. Supports Quick Mode (fast, single tool) and Heavy Mode (best quality, multi-tool merge with segment-level selection). Features PyMuPDF4LLM for LLM-optimized PDF conversion, pandoc for DOCX/PPTX structure preservation, quality validation with HTML reports, and image extraction with metadata", + "name": "capture-screen", + "description": "Programmatic screenshot capture on macOS. Get window IDs via Swift CGWindowListCopyWindowInfo, capture specific windows with screencapture -l, and control application windows via AppleScript. Supports multi-shot workflows for capturing different sections of the same window. Use when taking automated screenshots, capturing application windows, or creating visual documentation", "source": "./", "strict": false, - "version": "1.2.0", - "category": "document-conversion", + "version": "1.0.1", + "category": "utilities", "keywords": [ - "markdown", - "pdf", - "docx", - "pptx", - "pymupdf4llm", - "pandoc", - "markitdown", - "heavy-mode", - "quality-validation" + "screenshot", + "screencapture", + "macos", + "window-capture", + "swift", + "applescript", + "automation", + "visual-documentation" ], "skills": [ - "./markdown-tools" + "./capture-screen" ] }, { - "name": "mermaid-tools", - "description": "Generate Mermaid diagrams from markdown with automatic PNG/SVG rendering and extraction from documents", - "source": "./", + "name": "claude-code-history-files-finder", + "description": "Find and recover content from Claude Code session history files. Use when searching for deleted files, tracking changes across sessions, analyzing conversation history, or recovering code/documents from previous Claude interactions. Triggers include mentions of session history, recover deleted, find in history, previous conversation, or .claude/projects", + "source": "./daymade-claude-code/claude-code-history-files-finder", "strict": false, - "version": "1.0.0", - "category": "documentation", + "version": "1.0.3", + "category": "developer-tools", "keywords": [ - "mermaid", - "diagrams", - "visualization", - "flowchart", - "sequence" + "session-history", + "recovery", + "deleted-files", + "conversation-history", + "file-tracking", + "claude-code", + "history-analysis" ], "skills": [ - "./mermaid-tools" + "./" ] }, { - "name": "statusline-generator", - "description": "Configure Claude Code statuslines with multi-line layouts, cost tracking via ccusage, git status, and customizable colors", - "source": "./", + "name": "claude-export-txt-better", + "description": "Fixes broken line wrapping in Claude Code exported conversation files (.txt), reconstructing tables, paragraphs, paths, and tool calls that were hard-wrapped at fixed column widths. Includes an automated validation suite (generic, file-agnostic checks). Triggers when the user has a Claude Code export file with broken formatting, mentions \"fix export\", \"fix conversation\", \"exported conversation\", \"make export readable\", references a file matching YYYY-MM-DD-HHMMSS-*.txt, or has a .txt file with broken tables, split paths, or mangled tool output from Claude Code.", + "source": "./daymade-claude-code/claude-export-txt-better", "strict": false, - "version": "1.0.0", - "category": "customization", + "version": "1.0.1", + "category": "utilities", "keywords": [ - "statusline", - "ccusage", - "git-status", - "customization", - "prompt" + "claude-code", + "export", + "txt", + "fix", + "line-wrapping", + "formatting" ], "skills": [ - "./statusline-generator" + "./" ] }, { - "name": "teams-channel-post-writer", - "description": "Create professional Microsoft Teams channel posts with Adaptive Cards, formatted announcements, and corporate communication standards", - "source": "./", + "name": "claude-md-progressive-disclosurer", + "description": "Optimize user CLAUDE.md files by applying progressive disclosure principles. This skill should be used when users want to reduce CLAUDE.md bloat, move detailed content to references, extract reusable patterns into skills, or improve context efficiency. Triggers include optimize CLAUDE.md, reduce CLAUDE.md size, apply progressive disclosure, or complaints about CLAUDE.md being too long", + "source": "./daymade-claude-code/claude-md-progressive-disclosurer", "strict": false, - "version": "1.0.0", - "category": "communication", + "version": "1.2.1", + "category": "productivity", "keywords": [ - "teams", - "microsoft", - "adaptive-cards", - "communication", - "announcements" + "claude-md", + "progressive-disclosure", + "optimization", + "context-efficiency", + "configuration", + "token-savings" ], "skills": [ - "./teams-channel-post-writer" + "./" ] }, { - "name": "repomix-unmixer", - "description": "Extract files from repomix packaged formats (XML, Markdown, JSON) with automatic format detection and validation", - "source": "./", + "name": "claude-skills-troubleshooting", + "description": "Diagnose and resolve Claude Code plugin and skill configuration issues. Debug plugin installation, enablement, and activation problems with systematic workflows. Use when plugins are installed but not showing in available skills list, skills are not activating as expected, troubleshooting enabledPlugins configuration in settings.json, debugging 'plugin not working' or 'skill not showing' issues, or understanding plugin state architecture and lifecycle", + "source": "./daymade-claude-code/claude-skills-troubleshooting", "strict": false, - "version": "1.0.0", + "version": "1.0.1", "category": "utilities", "keywords": [ - "repomix", - "unmix", - "extract", - "xml", - "conversion" - ], - "skills": [ - "./repomix-unmixer" - ] - }, - { - "name": "llm-icon-finder", - "description": "Find and access AI/LLM model brand icons from lobe-icons library in SVG/PNG/WEBP formats", - "source": "./", - "strict": false, - "version": "1.0.0", - "category": "assets", - "keywords": [ - "icons", - "ai-models", - "llm", - "branding", - "lobe-icons" + "troubleshooting", + "debugging", + "plugins", + "skills", + "diagnostics", + "configuration", + "enabledPlugins", + "settings", + "marketplace" ], "skills": [ - "./llm-icon-finder" + "./" ] }, { @@ -203,275 +173,588 @@ ] }, { - "name": "ui-designer", - "description": "Extract design systems from reference UI images and generate implementation-ready UI design prompts. Use when users provide UI screenshots/mockups and want to create consistent designs", + "name": "competitors-analysis", + "description": "Analyze competitor repositories with evidence-based approach. Use when tracking competitors, creating competitor profiles, or generating competitive analysis. All analysis must be based on actual cloned code, never assumptions. Triggers include analyze competitor, add competitor, competitive analysis, or 竞品分析", "source": "./", "strict": false, + "version": "1.0.1", + "category": "productivity", + "keywords": [ + "competitors", + "competitive-analysis", + "competitor-tracking", + "evidence-based", + "market-research", + "竞品分析", + "code-analysis" + ], + "skills": [ + "./competitors-analysis" + ] + }, + { + "name": "continue-claude-work", + "description": "Recover actionable context from local `.claude` session artifacts and continue interrupted work without running `claude --resume`. Extracts compact boundary summaries, subagent workflow state, session end reason, and workspace drift via bundled Python script. Use when a user provides a Claude session ID, asks to continue prior work from local history, or wants to inspect `.claude` files before resuming implementation", + "source": "./daymade-claude-code/continue-claude-work", + "strict": false, + "version": "1.1.2", + "category": "developer-tools", + "keywords": [ + "claude-code", + "session-resume", + "context-recovery", + "jsonl", + "compaction", + "subagent", + "history", + "workflow-continuation", + "local-artifacts" + ], + "skills": [ + "./" + ] + }, + { + "name": "daymade-claude-code", + "description": "Claude Code operations suite that bundles session history recovery, interrupted-work continuation, plugin/skill troubleshooting, CLAUDE.md progressive disclosure optimization, statusline configuration, exported .txt repair, and plugin marketplace development under one shared namespace. Install once to get the full Claude Code power-user toolkit.", + "source": "./daymade-claude-code", + "strict": false, "version": "1.0.0", - "category": "design", + "category": "suite", "keywords": [ - "ui", - "design-system", - "mockup", - "screenshot", - "design-extraction", - "mvp" + "suite", + "claude-code", + "session-recovery", + "claude-md", + "statusline", + "troubleshooting", + "marketplace-dev" ], "skills": [ - "./ui-designer" + "./claude-code-history-files-finder", + "./continue-claude-work", + "./claude-skills-troubleshooting", + "./claude-md-progressive-disclosurer", + "./statusline-generator", + "./claude-export-txt-better", + "./marketplace-dev" ] }, { - "name": "ppt-creator", - "description": "Create professional slide decks from topics or documents. Generates structured content with data-driven charts, speaker notes, and complete PPTX files. Applies persuasive storytelling principles (Pyramid Principle, assertion-evidence). Supports multiple formats (Marp, PowerPoint). Use for presentations, pitches, slide decks, or keynotes", + "name": "daymade-docs", + "description": "Documentation suite plugin that exposes document conversion, Mermaid diagram generation, PDF/PPT creation, documentation cleanup, and meeting minutes skills under one shared namespace", + "source": "./daymade-docs", + "strict": false, + "version": "1.0.1", + "category": "suite", + "keywords": [ + "suite", + "documentation", + "markdown", + "mermaid", + "pdf", + "pptx", + "meeting-minutes" + ], + "skills": [ + "./doc-to-markdown", + "./mermaid-tools", + "./pdf-creator", + "./ppt-creator", + "./docs-cleaner", + "./meeting-minutes-taker" + ] + }, + { + "name": "deep-research", + "description": "Generate format-controlled research reports with evidence tracking, source governance, and multi-pass synthesis. V6.1 adds: source accessibility (circular verification forbidden, exclusive advantage encouraged). Enterprise Research Mode: six-dimension data collection, SWOT/barrier/risk frameworks, and three-level quality control for company research", + "source": "./", + "strict": false, + "version": "2.4.0", + "category": "documentation", + "keywords": [ + "research", + "report", + "analysis", + "literature-review", + "market-research", + "citations", + "evidence", + "deepresearch", + "enterprise", + "company-research", + "due-diligence" + ], + "skills": [ + "./deep-research" + ] + }, + { + "name": "doc-to-markdown", + "description": "Converts DOCX/PDF/PPTX to high-quality Markdown. Pandoc engine + 8 post-processing fixes: grid/simple tables to pipe tables, CJK bold spacing, JSON pretty-print, image path flattening, pandoc attribute cleanup, code block detection, bracket fixes. Benchmarked 7.6/10 (best-in-class vs Docling/MarkItDown/Mammoth). 31 unit tests. Trigger on \"convert document\", \"docx to markdown\", \"parse word\", \"doc to markdown\", \"解析word\", \"转换文档\".", + "source": "./daymade-docs/doc-to-markdown", + "strict": false, + "version": "2.1.1", + "category": "document-conversion", + "keywords": [ + "markdown", + "docx", + "pdf", + "pptx", + "converter", + "pandoc", + "document", + "cjk", + "chinese" + ], + "skills": [ + "./" + ] + }, + { + "name": "docs-cleaner", + "description": "Consolidates redundant documentation while preserving all valuable content. Use when cleaning up documentation bloat, merging redundant docs, reducing documentation sprawl, or consolidating multiple files covering the same topic", + "source": "./daymade-docs/docs-cleaner", + "strict": false, + "version": "1.0.1", + "category": "productivity", + "keywords": [ + "documentation", + "cleanup", + "consolidation", + "redundancy", + "merge", + "docs" + ], + "skills": [ + "./" + ] + }, + { + "name": "douban-skill", + "description": "Export and sync Douban (豆瓣) book/movie/music/game collections to local CSV files via Frodo API. Supports full export (all history) and RSS incremental sync (recent items). Use when the user wants to export Douban reading/watching/listening/gaming history, back up their Douban data, set up incremental sync, or mentions 豆瓣/douban collections. Triggers on: 豆瓣, douban, 读书记录, 观影记录, 书影音, 导出豆瓣, export, backup, sync, collection.", "source": "./", "strict": false, "version": "1.0.0", "category": "productivity", "keywords": [ - "presentation", - "powerpoint", - "pptx", - "slides", - "marp", - "charts", - "data-visualization", - "pyramid-principle" + "douban", + "豆瓣", + "csv", + "export", + "backup", + "rss", + "frodo" ], "skills": [ - "./ppt-creator" + "./douban-skill" ] }, { - "name": "youtube-downloader", - "description": "Download YouTube videos and HLS streams (m3u8) from platforms like Mux, Vimeo, etc. using yt-dlp and ffmpeg. Use when users request downloading videos, extracting audio, handling protected streams with authentication headers, or troubleshooting download issues like nsig extraction failures, 403 errors, or cookie extraction problems", + "name": "excel-automation", + "description": "Create, parse, and control Excel files on macOS. Professional formatting with openpyxl (font colors, fills, borders, conditional formatting), complex xlsm parsing with stdlib zipfile+xml for investment bank financial models, and Excel window control via AppleScript (zoom, scroll, select). Use when creating formatted Excel reports, parsing financial models, or automating Excel on macOS", "source": "./", "strict": false, - "version": "1.1.0", - "category": "utilities", + "version": "1.0.0", + "category": "productivity", "keywords": [ - "youtube", - "yt-dlp", - "video-download", - "audio-extraction", - "mp3", - "download", - "hls", - "m3u8", - "ffmpeg", - "streaming", - "mux", - "vimeo" + "excel", + "openpyxl", + "xlsm", + "spreadsheet", + "formatting", + "financial-model", + "applescript", + "macos", + "dcf", + "investment-banking" ], "skills": [ - "./youtube-downloader" + "./excel-automation" ] }, { - "name": "repomix-safe-mixer", - "description": "Safely package codebases with repomix by automatically detecting and removing hardcoded credentials before packing. Use when packaging code for distribution, creating reference packages, or when the user mentions security concerns about sharing code with repomix", + "name": "fact-checker", + "description": "Verifies factual claims in documents using web search and official sources, then proposes corrections with user confirmation. Use when the user asks to fact-check, verify information, validate claims, check accuracy, or update outdated information in documents. Supports AI model specs, technical documentation, statistics, and general factual statements", "source": "./", "strict": false, "version": "1.0.0", - "category": "security", + "category": "productivity", "keywords": [ - "repomix", - "security", - "credentials", - "secrets-scanning", - "safe-packaging", - "secret-detection", - "code-security" + "fact-checking", + "verification", + "accuracy", + "sources", + "validation", + "corrections", + "web-search" ], "skills": [ - "./repomix-safe-mixer" + "./fact-checker" ] }, { - "name": "transcript-fixer", - "description": "Corrects speech-to-text (ASR/STT) transcription errors in meeting notes, lecture recordings, interviews, and voice memos through dictionary-based rules and AI corrections. Supports Chinese domain names, AI fallback to Claude Code, and iterative dictionary building. Use when users mention transcript correction, ASR errors, speech-to-text mistakes, homophone errors, or working with transcription files", + "name": "financial-data-collector", + "description": "Collect real financial data for any US publicly traded company from free public sources (yfinance). Output structured JSON with market data, historical financials, WACC inputs, and analyst estimates. Handles NaN year detection, CapEx sign preservation, and FCF definition mismatches. Use when users request company financials, stock data, DCF inputs, or financial data collection for any US equity ticker", "source": "./", "strict": false, - "version": "1.1.0", + "version": "1.0.0", "category": "productivity", "keywords": [ - "transcription", - "asr", - "stt", - "speech-to-text", - "correction", - "ai", - "meeting-notes", - "nlp" + "finance", + "financial-data", + "yfinance", + "stock-data", + "dcf", + "wacc", + "market-data", + "investment-research", + "sec-filings" ], "skills": [ - "./transcript-fixer" + "./financial-data-collector" ] }, { - "name": "video-comparer", - "description": "Compare two videos and generate interactive HTML reports with quality metrics (PSNR, SSIM) and frame-by-frame visual comparisons. Use when analyzing compression results, evaluating codec performance, or assessing video quality differences", + "name": "gangtise-copilot", + "description": "One-stop installer and companion for the full Gangtise (岗底斯投研) OpenAPI skill suite — 19 official skills covering data retrieval (OHLC 行情, 财务, 估值, 研报, 首席观点, 会议纪要, 调研纪要), research workflows (个股研究 L1-L4, 观点 PK 对抗性分析, 主题研究, 事件复盘), and utility (股票池管理, 公开网页搜索). Zero-config install to Claude Code / OpenClaw / Codex with 4 preset modes (full / workshop / minimal / custom), guides accessKey + secretAccessKey setup with a live validation call against open.gangtise.com, and ships a read-only diagnostic script. Use this skill whenever the user mentions Gangtise, 岗底斯, gangtise-data-client, gangtise-kb-client, gangtise-file-client, gangtise-stock-research, gangtise-opinion-pk, installing any gangtise-* skill, configuring its credentials, or reports errors like 'token is invalid', '接口地址错误', 'the uri can't be accessed'. This is a wrapper around Gangtise's official skills — it installs and orchestrates them rather than replacing them.", + "source": "./", + "strict": false, + "version": "1.1.0", + "category": "developer-tools", + "keywords": [ + "gangtise", + "岗底斯", + "investment-research", + "financial-data", + "ohlc", + "research-report", + "installer", + "wrapper", + "claude-code", + "openclaw", + "codex" + ], + "skills": [ + "./gangtise-copilot" + ] + }, + { + "name": "github-contributor", + "description": "Strategic guide for becoming an effective GitHub contributor. Covers opportunity discovery, project selection, high-quality PR creation, and reputation building. Use when looking to contribute to open-source projects, building GitHub presence, or learning contribution best practices", + "source": "./", + "strict": false, + "version": "1.0.3", + "category": "developer-tools", + "keywords": [ + "github", + "open-source", + "contribution", + "pull-request", + "reputation", + "contributor", + "oss" + ], + "skills": [ + "./github-contributor" + ] + }, + { + "name": "github-ops", + "description": "Comprehensive GitHub operations using gh CLI and GitHub API for pull requests, issues, repositories, workflows, and API interactions", "source": "./", "strict": false, "version": "1.0.0", - "category": "media", + "category": "developer-tools", "keywords": [ - "video", - "comparison", - "quality-analysis", - "psnr", - "ssim", - "compression", - "ffmpeg", - "codec" + "github", + "gh-cli", + "pull-request", + "issues", + "workflows", + "api" ], "skills": [ - "./video-comparer" + "./github-ops" ] }, { - "name": "qa-expert", - "description": "Comprehensive QA testing infrastructure with autonomous LLM execution, Google Testing Standards (AAA pattern), and OWASP security testing. Use when establishing QA processes, writing test cases, executing test plans, tracking bugs with P0-P4 classification, calculating quality metrics, enforcing quality gates, or preparing third-party QA handoffs. Enables 100x faster test execution via master prompts", + "name": "i18n-expert", + "description": "Complete internationalization/localization setup and auditing for UI codebases. Configure i18n frameworks, replace hard-coded strings with translation keys, ensure locale parity between en-US and zh-CN, and validate pluralization and formatting. Use when setting up i18n for React/Next.js/Vue apps, auditing existing implementations, replacing hard-coded strings, ensuring proper error code mapping, or validating pluralization and date/time/number formatting across locales", "source": "./", "strict": false, "version": "1.0.0", "category": "developer-tools", "keywords": [ - "qa", - "testing", - "test-cases", - "bug-tracking", - "google-standards", - "owasp", - "security", - "automation", - "quality-gates", - "metrics" + "i18n", + "internationalization", + "localization", + "translation", + "react-i18next", + "next-intl", + "vue-i18n", + "locale", + "multilingual", + "globalization" ], "skills": [ - "./qa-expert" + "./i18n-expert" + ] + }, + { + "name": "iOS-APP-developer", + "description": "Develops iOS applications with XcodeGen, SwiftUI, and SPM. Use when configuring XcodeGen project.yml, resolving SPM dependency issues, deploying to devices, handling code signing, debugging camera/AVFoundation, iOS version compatibility issues, or fixing Library not loaded @rpath framework errors. Includes state machine testing patterns for @MainActor classes", + "source": "./", + "strict": false, + "version": "1.1.0", + "category": "developer-tools", + "keywords": [ + "ios", + "xcodegen", + "swiftui", + "spm", + "avfoundation", + "camera", + "code-signing", + "device-deployment", + "swift", + "xcode", + "testing" + ], + "skills": [ + "./iOS-APP-developer" + ] + }, + { + "name": "ima-copilot", + "description": "One-stop companion and installer for the official Tencent IMA skill (ima.qq.com). Installs upstream ima-skill to Claude Code/Codex/OpenClaw via npx skills add, guides API key setup, detects and fixes known issues (including the missing-YAML-frontmatter bug in submodule SKILL.md files) with user consent, and implements a personalized fan-out search strategy with priority-based knowledge base boosting and silent truncation detection", + "source": "./", + "strict": false, + "version": "1.0.1", + "category": "developer-tools", + "keywords": [ + "ima", + "tencent-ima", + "knowledge-base", + "note-search", + "fan-out-search", + "installer", + "wrapper", + "upstream-fix", + "claude-code", + "codex", + "openclaw" + ], + "skills": [ + "./ima-copilot" + ] + }, + { + "name": "llm-icon-finder", + "description": "Find and access AI/LLM model brand icons from lobe-icons library in SVG/PNG/WEBP formats", + "source": "./", + "strict": false, + "version": "1.0.0", + "category": "assets", + "keywords": [ + "icons", + "ai-models", + "llm", + "branding", + "lobe-icons" + ], + "skills": [ + "./llm-icon-finder" + ] + }, + { + "name": "macos-cleaner", + "description": "Intelligent macOS disk space analysis and cleanup with safety-first philosophy. Use when users report disk space issues, need to clean their Mac, or want to understand storage consumption. Analyzes system caches, application remnants, large files, and development environments (Docker, Homebrew, npm, pip) with risk categorization (Safe/Caution/Keep) and requires explicit user confirmation before any deletions. Includes Mole visual tool integration for hybrid workflow", + "source": "./", + "strict": false, + "version": "1.1.0", + "category": "utilities", + "keywords": [ + "macos", + "disk-space", + "cleanup", + "cache", + "storage", + "developer-tools", + "docker", + "homebrew", + "system-maintenance", + "safety" + ], + "skills": [ + "./macos-cleaner" ] }, { - "name": "prompt-optimizer", - "description": "Transform vague prompts into precise, well-structured specifications using EARS (Easy Approach to Requirements Syntax) methodology. Use when users provide loose requirements, ambiguous feature descriptions, need to enhance prompts for AI-generated code/products/documents, request prompt optimization, or want to improve requirements engineering. Applies domain theories (GTD, BJ Fogg, Gestalt, AIDA, Zero Trust) and structured Role/Skills/Workflows/Examples/Formats framework", - "source": "./", + "name": "marketplace-dev", + "description": "Converts any Claude Code skills repository into an official plugin marketplace. Analyzes existing skills, generates .claude-plugin/marketplace.json conforming to the Anthropic spec, validates with claude plugin validate, runs one-shot check_marketplace.sh (schema + resolution + reverse sync), tests real installation, and creates a PR. Encodes hard-won anti-patterns from real marketplace development.", + "source": "./daymade-claude-code/marketplace-dev", "strict": false, - "version": "1.1.0", - "category": "productivity", + "version": "1.2.1", + "category": "developer-tools", "keywords": [ - "prompt-engineering", - "ears", - "requirements", - "specifications", - "optimization", - "domain-theory", - "prompt-enhancement", - "ai-prompting" + "marketplace", + "plugin", + "distribution", + "packaging" ], "skills": [ - "./prompt-optimizer" - ] + "./" + ], + "hooks": { + "PostToolUse": [ + { + "matcher": "Write|Edit", + "hooks": [ + { + "type": "command", + "command": "${CLAUDE_PLUGIN_ROOT}/hooks/post_edit_validate.sh" + } + ] + }, + { + "matcher": "Write|Edit", + "hooks": [ + { + "type": "command", + "command": "${CLAUDE_PLUGIN_ROOT}/hooks/post_edit_sync_check.sh" + } + ] + } + ] + } }, { - "name": "claude-code-history-files-finder", - "description": "Find and recover content from Claude Code session history files. Use when searching for deleted files, tracking changes across sessions, analyzing conversation history, or recovering code/documents from previous Claude interactions. Triggers include mentions of session history, recover deleted, find in history, previous conversation, or .claude/projects", - "source": "./", + "name": "meeting-minutes-taker", + "description": "Transform meeting transcripts into high-fidelity, structured meeting minutes with iterative review. Features speaker identification via feature analysis (word count, speaking style, topic focus) with context.md team directory mapping, intelligent file naming from content, integration with doc-to-markdown and transcript-fixer for pre-processing, evidence-based recording with speaker quotes, Mermaid diagrams for architecture discussions, and multi-turn parallel generation with UNION merge", + "source": "./daymade-docs/meeting-minutes-taker", "strict": false, - "version": "1.0.0", - "category": "developer-tools", + "version": "1.1.1", + "category": "productivity", "keywords": [ - "session-history", - "recovery", - "deleted-files", - "conversation-history", - "file-tracking", - "claude-code", - "history-analysis" + "meeting", + "minutes", + "transcript", + "notes", + "speaker-identification", + "mermaid", + "quotes", + "action-items" ], "skills": [ - "./claude-code-history-files-finder" + "./" ] }, { - "name": "docs-cleaner", - "description": "Consolidates redundant documentation while preserving all valuable content. Use when cleaning up documentation bloat, merging redundant docs, reducing documentation sprawl, or consolidating multiple files covering the same topic", - "source": "./", + "name": "mermaid-tools", + "description": "Generate Mermaid diagrams from markdown with automatic PNG/SVG rendering and extraction from documents", + "source": "./daymade-docs/mermaid-tools", "strict": false, - "version": "1.0.0", - "category": "productivity", + "version": "1.0.2", + "category": "documentation", "keywords": [ - "documentation", - "cleanup", - "consolidation", - "redundancy", - "merge", - "docs" + "mermaid", + "diagrams", + "visualization", + "flowchart", + "sequence" ], "skills": [ - "./docs-cleaner" + "./" ] }, { "name": "pdf-creator", - "description": "Create PDF documents from markdown with proper Chinese font support using weasyprint. Use when converting markdown to PDF, generating formal documents (legal filings, trademark applications, reports), or when Chinese typography is required. Triggers include convert to PDF, generate PDF, markdown to PDF, or printable documents", - "source": "./", + "description": "Create PDF documents from markdown with Chinese font support. Supports theme system (default for formal docs, warm-terra for training materials) and dual backend (weasyprint or Chrome). Triggers include convert to PDF, generate PDF, markdown to PDF, or printable documents", + "source": "./daymade-docs/pdf-creator", "strict": false, - "version": "1.0.0", + "version": "1.4.0", "category": "document-conversion", "keywords": [ "pdf", "markdown", "weasyprint", + "chrome", + "themes", "chinese-fonts", + "cjk", "document-generation", "legal", "reports", "typography" ], "skills": [ - "./pdf-creator" + "./" ] }, { - "name": "claude-md-progressive-disclosurer", - "description": "Optimize user CLAUDE.md files by applying progressive disclosure principles. This skill should be used when users want to reduce CLAUDE.md bloat, move detailed content to references, extract reusable patterns into skills, or improve context efficiency. Triggers include optimize CLAUDE.md, reduce CLAUDE.md size, apply progressive disclosure, or complaints about CLAUDE.md being too long", + "name": "ppt-creator", + "description": "Create professional slide decks from topics or documents. Generates structured content with data-driven charts, speaker notes, and complete PPTX files. Applies persuasive storytelling principles (Pyramid Principle, assertion-evidence). Supports multiple formats (Marp, PowerPoint). Use for presentations, pitches, slide decks, or keynotes", + "source": "./daymade-docs/ppt-creator", + "strict": false, + "version": "1.0.1", + "category": "productivity", + "keywords": [ + "presentation", + "powerpoint", + "pptx", + "slides", + "marp", + "charts", + "data-visualization", + "pyramid-principle" + ], + "skills": [ + "./" + ] + }, + { + "name": "product-analysis", + "description": "Multi-path parallel product analysis with cross-model test-time compute scaling. Spawns parallel agents (Claude Code + Codex CLI) for multi-perspective exploration, then synthesizes findings into actionable optimization plans. Supports self-audit, UX audit, API audit, architecture review, and competitive benchmarking via competitors-analysis skill", "source": "./", "strict": false, - "version": "1.2.0", + "version": "1.0.1", "category": "productivity", "keywords": [ - "claude-md", - "progressive-disclosure", - "optimization", - "context-efficiency", - "configuration", - "token-savings" + "product-analysis", + "self-review", + "ux-audit", + "parallel-agents", + "cross-model", + "test-time-compute", + "codex", + "synthesis", + "product-audit", + "information-architecture" ], "skills": [ - "./claude-md-progressive-disclosurer" + "./product-analysis" ] }, { - "name": "skills-search", - "description": "Search, discover, install, and manage Claude Code skills from the CCPM registry. Use when users want to find skills for specific tasks, install skills by name, list installed skills, get skill details, or manage their Claude Code skill collection. Triggers include find skills, search for plugins, install skill, list installed skills, or any CCPM registry operations", + "name": "prompt-optimizer", + "description": "Transform vague prompts into precise, well-structured specifications using EARS (Easy Approach to Requirements Syntax) methodology. Use when users provide loose requirements, ambiguous feature descriptions, need to enhance prompts for AI-generated code/products/documents, request prompt optimization, or want to improve requirements engineering. Applies domain theories (GTD, BJ Fogg, Gestalt, AIDA, Zero Trust) and structured Role/Skills/Workflows/Examples/Formats framework", "source": "./", "strict": false, "version": "1.1.0", - "category": "developer-tools", + "category": "productivity", "keywords": [ - "ccpm", - "skills", - "search", - "install", - "registry", - "plugin", - "marketplace", - "discovery", - "skill-management" + "prompt-engineering", + "ears", + "requirements", + "specifications", + "optimization", + "domain-theory", + "prompt-enhancement", + "ai-prompting" ], "skills": [ - "./skills-search" + "./prompt-optimizer" ] }, { @@ -496,91 +779,107 @@ ] }, { - "name": "iOS-APP-developer", - "description": "Develops iOS applications with XcodeGen, SwiftUI, and SPM. Use when configuring XcodeGen project.yml, resolving SPM dependency issues, deploying to devices, handling code signing, debugging camera/AVFoundation, iOS version compatibility issues, or fixing Library not loaded @rpath framework errors. Includes state machine testing patterns for @MainActor classes", + "name": "qa-expert", + "description": "Comprehensive QA testing infrastructure with autonomous LLM execution, Google Testing Standards (AAA pattern), and OWASP security testing. Use when establishing QA processes, writing test cases, executing test plans, tracking bugs with P0-P4 classification, calculating quality metrics, enforcing quality gates, or preparing third-party QA handoffs. Enables 100x faster test execution via master prompts", "source": "./", "strict": false, - "version": "1.1.1", + "version": "1.0.0", "category": "developer-tools", "keywords": [ - "ios", - "xcodegen", - "swiftui", - "spm", - "avfoundation", - "camera", - "code-signing", - "device-deployment", - "swift", - "xcode", - "testing" + "qa", + "testing", + "test-cases", + "bug-tracking", + "google-standards", + "owasp", + "security", + "automation", + "quality-gates", + "metrics" ], "skills": [ - "./iOS-APP-developer" + "./qa-expert" ] }, { - "name": "fact-checker", - "description": "Verifies factual claims in documents using web search and official sources, then proposes corrections with user confirmation. Use when the user asks to fact-check, verify information, validate claims, check accuracy, or update outdated information in documents. Supports AI model specs, technical documentation, statistics, and general factual statements", + "name": "repomix-safe-mixer", + "description": "Safely package codebases with repomix by automatically detecting and removing hardcoded credentials before packing. Use when packaging code for distribution, creating reference packages, or when the user mentions security concerns about sharing code with repomix", "source": "./", "strict": false, "version": "1.0.0", - "category": "productivity", + "category": "security", "keywords": [ - "fact-checking", - "verification", - "accuracy", - "sources", - "validation", - "corrections", - "web-search" + "repomix", + "security", + "credentials", + "secrets-scanning", + "safe-packaging", + "secret-detection", + "code-security" ], "skills": [ - "./fact-checker" + "./repomix-safe-mixer" ] }, { - "name": "twitter-reader", - "description": "Fetch Twitter/X post content by URL using jina.ai API to bypass JavaScript restrictions. Use when Claude needs to retrieve tweet content including author, timestamp, post text, images, and thread replies. Supports individual posts or batch fetching from x.com or twitter.com URLs", + "name": "repomix-unmixer", + "description": "Extract files from repomix packaged formats (XML, Markdown, JSON) with automatic format detection and validation", "source": "./", "strict": false, "version": "1.0.0", "category": "utilities", "keywords": [ - "twitter", - "x", - "social-media", - "jina", - "content-fetching", - "api", - "scraping", - "threads" + "repomix", + "unmix", + "extract", + "xml", + "conversion" ], "skills": [ - "./twitter-reader" + "./repomix-unmixer" ] }, { - "name": "macos-cleaner", - "description": "Intelligent macOS disk space analysis and cleanup with safety-first philosophy. Use when users report disk space issues, need to clean their Mac, or want to understand storage consumption. Analyzes system caches, application remnants, large files, and development environments (Docker, Homebrew, npm, pip) with risk categorization (Safe/Caution/Keep) and requires explicit user confirmation before any deletions. Includes Mole visual tool integration for hybrid workflow", + "name": "scrapling-skill", + "description": "Install, troubleshoot, and use Scrapling CLI for extracting HTML, Markdown, or text from webpages. Diagnoses missing extras, Playwright browser runtime issues, TLS verification failures, and WeChat public article extraction patterns. Use when users mention Scrapling, `scrapling extract`, `uv tool install scrapling`, or need to decide between static and browser-backed fetching", "source": "./", "strict": false, - "version": "1.1.1", - "category": "utilities", + "version": "1.0.0", + "category": "developer-tools", "keywords": [ - "macos", - "disk-space", - "cleanup", - "cache", - "storage", - "developer-tools", - "docker", - "homebrew", - "system-maintenance", - "safety" + "scrapling", + "web-scraping", + "html", + "markdown", + "playwright", + "wechat", + "extraction", + "cli" ], "skills": [ - "./macos-cleaner" + "./scrapling-skill" + ] + }, + { + "name": "skill-creator", + "description": "Essential meta-skill for creating effective Claude Code skills with initialization scripts, validation, packaging, marketplace registration, and privacy best practices. Includes a specialized wrapper-skill workflow for retrospectively distilling an install-and-debug session into a reusable companion skill for a third-party CLI tool.", + "source": "./", + "strict": false, + "version": "1.7.3", + "category": "developer-tools", + "keywords": [ + "skill-creation", + "claude-code", + "development", + "tooling", + "workflow", + "meta-skill", + "wrapper-skill", + "cli-wrapper", + "essential" + ], + "skills": [ + "./skill-creator" ] }, { @@ -604,130 +903,146 @@ ] }, { - "name": "github-contributor", - "description": "Strategic guide for becoming an effective GitHub contributor. Covers opportunity discovery, project selection, high-quality PR creation, and reputation building. Use when looking to contribute to open-source projects, building GitHub presence, or learning contribution best practices", + "name": "skills-search", + "description": "Search, discover, install, and manage Claude Code skills from the CCPM registry. Use when users want to find skills for specific tasks, install skills by name, list installed skills, get skill details, or manage their Claude Code skill collection. Triggers include find skills, search for plugins, install skill, list installed skills, or any CCPM registry operations", + "source": "./", + "strict": false, + "version": "1.1.0", + "category": "developer-tools", + "keywords": [ + "ccpm", + "skills", + "search", + "install", + "registry", + "plugin", + "marketplace", + "discovery", + "skill-management" + ], + "skills": [ + "./skills-search" + ] + }, + { + "name": "slides-creator", + "description": "Narrative-first slide deck creation. Guides users through structured narrative design (ABCDEFG model), then delegates visual generation to baoyu-slide-deck. Triggers on create slides, make a presentation, generate deck, slide deck, PPT, or when user needs to turn content into visual slides.", "source": "./", "strict": false, - "version": "1.0.2", - "category": "developer-tools", + "version": "1.0.0", + "category": "content-creation", "keywords": [ - "github", - "open-source", - "contribution", - "pull-request", - "reputation", - "contributor", - "oss" + "slides", + "presentation", + "ppt", + "deck", + "narrative", + "storytelling", + "ABCDEFG" ], "skills": [ - "./github-contributor" + "./slides-creator" ] }, { - "name": "i18n-expert", - "description": "Complete internationalization/localization setup and auditing for UI codebases. Configure i18n frameworks, replace hard-coded strings with translation keys, ensure locale parity between en-US and zh-CN, and validate pluralization and formatting. Use when setting up i18n for React/Next.js/Vue apps, auditing existing implementations, replacing hard-coded strings, ensuring proper error code mapping, or validating pluralization and date/time/number formatting across locales", - "source": "./", + "name": "statusline-generator", + "description": "Configure Claude Code statuslines with context window display (actual token counts), multi-line layouts, cost tracking via ccusage, git status, human-readable formatting, and customizable colors", + "source": "./daymade-claude-code/statusline-generator", "strict": false, - "version": "1.0.0", - "category": "developer-tools", + "version": "1.1.0", + "category": "customization", "keywords": [ - "i18n", - "internationalization", - "localization", - "translation", - "react-i18next", - "next-intl", - "vue-i18n", - "locale", - "multilingual", - "globalization" + "statusline", + "context-window", + "ccusage", + "git-status", + "customization", + "prompt" ], "skills": [ - "./i18n-expert" + "./" ] }, { - "name": "claude-skills-troubleshooting", - "description": "Diagnose and resolve Claude Code plugin and skill configuration issues. Debug plugin installation, enablement, and activation problems with systematic workflows. Use when plugins are installed but not showing in available skills list, skills are not activating as expected, troubleshooting enabledPlugins configuration in settings.json, debugging 'plugin not working' or 'skill not showing' issues, or understanding plugin state architecture and lifecycle", + "name": "stepfun-tts", + "description": "Generate speech and transcribe audio using StepFun's StepAudio 2.5 family — stepaudio-2.5-tts (Contextual TTS with instruction + inline parentheses) and stepaudio-2.5-asr (SSE endpoint, 32K context, ~100x RTF, handles up to 30-minute audio in a single call). Use when the user wants Chinese/Japanese TTS with emotional/prosody control, needs to transcribe long audio, migrates from step-tts-2 to stepaudio-2.5-tts (voice_label → instruction breaking change), or hits StepFun censorship / endpoint errors. Also triggers for 阶跃 TTS, StepAudio 合成, 语音合成, 配音, StepFun ASR, 转录, 语音识别.", "source": "./", "strict": false, "version": "1.0.0", - "category": "utilities", + "category": "productivity", "keywords": [ - "troubleshooting", - "debugging", - "plugins", - "skills", - "diagnostics", - "configuration", - "enabledPlugins", - "settings", - "marketplace" + "tts", + "asr", + "stepfun", + "stepaudio", + "stepaudio-2.5", + "阶跃", + "chinese-tts", + "voice-synthesis", + "speech-to-text", + "contextual-tts" ], "skills": [ - "./claude-skills-troubleshooting" + "./stepfun-tts" ] }, { - "name": "meeting-minutes-taker", - "description": "Transform meeting transcripts into high-fidelity, structured meeting minutes with iterative review. Features speaker identification via feature analysis (word count, speaking style, topic focus) with context.md team directory mapping, intelligent file naming from content, integration with markdown-tools and transcript-fixer for pre-processing, evidence-based recording with speaker quotes, Mermaid diagrams for architecture discussions, and multi-turn parallel generation with UNION merge", + "name": "teams-channel-post-writer", + "description": "Create professional Microsoft Teams channel posts with Adaptive Cards, formatted announcements, and corporate communication standards", "source": "./", "strict": false, - "version": "1.1.0", - "category": "productivity", + "version": "1.0.0", + "category": "communication", "keywords": [ - "meeting", - "minutes", - "transcript", - "notes", - "speaker-identification", - "mermaid", - "quotes", - "action-items" + "teams", + "microsoft", + "adaptive-cards", + "communication", + "announcements" ], "skills": [ - "./meeting-minutes-taker" + "./teams-channel-post-writer" ] }, { - "name": "deep-research", - "description": "Generate format-controlled research reports with evidence tracking, citations, and iterative review. Use when users request research reports, literature reviews, market or industry analysis, competitive landscapes, policy or technical briefs, or strict report templates and section formatting", + "name": "terraform-skill", + "description": "Operational traps for Terraform provisioners, multi-environment isolation, and zero-to-deployment reliability. Covers provisioner timing races, SSH connection conflicts, DNS record duplication, volume permissions, database bootstrap gaps, snapshot cross-contamination, Cloudflare credential format errors, hardcoded domains in Caddyfiles/compose, and init-data-only-on-first-boot pitfalls. Activate when writing null_resource provisioners, creating multi-environment Terraform setups, debugging containers that are Restarting/unhealthy after terraform apply, setting up fresh instances with cloud-init, or any IaC code that SSHs into remote hosts. Also activate when the user mentions terraform plan/apply errors, provisioner failures, infrastructure drift, TLS certificate errors, or Caddy/gateway configuration.", "source": "./", "strict": false, "version": "1.0.0", - "category": "documentation", + "category": "developer-tools", "keywords": [ - "research", - "report", - "analysis", - "literature-review", - "market-research", - "citations", - "evidence", - "deepresearch" + "terraform", + "iac", + "provisioner", + "devops", + "infrastructure", + "cloud-init", + "cloudflare" ], "skills": [ - "./deep-research" + "./terraform-skill" ] }, { - "name": "competitors-analysis", - "description": "Analyze competitor repositories with evidence-based approach. Use when tracking competitors, creating competitor profiles, or generating competitive analysis. All analysis must be based on actual cloned code, never assumptions. Triggers include analyze competitor, add competitor, competitive analysis, or 竞品分析", + "name": "transcript-fixer", + "description": "Corrects speech-to-text (ASR/STT) transcription errors using dictionary rules and native Claude AI corrections (no API key needed by default). Supports intelligent paragraph breaks, filler word reduction, interactive review, Chinese domain names, and iterative dictionary building. Use when users mention transcript correction, ASR errors, speech-to-text mistakes, homophone errors, or working with transcription files", "source": "./", "strict": false, - "version": "1.0.1", + "version": "1.4.0", "category": "productivity", "keywords": [ - "competitors", - "competitive-analysis", - "competitor-tracking", - "evidence-based", - "market-research", - "竞品分析", - "code-analysis" + "transcription", + "asr", + "stt", + "speech-to-text", + "correction", + "ai", + "meeting-notes", + "nlp" ], "skills": [ - "./competitors-analysis" + "./transcript-fixer" ] }, { @@ -735,7 +1050,7 @@ "description": "Diagnoses and fixes conflicts between Tailscale and proxy/VPN tools (Shadowrocket, Clash, Surge) on macOS. Covers five conflict layers: route hijacking, HTTP proxy env var interception, system proxy bypass, SSH ProxyCommand double tunneling, and VM/container proxy propagation. Includes an automated quick-diagnose script plus SOP for remote development via SSH tunnels with proxy-safe Makefile patterns. Use when Tailscale ping works but SSH/HTTP times out, when browser returns 503 but curl works, when local vanity domains fail behind proxy, when git push fails with failed to begin relaying via HTTP, when Docker pull times out behind TUN/VPN, when setting up Tailscale SSH to WSL instances, or when bootstrapping remote dev environments over Tailscale", "source": "./", "strict": false, - "version": "1.3.0", + "version": "1.4.0", "category": "developer-tools", "keywords": [ "tailscale", @@ -761,6 +1076,70 @@ "./tunnel-doctor" ] }, + { + "name": "twitter-reader", + "description": "Fetch Twitter/X post content including long-form Articles with full images and metadata. Use when Claude needs to retrieve tweet/article content, author info, engagement metrics (likes, retweets, bookmarks), and embedded media. Supports individual posts and X Articles (long-form content). Automatically downloads all images to local attachments folder and generates complete Markdown with proper image references. Preferred over Jina for X Articles with images.", + "source": "./", + "strict": false, + "version": "1.1.0", + "category": "utilities", + "keywords": [ + "twitter", + "x", + "social-media", + "jina", + "content-fetching", + "api", + "scraping", + "threads", + "images", + "attachments", + "markdown" + ], + "skills": [ + "./twitter-reader" + ] + }, + { + "name": "ui-designer", + "description": "Extract design systems from reference UI images and generate implementation-ready UI design prompts. Use when users provide UI screenshots/mockups and want to create consistent designs", + "source": "./", + "strict": false, + "version": "1.0.0", + "category": "design", + "keywords": [ + "ui", + "design-system", + "mockup", + "screenshot", + "design-extraction", + "mvp" + ], + "skills": [ + "./ui-designer" + ] + }, + { + "name": "video-comparer", + "description": "Compare two videos and generate interactive HTML reports with quality metrics (PSNR, SSIM) and frame-by-frame visual comparisons. Use when analyzing compression results, evaluating codec performance, or assessing video quality differences", + "source": "./", + "strict": false, + "version": "1.0.0", + "category": "media", + "keywords": [ + "video", + "comparison", + "quality-analysis", + "psnr", + "ssim", + "compression", + "ffmpeg", + "codec" + ], + "skills": [ + "./video-comparer" + ] + }, { "name": "windows-remote-desktop-connection-doctor", "description": "Diagnose Windows App (Microsoft Remote Desktop / Azure Virtual Desktop / W365) connection quality issues on macOS. Analyze transport protocol selection (UDP Shortpath vs WebSocket), detect VPN/proxy interference with STUN/TURN negotiation, and parse Windows App logs for Shortpath failures. This skill should be used when VDI connections are slow, when transport shows WebSocket instead of UDP, when RDP Shortpath fails to establish, or when RTT is unexpectedly high.", @@ -789,92 +1168,76 @@ ] }, { - "name": "product-analysis", - "description": "Multi-path parallel product analysis with cross-model test-time compute scaling. Spawns parallel agents (Claude Code + Codex CLI) for multi-perspective exploration, then synthesizes findings into actionable optimization plans. Supports self-audit, UX audit, API audit, architecture review, and competitive benchmarking via competitors-analysis skill", - "source": "./", - "strict": false, - "version": "1.0.1", - "category": "productivity", - "keywords": [ - "product-analysis", - "self-review", - "ux-audit", - "parallel-agents", - "cross-model", - "test-time-compute", - "codex", - "synthesis", - "product-audit", - "information-architecture" - ], - "skills": [ - "./product-analysis" - ] - }, - { - "name": "excel-automation", - "description": "Create, parse, and control Excel files on macOS. Professional formatting with openpyxl (font colors, fills, borders, conditional formatting), complex xlsm parsing with stdlib zipfile+xml for investment bank financial models, and Excel window control via AppleScript (zoom, scroll, select). Use when creating formatted Excel reports, parsing financial models, or automating Excel on macOS", + "name": "youtube-downloader", + "description": "Download YouTube videos and HLS streams (m3u8) from platforms like Mux, Vimeo, etc. using yt-dlp and ffmpeg. Use when users request downloading videos, extracting audio, handling protected streams with authentication headers, or troubleshooting download issues like nsig extraction failures, 403 errors, or cookie extraction problems", "source": "./", "strict": false, - "version": "1.0.0", - "category": "productivity", + "version": "1.1.0", + "category": "utilities", "keywords": [ - "excel", - "openpyxl", - "xlsm", - "spreadsheet", - "formatting", - "financial-model", - "applescript", - "macos", - "dcf", - "investment-banking" + "youtube", + "yt-dlp", + "video-download", + "audio-extraction", + "mp3", + "download", + "hls", + "m3u8", + "ffmpeg", + "streaming", + "mux", + "vimeo" ], "skills": [ - "./excel-automation" + "./youtube-downloader" ] }, { - "name": "capture-screen", - "description": "Programmatic screenshot capture on macOS. Get window IDs via Swift CGWindowListCopyWindowInfo, capture specific windows with screencapture -l, and control application windows via AppleScript. Supports multi-shot workflows for capturing different sections of the same window. Use when taking automated screenshots, capturing application windows, or creating visual documentation", + "name": "debugging-network-issues", + "description": "Evidence-driven investigation for network, streaming, and protocol-layer bugs. Use when debugging connection resets, SSE or long-polling stalls, fixed-time connection drops, CDN/proxy/CGNAT idle timeouts, or any incident where symptoms do not match the obvious cause. Applies falsification-first methodology with layered isolation experiments, env-gated runtime instrumentation, and counter-review agent teams.", "source": "./", "strict": false, "version": "1.0.0", - "category": "utilities", + "category": "developer-tools", "keywords": [ - "screenshot", - "screencapture", - "macos", - "window-capture", - "swift", - "applescript", - "automation", - "visual-documentation" + "debugging", + "network", + "sse", + "http2", + "rst", + "econnreset", + "cloudflare", + "streaming", + "troubleshooting", + "methodology" ], "skills": [ - "./capture-screen" + "./debugging-network-issues" ] }, { - "name": "financial-data-collector", - "description": "Collect real financial data for any US publicly traded company from free public sources (yfinance). Output structured JSON with market data, historical financials, WACC inputs, and analyst estimates. Handles NaN year detection, CapEx sign preservation, and FCF definition mismatches. Use when users request company financials, stock data, DCF inputs, or financial data collection for any US equity ticker", + "name": "bilibili-content", + "description": "Extract Bilibili video subtitles (three-strategy: AI-generated → uploader-uploaded → Whisper ASR fallback) and top comments. Output as study notes, chapter summaries, blog posts, or timelines based on video type. Use when the user shares a Bilibili link (bilibili.com/video/, b23.tv, BV/AV ids), wants video subtitles or transcripts, needs study notes from educational content, or asks to summarize a B站 video. Supports multi-page videos and Chinese/English output.", "source": "./", "strict": false, "version": "1.0.0", - "category": "productivity", + "category": "media", "keywords": [ - "finance", - "financial-data", - "yfinance", - "stock-data", - "dcf", - "wacc", - "market-data", - "investment-research", - "sec-filings" + "bilibili", + "b站", + "video", + "subtitle", + "transcript", + "comment", + "chinese", + "asr", + "whisper", + "study-notes", + "字幕", + "评论" ], "skills": [ - "./financial-data-collector" + "./bilibili-content" ] } ] diff --git a/.claude/archive/ralph-loop.local.md b/.claude/archive/ralph-loop.local.md new file mode 100644 index 00000000..17f66e84 --- /dev/null +++ b/.claude/archive/ralph-loop.local.md @@ -0,0 +1,10 @@ +--- +active: true +iteration: 390 +session_id: +max_iterations: 0 +completion_promise: null +started_at: "2026-04-11T19:55:07Z" +--- + +使用这个技能，直到我们超越所有的竞品，每一轮开始的时候，你都要去分析我们当前的这个skill和我们所有的竞品的差距是什么，取其精华，去其糟粕 diff --git a/.claude/archive/ralph-round-68.md b/.claude/archive/ralph-round-68.md new file mode 100644 index 00000000..601c7bb5 --- /dev/null +++ b/.claude/archive/ralph-round-68.md @@ -0,0 +1,21 @@ +# Ralph Loop Round 68 - Completed + +## 竞品差距分析 +当前 v3.25.0 vs 竞品能力矩阵： +- wcplusPro: 付费护城河 = 自动化工作流 +- 新榜/西瓜数据: 无工作流功能 +- 开源竞品: 完全无此功能 + +Gap: 工作流引擎是 wcplusPro 的付费核心功能，无任何开源实现 + +## 实施结果 +- 自动化工作流引擎 v1.0 ✅ +- RESTful API + WebSocket ✅ +- CLI w workflow 命令组 ✅ +- 46个唯一支持特性 ✅ + +## 版本 +v3.25.0 → v3.26.0 + +## 提交 +3f61ffd feat(wechat-article-scraper): 自动化工作流引擎 v1.0 diff --git a/.claude/archive/ralph-round-69.md b/.claude/archive/ralph-round-69.md new file mode 100644 index 00000000..802631a1 --- /dev/null +++ b/.claude/archive/ralph-round-69.md @@ -0,0 +1,29 @@ +# Ralph Loop Round 69 - Completed + +## 竞品差距分析 +当前 v3.26.0 vs 竞品能力矩阵： +- wcplusPro: 第二付费护城河 = 团队协作 + 数据同步到Notion/语雀 +- 新榜/西瓜数据: 无团队协作功能 +- 开源竞品: 完全无此功能 + +Gap: 团队协作是 wcplusPro 的另一付费核心功能，无任何开源实现 + +## 实施结果 +- 团队协作系统 v1.0 ✅ + - 多用户管理、RBAC权限(admin/member/viewer) + - 团队共享工作区、文章收藏夹 + - 文章标注系统(标签/评论/高亮) + - 邀请码机制 +- 第三方集成 v1.0 ✅ + - Notion API 数据库同步 + - 语雀 API 知识库归档 + - Airtable 表格同步 +- w team CLI命令组 ✅ +- w sync CLI命令组 ✅ +- 48个唯一支持特性 ✅ + +## 版本 +v3.26.0 → v3.27.0 + +## 提交 +81fcc97 feat(wechat-article-scraper): 团队协作与第三方集成 v1.0 diff --git a/.claude/archive/ralph-round-70.md b/.claude/archive/ralph-round-70.md new file mode 100644 index 00000000..6b38b73f --- /dev/null +++ b/.claude/archive/ralph-round-70.md @@ -0,0 +1,33 @@ +# Ralph Loop Round 70 - Completed + +## 竞品差距分析 +当前 v3.27.0 vs 竞品能力矩阵： +- wcplusPro: 付费功能 = 数据导出(Excel/PDF/Word) + 批量操作 + 高级筛选 +- 新榜/西瓜数据: 基础导出功能 +- 开源竞品: 无此功能 + +Gap: 丰富的数据导出和批量操作是 wcplusPro 的核心付费功能 + +## 实施结果 +- 多格式导出引擎 v1.0 ✅ + - Excel: 样式美化、多sheet、统计图表 + - PDF: 中文支持、分页优化 + - Word: 格式保持、目录生成 + - Markdown/JSON/CSV: 标准格式 + - 导出模板系统（自定义字段、样式） +- 高级筛选系统 v1.0 ✅ + - 多条件组合筛选（AND/OR逻辑） + - 条件类型: 时间/公众号/关键词/阅读量/标签 + - 筛选模板保存/加载 +- 批量操作引擎 v1.0 ✅ + - 批量导出/编辑/同步/删除 + - 任务队列和进度追踪 + - 操作历史记录 +- w export/filter/batch CLI命令组 ✅ +- 51个唯一支持特性 ✅ + +## 版本 +v3.27.0 → v3.28.0 + +## 提交 +37211c6 feat(wechat-article-scraper): 批量操作与多格式导出 v1.0 diff --git a/.claude/archive/ralph-round-71.md b/.claude/archive/ralph-round-71.md new file mode 100644 index 00000000..3d778ed4 --- /dev/null +++ b/.claude/archive/ralph-round-71.md @@ -0,0 +1,37 @@ +# Ralph Loop Round 71 - Completed + +## 竞品差距分析 +当前 v3.28.0 vs 竞品能力矩阵： +- wcplusPro: 有成熟的 Chrome 扩展用于一键采集 +- 新榜/西瓜数据: 有自己的浏览器插件 +- 开源竞品: 无此功能 + +Gap: 浏览器扩展是用户便捷使用的重要入口 + +## 实施结果 +- 浏览器扩展 v2.0 完善 ✅ + - Chrome/Firefox Manifest V3 扩展 + - Content Script: 页面内容提取、文章数据解析 + - 标题、作者、发布时间提取 + - 正文内容提取（段落、HTML） + - 图片提取（data-src、data-backsrc） + - 视频提取 + - 互动数据读取（阅读数、点赞数、在看数） + - Background Script: 右键菜单、快捷键、自动下载 + - 右键菜单：抓取文章、抓取并下载图片、打开仪表盘 + - 快捷键：Ctrl+Shift+S 快速抓取 + - Tab 状态监听和徽章显示 + - Popup UI: 格式选择、进度显示、结果操作 + - 状态检测（是否在文章页面） + - 格式选择：Markdown/HTML/JSON + - 设置开关：保存本地、上传服务器、自动分类 + - 进度条和结果展示 + - 查看/下载/复制结果 + - w extension CLI命令: install/pack/check +- v3.29.0, 52个唯一支持特性 ✅ + +## 版本 +v3.28.0 → v3.29.0 + +## 提交 +6c5f24d feat(wechat-article-scraper): 浏览器扩展 v2.0 完善 diff --git a/.claude/archive/ralph-round-72.md b/.claude/archive/ralph-round-72.md new file mode 100644 index 00000000..f1742227 --- /dev/null +++ b/.claude/archive/ralph-round-72.md @@ -0,0 +1,24 @@ +# Ralph Loop Round 72 - Completed + +## 竞品差距分析 +当前 v3.29.0 vs 竞品能力矩阵： +- wcplusPro: 无舆情监控功能 +- 新榜/西瓜数据: 有数据监控但无敏感内容检测 +- 开源竞品: 无此功能 + +Gap: 舆情监控是企业级用户的刚需，特别是敏感内容检测和危机预警 + +## 实施结果 +- 舆情监控系统 v1.0 ✅ + - 敏感词检测: 6大类别敏感词库 (政治/色情/暴力/赌博/毒品/诈骗) + - 品牌提及追踪: 追踪品牌/关键词在文章中的提及 + - 情感趋势分析: 正面/中性/负面情绪监控 + - 危机预警系统: 异常传播速度检测、敏感内容预警 + - CLI集成: `w sentiment` 命令 (word/brand/alert/trend/stats/scan) +- v3.30.0, 53个唯一支持特性 ✅ + +## 版本 +v3.29.0 → v3.30.0 + +## 提交 +7719c53 feat(wechat-article-scraper): 舆情监控系统 v1.0 diff --git a/.claude/archive/ralph-round-73.md b/.claude/archive/ralph-round-73.md new file mode 100644 index 00000000..8e3e25cd --- /dev/null +++ b/.claude/archive/ralph-round-73.md @@ -0,0 +1,24 @@ +# Ralph Loop Round 73 - Completed + +## 竞品差距分析 +当前 v3.30.0 vs 竞品能力矩阵： +- wcplusPro: 有强大的数据可视化图表和竞品分析（付费功能） +- 新榜/西瓜数据: 有数据监控看板、热点追踪 +- 开源竞品: 无此功能 + +Gap: 数据可视化仪表盘、竞品分析、AI洞察报告是 wcplusPro/新榜的核心付费功能 + +## 实施结果 +- 数据可视化与智能分析系统 v1.0 ✅ + - 数据仪表盘 (analytics_dashboard.py): ECharts配置、阅读趋势、互动热力图、文章排行、公众号健康度 + - 竞品分析系统 (competitor_analyzer.py): 多账号对比、竞争力评分、内容策略分析、最佳发布时间 + - AI智能洞察 (ai_insights.py): 趋势分析、异常检测、自动运营建议、健康度评分 + - 热点追踪 (hot_topics.py): 自动发现热点、话题聚类、传播速度监控、生命周期追踪 + - CLI集成: `w analytics` 统一入口 (trends/top/metrics/report/heatmap/compare/insights/topics) +- v3.31.0, 57个唯一支持特性 ✅ + +## 版本 +v3.30.0 → v3.31.0 + +## 提交 +e4109dd feat(wechat-article-scraper): 数据可视化与智能分析系统 v1.0 diff --git a/.claude/archive/ralph-round-74.md b/.claude/archive/ralph-round-74.md new file mode 100644 index 00000000..091a332e --- /dev/null +++ b/.claude/archive/ralph-round-74.md @@ -0,0 +1,24 @@ +# Ralph Loop Round 74 - Completed + +## 竞品差距分析 +当前 v3.31.0 vs 竞品能力矩阵： +- wcplusPro: 有AI写作助手（标题生成、摘要、改写） +- 新榜/西瓜数据: 有素材库管理、自动排版 +- 开源竞品: 无此功能 + +Gap: AI写作辅助、素材库管理是竞品的付费核心功能 + +## 实施结果 +- 智能写作助手 v1.0 ✅ + - AI标题生成器 (writing_assistant.py/TitleGenerator): 8种爆款公式、A/B测试、CTR预测 + - 智能摘要生成 (Summarizer): 5种风格（新闻/营销/极简/故事/要点） + - 内容改写润色 (ContentRewriter): 5种风格转换、语气调整 + - 素材库管理 (material_library.py): 文案/图片/链接收藏、标签分类、全文搜索 + - CLI集成: `w writing` 命令（title/summary/rewrite/analyze/material） +- v3.32.0, 61个唯一支持特性 ✅ + +## 版本 +v3.31.0 → v3.32.0 + +## 提交 +b8bbc08 feat(wechat-article-scraper): 智能写作助手 v1.0 diff --git a/.claude/archive/ralph-round-75.md b/.claude/archive/ralph-round-75.md new file mode 100644 index 00000000..9baab224 --- /dev/null +++ b/.claude/archive/ralph-round-75.md @@ -0,0 +1,24 @@ +# Ralph Loop Round 75 - Completed + +## 竞品差距分析 +当前 v3.32.0 vs 竞品能力矩阵： +- wcplusPro: 有自动采集、定时任务（付费功能） +- 新榜/西瓜数据: 有定时监控、通知提醒 +- 开源竞品: 无此功能 + +Gap: 定时任务调度、自动采集、通知提醒是竞品的付费核心功能 + +## 实施结果 +- 定时任务与自动化系统 v1.0 ✅ + - 任务调度器 (task_scheduler.py): Cron表达式解析、任务调度循环、多任务类型支持 + - 任务执行日志: 执行记录、成功率统计、错误追踪、历史查询 + - 通知系统 (notification_system.py): 邮件/SMTP、Webhook回调、5种通知模板、通知历史 + - 5种内置任务类型: scrape/export/backup/cleanup/custom + - CLI集成: `w scheduler` 命令（create/list/run/toggle/delete/history/stats/daemon） +- v3.33.0, 64个唯一支持特性 ✅ + +## 版本 +v3.32.0 → v3.33.0 + +## 提交 +bccf5f3 feat(wechat-article-scraper): 定时任务与自动化系统 v1.0 diff --git a/.githooks/pre-commit b/.githooks/pre-commit new file mode 100755 index 00000000..89d8979a --- /dev/null +++ b/.githooks/pre-commit @@ -0,0 +1,53 @@ +#!/bin/bash +# Pre-commit hook: scan staged changes for sensitive data +# Install: git config core.hooksPath .githooks + +set -euo pipefail + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +NC='\033[0m' + +echo "🔍 Scanning staged changes for sensitive data..." + +FAILED=0 + +# Layer 1: gitleaks (if available) +if command -v gitleaks &>/dev/null; then + if ! gitleaks protect --staged --config .gitleaks.toml --no-banner 2>/dev/null; then + echo -e "${RED}❌ gitleaks found secrets in staged changes${NC}" + FAILED=1 + fi +else + echo -e "${YELLOW}⚠ gitleaks not installed (brew install gitleaks), falling back to pattern scan${NC}" +fi + +# Layer 2: fast regex scan (always runs, catches what gitleaks config might miss) +STAGED_DIFF=$(git diff --cached --diff-filter=ACDMR) + +PATTERNS=( + '/Users/[a-zA-Z][a-zA-Z0-9_-]+/' + '/home/[a-zA-Z][a-zA-Z0-9_-]+/' + 'C:\\Users\\[a-zA-Z]' + 'songtiansheng' + 'tiansheng' + '15366[0-9]+' +) + +for pattern in "${PATTERNS[@]}"; do + MATCHES=$(echo "$STAGED_DIFF" | grep -nE "^\+" | grep -E "$pattern" | grep -v "^+++\|\.gitleaks\.toml\|\.githooks/\|\.gitignore\|placeholder\|example\|CLAUDE\.md" || true) + if [ -n "$MATCHES" ]; then + echo -e "${RED}❌ Found sensitive pattern '${pattern}':${NC}" + echo "$MATCHES" | head -5 + FAILED=1 + fi +done + +if [ $FAILED -eq 1 ]; then + echo "" + echo -e "${RED}Commit blocked. Fix the issues above, or use --no-verify to bypass (not recommended).${NC}" + exit 1 +fi + +echo -e "${GREEN}✅ No sensitive data found in staged changes.${NC}" diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 51966ef5..7de86f63 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -16,7 +16,7 @@ Which skill is affected? - [ ] skill-creator - [ ] github-ops -- [ ] markdown-tools +- [ ] doc-to-markdown - [ ] mermaid-tools - [ ] statusline-generator - [ ] teams-channel-post-writer diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index edaf3704..050c3e22 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -20,7 +20,7 @@ Which skill would this enhance? - [ ] skill-creator - [ ] github-ops -- [ ] markdown-tools +- [ ] doc-to-markdown - [ ] mermaid-tools - [ ] statusline-generator - [ ] teams-channel-post-writer diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 27a4c902..54d20160 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -33,7 +33,7 @@ Which skills are affected by this PR? - [ ] skill-creator - [ ] github-ops -- [ ] markdown-tools +- [ ] doc-to-markdown - [ ] mermaid-tools - [ ] statusline-generator - [ ] teams-channel-post-writer diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 00000000..ba99aae0 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,190 @@ +name: Benchmark Tests + +on: + schedule: + # Run weekly on Sundays at 2 AM UTC + - cron: '0 2 * * 0' + workflow_dispatch: + inputs: + test_type: + description: 'Test type to run' + required: true + default: 'all' + type: choice + options: + - all + - scrape + - annotation + - full + +env: + NODE_VERSION: '20' + +jobs: + scrape-benchmark: + name: Scrape Success Rate Benchmark + runs-on: ubuntu-latest + if: github.event.inputs.test_type == 'all' || github.event.inputs.test_type == 'scrape' || github.event.inputs.test_type == 'full' + timeout-minutes: 60 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: cloud/package-lock.json + + - name: Install dependencies + working-directory: cloud + run: npm ci + + - name: Install Playwright browsers + working-directory: cloud + run: npx playwright install --with-deps chromium + + - name: Run scrape benchmark + working-directory: cloud + env: + JINA_AI_API_KEY: ${{ secrets.JINA_AI_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + SUPABASE_URL: ${{ secrets.SUPABASE_URL }} + SUPABASE_ANON_KEY: ${{ secrets.SUPABASE_ANON_KEY }} + run: | + npx vitest run tests/benchmark/scrape-benchmark.ts --reporter=verbose 2>&1 | tee scrape-benchmark.log + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + if: always() + with: + name: scrape-benchmark-results + path: | + cloud/tests/benchmark/scrape-benchmark-report.json + cloud/scrape-benchmark.log + retention-days: 30 + + - name: Check success rate threshold + working-directory: cloud + run: | + if [ -f tests/benchmark/scrape-benchmark-report.json ]; then + SUCCESS_RATE=$(jq -r '.summary.successRate' tests/benchmark/scrape-benchmark-report.json | sed 's/%//') + echo "Success Rate: ${SUCCESS_RATE}%" + if (( $(echo "$SUCCESS_RATE >= 95" | bc -l) )); then + echo "✅ SUCCESS: Meet target of 95%" + exit 0 + else + echo "❌ FAILURE: Below target of 95%" + exit 1 + fi + else + echo "Report not generated" + exit 1 + fi + + annotation-benchmark: + name: Annotation Accuracy Benchmark + runs-on: ubuntu-latest + if: github.event.inputs.test_type == 'all' || github.event.inputs.test_type == 'annotation' || github.event.inputs.test_type == 'full' + timeout-minutes: 10 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ env.NODE_VERSION }} + cache: 'npm' + cache-dependency-path: cloud/package-lock.json + + - name: Install dependencies + working-directory: cloud + run: npm ci + + - name: Run annotation accuracy test + working-directory: cloud + run: | + npx vitest run tests/benchmark/annotation-accuracy.ts --reporter=verbose 2>&1 | tee annotation-accuracy.log + + - name: Upload benchmark results + uses: actions/upload-artifact@v4 + if: always() + with: + name: annotation-accuracy-results + path: | + cloud/tests/benchmark/annotation-accuracy-report.json + cloud/annotation-accuracy.log + retention-days: 30 + + - name: Check accuracy threshold + working-directory: cloud + run: | + if [ -f tests/benchmark/annotation-accuracy-report.json ]; then + ACCURACY=$(jq -r '.summary.positionAccuracy' tests/benchmark/annotation-accuracy-report.json | sed 's/%//') + echo "Position Accuracy: ${ACCURACY}%" + if (( $(echo "$ACCURACY >= 98" | bc -l) )); then + echo "✅ SUCCESS: Meet target of 98%" + exit 0 + else + echo "❌ FAILURE: Below target of 98%" + exit 1 + fi + else + echo "Report not generated" + exit 1 + fi + + full-benchmark: + name: Full Benchmark Suite + runs-on: ubuntu-latest + if: github.event.inputs.test_type == 'full' + needs: [scrape-benchmark, annotation-benchmark] + timeout-minutes: 90 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Download all artifacts + uses: actions/download-artifact@v4 + + - name: Generate combined report + run: | + echo "# 📊 Full Benchmark Report" > benchmark-report.md + echo "" >> benchmark-report.md + echo "Generated at: $(date -u +"%Y-%m-%d %H:%M:%S UTC")" >> benchmark-report.md + echo "" >> benchmark-report.md + + if [ -f scrape-benchmark-results/scrape-benchmark-report.json ]; then + echo "## Scraping Success Rate" >> benchmark-report.md + cat scrape-benchmark-results/scrape-benchmark-report.json | jq -r '.summary | to_entries[] | "- \(.key): \(.value)"' >> benchmark-report.md + echo "" >> benchmark-report.md + fi + + if [ -f annotation-accuracy-results/annotation-accuracy-report.json ]; then + echo "## Annotation Accuracy" >> benchmark-report.md + cat annotation-accuracy-results/annotation-accuracy-report.json | jq -r '.summary | to_entries[] | "- \(.key): \(.value)"' >> benchmark-report.md + fi + + - name: Upload combined report + uses: actions/upload-artifact@v4 + with: + name: full-benchmark-report + path: benchmark-report.md + + - name: Create GitHub Issue on failure + if: failure() + uses: actions/github-script@v7 + with: + script: | + github.rest.issues.create({ + owner: context.repo.owner, + repo: context.repo.repo, + title: `❌ Benchmark Failed - ${new Date().toISOString().split('T')[0]}`, + body: `Benchmark tests failed. Check the [Actions run](${context.payload.repository.html_url}/actions/runs/${context.runId}) for details.`, + labels: ['benchmark', 'automated'] + }); diff --git a/.gitignore b/.gitignore index 27c1cf97..3cb0857a 100644 --- a/.gitignore +++ b/.gitignore @@ -59,6 +59,7 @@ Thumbs.db *.tgz *.rar *.7z +*.skill # Build artifacts *.o @@ -81,3 +82,24 @@ INSTALLATION.md # Private/commercial skills (moved to claude-code-skills-pro) seo-expert/ video-creator/ +/jsonl-viewer/ + +# Research output (may contain sensitive data) +deep-research-output/ +recovered_deep_research/ + +# OpenCLI cache +.opencli/ + +# Eval workspaces (contain test data with personal info) +douban-skill-workspace/ +debugging-network-issues-workspace/ +.gstack/ + +# Claude Code local settings +.claude/settings.local.json +.claude/archive/ + +# Generated artifacts +coverage/ +node_modules/ diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000..b1b1df73 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,53 @@ +# Gitleaks custom rules for claude-code-skills repo +# Catches personal info that shouldn't be in an open source repo + +title = "claude-code-skills sensitive data rules" + +[extend] +useDefault = true + +# Global allowlist: files that are allowed to contain patterns +# (the config file itself, hooks, and contribution guides) +[allowlist] +paths = [ + '''\.gitleaks\.toml$''', + '''\.githooks/''', + '''CONTRIBUTING\.md$''', + '''CLAUDE\.md$''', +] + +[[rules]] +id = "absolute-user-path-macos" +description = "Hardcoded macOS user home directory path" +regex = '''/Users/[a-zA-Z][a-zA-Z0-9_-]+/''' +tags = ["pii", "path"] + +[[rules]] +id = "absolute-user-path-linux" +description = "Hardcoded Linux home directory path" +regex = '''/home/[a-zA-Z][a-zA-Z0-9_-]+/''' +tags = ["pii", "path"] + +[[rules]] +id = "windows-user-path" +description = "Hardcoded Windows user profile path" +regex = '''C:\\Users\\[a-zA-Z][a-zA-Z0-9_-]+\\''' +tags = ["pii", "path"] + +[[rules]] +id = "phone-number-cn" +description = "Chinese mobile phone number" +regex = '''1[3-9]\d{9}''' +tags = ["pii", "phone"] + +[[rules]] +id = "douban-user-id-literal" +description = "Hardcoded Douban user ID" +regex = '''songtiansheng''' +tags = ["pii", "username"] + +[[rules]] +id = "email-personal" +description = "Personal email address" +regex = '''[a-zA-Z0-9._%+-]+@(gmail|qq|163|126|outlook|hotmail|yahoo|icloud|foxmail)\.[a-zA-Z]{2,}''' +tags = ["pii", "email"] diff --git a/.pii-path-patterns b/.pii-path-patterns new file mode 100644 index 00000000..81bb463f --- /dev/null +++ b/.pii-path-patterns @@ -0,0 +1,6 @@ +# Repo-specific local/generated artifact paths that must never be committed +(^|/)coverage(/|$) +(^|/)node_modules(/|$) +(^|/).*\.db$ +(^|/)wechat-article-scraper/web/backend/storage/images(/|$) +(^|/)wechat-article-scraper/web/feeds(/|$) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..370d60a2 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,34 @@ +minimum_pre_commit_version: "3.2.0" +default_install_hook_types: + - pre-commit + - pre-push +default_stages: + - pre-commit + - pre-push + +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-merge-conflict + - id: check-added-large-files + args: + - --maxkb=1500 + + - repo: https://github.com/gitleaks/gitleaks + rev: v8.30.0 + hooks: + - id: gitleaks + stages: + - pre-commit + + - repo: local + hooks: + - id: repo-path-guard + name: repo path guard + entry: python3 scripts/repo_path_guard.py + language: system + pass_filenames: true + stages: + - pre-commit + - pre-push diff --git a/CHANGELOG.md b/CHANGELOG.md index 14c1c277..ac9556a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,8 +7,226 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.51.0] - 2026-04-26 + ### Added -- None +- **debugging-network-issues** v1.0.0: Evidence-driven, falsification-first methodology for network, streaming, and protocol-layer bugs where the obvious cause is probably wrong. Built from a real 5-hour production case (SSE RST_STREAM at exactly 130s, traced to a CGNAT idle timeout). Provides layered-isolation experiments (run the same logical request through 3+ paths differing by one hop), env-gated runtime instrumentation patterns, and a counter-review four-question filter to challenge single-cause assumptions before shipping a fix. Bundles probe scripts (`layered-isolation-probe.sh`, `mock-idle-upstream.py`) and reference docs covering counter-review, packet-capture recipes, instrumentation patterns, and cognitive traps. Triggers on `ECONNRESET`, HTTP/2 `RST_STREAM`, `INTERNAL_ERROR`, fixed-time SSE drops, CDN/proxy/CGNAT idle timeouts, and "works sometimes / fails after N seconds" patterns. +- **stepfun-tts** v1.0.0: Generate Chinese/Japanese speech with `stepaudio-2.5-tts` and transcribe long audio with `stepaudio-2.5-asr` (SSE endpoint, 32K context, ~100x RTF, up to 30-minute single call). Encapsulates the three non-obvious StepAudio 2.5 pitfalls that cost hours: `voice_label` removal (replaced by `instruction` + inline `()` prosody), `/v1/audio/asr/sse` endpoint mismatch (returns misleading `model not supported` error otherwise), and stricter censorship rules. Bundled scripts: `tts_generate.py` (with `--batch `), `asr_transcribe.py`, `ab_compare.sh`. API key resolution: `$STEPFUN_API_KEY` → `${CLAUDE_PLUGIN_DATA}/config.json` fallback. Reference docs cover migration from `step-tts-2`, the censorship rewrite list, and the verified-on-2026-04-23 known-issues registry. + +### Changed +- Marketplace skill count: 49 → 51 +- Marketplace plugin entry count: 53 → 55 +- Marketplace version: 1.50.0 → 1.51.0 +- README.md, README.zh-CN.md: badges, descriptions, skill sections (#49 + #50), Use Cases entries, Documentation Quick Links, Requirements +- CLAUDE.md: overview count, marketplace plugin count, Available Skills list + +### Note +Plugin entries for these two skills were inadvertently committed in v1.50.0's path-rewrite operation (the entries existed as uncommitted draft modifications in `marketplace.json` and were carried along when that file was rewritten). v1.51.0 completes the registration that v1.50.0 left half-done by landing the skill directories themselves and synchronizing all documentation surfaces. + +## [1.50.0] - 2026-04-26 + +### Changed +- **Suite directory flattening**: Moved both suite directories from `suites//` to the repo root: `suites/daymade-docs/` → `daymade-docs/` and `suites/daymade-claude-code/` → `daymade-claude-code/`. The `suites/` intermediate directory has been removed. Plugin names, install commands, and skill invocations are unchanged for end users — only the on-disk layout and the `source` paths in `marketplace.json` (and doc links) were affected. `claude plugin update` will re-fetch from the new paths automatically. +- Updated all 15 `source` entries in `.claude-plugin/marketplace.json` from `./suites//...` to `.//...`. +- Updated documentation references in `CLAUDE.md`, `README.md`, `README.zh-CN.md`, `references/new-skill-guide.md`, `daymade-claude-code/marketplace-dev/SKILL.md`, and `daymade-claude-code/marketplace-dev/references/cache_and_source_patterns.md`. +- Fixed pre-existing double-prefix typo (`suites/daymade-claude-code/suites/daymade-claude-code/...`) in two README locations during the path rewrite. + +## [1.49.0] - 2026-04-19 + +### Added +- **slides-creator** v1.0.0: Narrative-first slide deck creation. Guides users through structured narrative design (ABCDEFG model), then delegates visual generation to baoyu-slide-deck. Focuses on what machines can't do — narrative co-design with humans. Six-phase workflow: source collection → narrative discussion → content structuring → prompt generation → image generation → post-processing with directory reorganization and speaker notes extraction. Triggers on "create slides", "make a presentation", "generate deck", "slide deck", "PPT", or when user needs to turn content into visual slides. + +### Changed +- Updated marketplace skills count from 48 to 49 +- Updated marketplace plugin entries from 52 to 53 +- Updated marketplace version from 1.48.0 to 1.49.0 +- Updated README.md badges, skill listings, use cases, and documentation quick links +- Updated README.zh-CN.md badges, skill listings, use cases, and documentation quick links +- Updated CLAUDE.md skill count (48 → 49), plugin entry count (52 → 53), and Available Skills list + +## [1.48.0] - 2026-04-19 + +### Added +- **daymade-claude-code** suite v1.0.0: Claude Code operations suite bundling 7 power-user skills (`claude-code-history-files-finder`, `continue-claude-work`, `claude-skills-troubleshooting`, `claude-md-progressive-disclosurer`, `statusline-generator`, `claude-export-txt-better`, `marketplace-dev`) under one shared namespace. One command gets the full Claude Code toolkit and invocations render as `daymade-claude-code:` instead of the redundant `:` form. + +### Changed +- **Canonical source migration**: The 7 Claude Code-related skills were physically moved from the repo root into `suites/daymade-claude-code//`, mirroring the `daymade-docs` suite pattern. Both the suite and the 7 individual single-skill plugins now install from the same canonical location, keeping plugin caches narrow (only the suite's own files, not the whole repo). Transparent to existing users: plugin names and invocation remain identical; `claude plugin update` fetches from the new path automatically. +- Patch bumps for the 7 migrated skills to reflect the manifest/source change: + - `claude-code-history-files-finder` 1.0.2 → 1.0.3 + - `continue-claude-work` 1.1.1 → 1.1.2 + - `claude-skills-troubleshooting` 1.0.0 → 1.0.1 + - `claude-md-progressive-disclosurer` 1.2.0 → 1.2.1 + - `statusline-generator` 1.0.0 → 1.0.1 + - `claude-export-txt-better` 1.0.0 → 1.0.1 + - `marketplace-dev` 1.2.0 → 1.2.1 (also simplified hook paths from `${CLAUDE_PLUGIN_ROOT}/marketplace-dev/hooks/...` to `${CLAUDE_PLUGIN_ROOT}/hooks/...` now that the cache root is the skill dir itself) +- Updated marketplace version from 1.47.0 to 1.48.0 +- Updated marketplace plugin entries from 51 to 52 +- README / README.zh-CN / CLAUDE.md / references/new-skill-guide.md: all doc links to these 7 skills now point to `suites/daymade-claude-code//` + +## [1.47.0] - 2026-04-12 + +### Added +- **wechat-article-scraper** v2.9.0: World-class WeChat article extraction with 6-level strategy routing (fast→adaptive→stable→reliable→zero_dep→jina_ai), OG metadata fallback, image-paragraph association, lazy loading handling, local image download, and Sogou search discovery. Supports Markdown/JSON/HTML/PDF export. Includes 15 unique/leading features surpassing all competitors. + +### Changed +- Updated marketplace skills count from 47 to 48 +- Updated marketplace version from 1.46.0 to 1.47.0 + +### Added +- **gangtise-copilot** v1.0.0: One-stop installer and companion for the full Gangtise (岗底斯投研) OpenAPI skill suite — 19 official skills covering data retrieval (OHLC 行情, 财务, 估值, 研报, 首席观点, 会议纪要, 调研纪要), research workflows (个股研究 L1-L4, 观点 PK 对抗性分析, 主题研究, 事件复盘, 公告摘要), and utility (股票池管理, 公开网页搜索). Distilled from a 5-round discovery session that reverse-engineered the complete Gangtise skill catalog — the Gangtise OBS bucket has LIST permission disabled, so the full 19-skill inventory is not discoverable from any public manifest. Ships with 4 preset install modes (full / workshop / minimal / custom), zero-config multi-agent distribution to Claude Code / OpenClaw / Codex via symlink from a single canonical install location, shared XDG credential file at `~/.config/gangtise/authorization.json` that rotates all 19 skills in one edit, and a read-only diagnostic script with scoped liveness checks (`auth` scope + `rag` scope). Ships: `scripts/install_gangtise.sh` (408 lines), `scripts/configure_auth.sh` (310 lines), `scripts/diagnose.sh` (320 lines), and 5 reference docs covering installation flow, credentials setup, the complete 19-skill registry with per-script capability matrix, known ecosystem traps (parallel product lines, bundle-only hidden skills, double-Bearer token bug, admin endpoint 1009 errors), and workshop best practices. Target use case: the 2026 Q2 investor Workshop series where students need to install a large skill suite quickly without reverse-engineering the catalog themselves. + +### Changed +- **Renamed**: `markdown-tools` → `doc-to-markdown` — clearer name for DOCX/PDF/PPTX → Markdown conversion +- **doc-to-markdown**: Added 8 DOCX post-processing fixes (grid tables, simple tables, CJK bold spacing, JSON pretty-print, image path flattening, pandoc attribute cleanup, code block detection, bracket fixes) +- **doc-to-markdown**: Added 31 unit tests (`test_convert.py`) +- **doc-to-markdown**: Added 5-tool benchmark report (`references/benchmark-2026-03-22.md`) +- **marketplace-dev** v1.0.0 → v1.1.0: Added evidence intake from Claude Code history, plugin boundary decision guidance, source/cache patterns for single-skill and suite plugins, source+skills resolution validation, and cache footprint testing based on real marketplace debugging sessions. +- **marketplace-dev** v1.1.0 → v1.2.0: Refined against Anthropic's official skill-authoring best practices. Extracted the inline Node.js resolution check and diff pipeline into `scripts/check_marketplace.sh` — a one-shot validator that runs JSON syntax → `claude plugin validate` → source+skills resolution → reverse sync (disk SKILL.md → manifest) in a single command. Moved the two PostToolUse hook scripts from `scripts/` to `hooks/` for semantic clarity (scripts execute during skill workflow, hooks guard the editor) and updated the plugin manifest's hook paths accordingly. Added tables of contents to `anti_patterns.md` and `cache_and_source_patterns.md` (both >100 lines, per best practices). Corrected Phase 0 subagent history-mining paths to `/subagents/agent-*.jsonl`. Documented the auto-activated hook behaviour in a new "Bundled hooks" section. + +## [1.46.0] - 2026-04-11 + +### Added +- **claude-export-txt-better** v1.0.0: Fixes broken line wrapping in Claude Code exported `.txt` conversation files. Reconstructs tables, paragraphs, paths, and tool calls that were hard-wrapped at fixed column widths. Ships with an automated validation suite of 53 generic, file-agnostic checks. Triggers on export files with broken formatting or when the user mentions "fix export" / "fix conversation" / references a `YYYY-MM-DD-HHMMSS-*.txt` file. Bundled: `scripts/fix-claude-export.py`, `scripts/validate-claude-export-fix.py`, `evals/`. +- **douban-skill** v1.0.0: Exports and syncs Douban (豆瓣) book / movie / music / game collections to local CSV files via the reverse-engineered Frodo API. Supports full export and RSS incremental sync. No login, no cookies, no browser. Pre-flight user-ID validation and CSV output with UTF-8 BOM (Excel-compatible). Ships with a complete troubleshooting log of 7 tested scraping approaches and why each failed. Bundled: `scripts/douban-frodo-export.py`, `scripts/douban-rss-sync.py`, `references/troubleshooting.md`, `.gitleaks.toml` (allowlisting the public APK credentials). +- **terraform-skill** v1.0.0: Operational traps for Terraform provisioners, multi-environment isolation, and zero-to-deployment reliability. Every failure pattern documented caused a real incident. Covers provisioner timing races, SSH connection conflicts, DNS record duplication, volume permissions, database bootstrap gaps, snapshot cross-contamination, Cloudflare credential format errors, hardcoded domains in Caddyfiles/compose, and init-data-only-on-first-boot pitfalls. Organised as *exact error → root cause → copy-paste fix*. Bundled: `references/` with detailed remediation patterns. + +### Changed +- Updated marketplace skills count from 44 to 47 +- Updated marketplace version from 1.45.1 to 1.46.0 +- Updated marketplace plugin entries from 47 to 50 +- Updated README.md badges and skill listings (English and Chinese) +- Updated CLAUDE.md skill count (44 → 47) and plugin entry count (47 → 50) + +## [1.45.1] - 2026-04-11 + +### Fixed +- **daymade-docs** v1.0.0 → v1.0.1: Narrowed the suite plugin source to `suites/daymade-docs/` so the installed cache contains only the documentation skills in the suite instead of a full repository snapshot. +- Moved the daymade-docs member skills under `suites/daymade-docs/` as their canonical source and repointed the corresponding single-skill plugin entries to those same directories. +- **doc-to-markdown** v2.1.0 → v2.1.1, **mermaid-tools** v1.0.1 → v1.0.2, **ppt-creator** v1.0.0 → v1.0.1, **pdf-creator** v1.3.1 → v1.3.2, **docs-cleaner** v1.0.0 → v1.0.1, and **meeting-minutes-taker** v1.1.0 → v1.1.1 now install from their suite canonical source paths. + +### Changed +- Updated marketplace version from 1.45.0 to 1.45.1 + +## [1.45.0] - 2026-04-11 + +### Added +- **daymade-docs** v1.0.0: Documentation suite plugin that exposes `doc-to-markdown`, `mermaid-tools`, `pdf-creator`, `ppt-creator`, `docs-cleaner`, and `meeting-minutes-taker` under one namespace. This keeps the existing single-skill plugins available while providing `/daymade-docs:` slash commands for users who want a combined documentation workflow install. + +### Changed +- Updated marketplace version from 1.44.0 to 1.45.0 +- Updated README.md, README.zh-CN.md, and CLAUDE.md to document suite plugin architecture while preserving the existing single-skill plugin model. + +## [1.44.0] - 2026-04-11 + +### Added +- **skill-creator** v1.7.1 → v1.7.2: Completeness pass for the `workflows/wrapper-skill/` methodology within its scope (zip-archive skill packages distributed via `npx skills add`). A fifth adversarial agent review audited the wrapper-skill workflow docs against the canonical `ima-copilot` implementation and surfaced 13 on-scope lessons that were implicit in the reference code but not elevated to named patterns in the workflow. This release lands all 13. + - `patterns.md` install template: replaced the `` placeholder with a concrete defensive block covering `curl --fail` with HTTP-code branching, `wc -c` download-size sanity check rejecting suspiciously small archives before extraction, Node.js ≥18 numeric check (separate from `command -v node`), and a documented zero-agents-detected fallback policy (abort vs silent-skip vs default-to-claude-code, with the session's chosen answer named). Every defensive pattern has an accompanying "Lessons baked into this template" bullet explaining *why* it's there. + - `patterns.md` known_issues template: added `**Why upstream probably hasn't fixed it**` as a required field (the field that keeps repair blocks load-bearing across upstream upgrades), added `Strategy skip` as a first-class documented third option (users on tolerant platforms may legitimately not want the repair and naming the skip path explicit prevents the "did I forget?" failure mode), and added detailed notes on the `[ -f ... ] && \` guard rationale, `sed -i.bak ... && command rm -f *.bak` BSD/GNU portability dance, and backup directory naming convention. + - `patterns.md` diagnose template: added a new "Detection function return-code contract" subsection spelling out the required return codes for every post-repair state (untouched-good, untouched-broken, not-present, each Strategy-applied state, and the dual-state conflicted code). The dual-state code is the single hardest lesson from the ima-copilot session — a detection function that doesn't recognize it silently passes conflicted installs as healthy. + - `patterns.md` diagnose template: added variadic `find_install` rationale explaining that agents whose home-directory layout has not stabilized (like OpenClaw) should be probed against an ordered list of candidate paths, and that designing the helper as variadic from day one avoids a painful refactor when a second candidate path becomes necessary. + - `patterns.md` SKILL.md template: added explicit checklist for the description field (literal error strings from the session, tool name in every language the session used, self-disambiguation clause naming the upstream package to prevent wrapper-vs-upstream trigger fighting, symptoms that triggered the original session), plus a reference to the enforced 1024-character cap in `quick_validate.py:184`. Added "when in doubt → diagnose" as a recommended routing table default since diagnose is the only read-only entry point. + - `patterns.md` credentials section: added explicit guidance that liveness checks must match on **response-body shape**, not just HTTP status. Many APIs return 200 OK with an error JSON body, and a naive `curl --fail` check will pass a credential that fails the first real operation. + - `workflow.md` Step 5: expanded the install-script bullet list with prerequisite-check discipline (curl/unzip/npx loop plus separate Node.js ≥18 parse), download integrity defense in depth (HTTP code branching + size sanity), and the zero-agents fallback policy. + - `workflow.md` Step 6: expanded the known_issues schema to include the `Why upstream probably hasn't fixed it` field and the `Strategy skip` branch, and documented the `sed -i.bak` cross-BSD/GNU portability rule alongside the existing `command cp/mv` guidance. + - `workflow.md` Step 7: replaced the "returns OK / TRIGGERED / N/A / post-fix-state" shorthand with an explicit enumeration of the return-code contract, and added the variadic `find_install` guidance for agents with unstabilized layouts. + +### Changed +- Updated marketplace version from 1.43.0 to 1.44.0 + +## [1.43.0] - 2026-04-11 + +### Fixed +- **ima-copilot** v1.0.0 → v1.0.1: Contract compliance and dogfood-driven fixes + - `SKILL.md`, `references/known_issues.md`, `references/installation_flow.md`: removed hardcoded references to upstream version `1.1.2`. Install script keeps the version as an overridable default which is explicitly allowed by the architecture contract. Fixes a principle 6 (independent evolution) violation that would have forced a skill version bump on every upstream release. + - `references/known_issues.md`: added `command` prefix to the `sed -i.bak` and `rm -f` commands in Strategy A repair block and to the `rm -f` command in Strategy A rollback, matching the contract's alias-safe requirement. Previously, a user shell with `alias rm='rm -i'` or `alias sed='sed -i'` would hang the repair on an interactive prompt. + - `scripts/install_ima_skill.sh`: added a Node.js ≥18 preflight check. The `npx skills add` distribution path needs a modern Node runtime and the failure message on old Node is opaque. + - `scripts/diagnose.sh`: `check_submodule` now recognizes and explicitly warns on the dual-state where both `SKILL.md` and `MODULE.md` exist simultaneously (can happen when a user switched repair strategies mid-session or restored a partial backup). Previously this reported clean while the install was in a conflicted state. + - `scripts/search_fanout.py`: `rank_groups` now sorts tied hit counts by KB name for deterministic byte-identical output. Previously the tie-break depended on `concurrent.futures.ThreadPoolExecutor.map` completion order, which varied with network timing. +- **skill-creator** v1.7.0 → v1.7.1: Wrapper-skill workflow hardening from counter-review findings + - `workflows/wrapper-skill/workflow.md` Step 2: added a "How to access the conversation" subsection with concrete guidance for three cases (same session / follow-up session / neither available) and an explicit "do not fabricate content" rule for the last case. Fresh agents were previously left to guess. + - `workflows/wrapper-skill/workflow.md` Step 1: added an "AskUserQuestion fallback" subsection explaining that the consent requirement is the explicit user choice, not the specific tool name, and showing a plain-text fallback pattern for harnesses without `AskUserQuestion`. + - `workflows/wrapper-skill/patterns.md`: added a new "Runtime-logic patterns shared across wrappers" section with three generalizable insights distilled from ima-copilot's `search_fanout.py` — **capability partitioning** (enumerate vs operate permission asymmetry with four-way result bucketing), **undocumented limit detection** (silent truncation heuristics for APIs that cap results without emitting pagination tokens), and **scoped liveness checks** (probe the lowest-privilege operation the skill actually performs, not the easiest API call). Each pattern includes example code, real-world examples across multiple APIs (GitHub, Slack, Notion, Google Drive), and a cross-reference to the ima-copilot implementation. + - `workflows/wrapper-skill/verification_protocol.md`: restructured into Track 1 (session cross-reference for literal transcriptions) and Track 2 (smoke test / unit test for runtime logic). The previous "verification is not dogfood" dogma was too strict — it correctly applied to Track 1 files but wrongly exempted Track 2 runtime code from end-to-end testing. Track 2 files like `search_fanout.py` now have an explicit mandatory-smoke-test rule. + +### Changed +- Updated marketplace version from 1.42.0 to 1.43.0 + +## [1.42.0] - 2026-04-11 + +### Added +- **skill-creator** v1.6.0 → v1.7.0: New `workflows/wrapper-skill/` specialized workflow for retrospectively distilling an install-and-debug session into a reusable companion skill for a third-party CLI tool + - `workflows/wrapper-skill/workflow.md` — the retrospective distillation workflow with Step 2 conversation mining at its core (install flow, credential setup, bugs encountered and resolved, design decisions made, noise to discard) + - `workflows/wrapper-skill/architecture_contract.md` — seven non-negotiable principles that every generated wrapper skill must follow (never vendor upstream, runtime repair over ship-time patches, explicit user consent for any upstream file modification, idempotent/reversible/alias-safe repair commands, teaching agents over humans, independent evolution from upstream, private preferences stay private) + - `workflows/wrapper-skill/patterns.md` — copy-pasteable templates for SKILL.md, install script, diagnose script, known_issues registry, and credential setup, each annotated with the lessons baked in and cross-referenced to the canonical ima-copilot implementation + - `workflows/wrapper-skill/verification_protocol.md` — post-generation verification focused on cross-referencing generated artifacts against the source conversation rather than re-running the full install (the install already ran in the source session) + - `workflows/wrapper-skill/scripts/init_wrapper_skill.py` — bootstrap scaffold that creates the wrapper skill directory layout with placeholder markers pointing back at specific steps in the workflow + - `SKILL.md` root entry now includes a "Specialized Workflow: Wrapper Skills for Third-Party CLI Tools" routing section between Capture Intent and Prior Art Research that redirects agents to the wrapper workflow when the signals apply + - Canonical reference implementation: [`ima-copilot`](./ima-copilot) — the Tencent IMA wrapper that was the first product of this methodology, distilled during a real session whose lessons (shell alias bypass, root SKILL.md detection, realpath-based symlink dedup, idempotent reversible repairs) were captured in the patterns and propagated into this workflow + +### Changed +- Updated marketplace version from 1.41.0 to 1.42.0 + +## [1.41.0] - 2026-04-11 + +### Added +- **New Skill**: ima-copilot v1.0.0 — One-stop companion and installer for the official Tencent IMA skill (ima.qq.com), with wrapper-layer architecture that never vendors upstream files + - Zero-config installation to Claude Code, Codex, and OpenClaw via `npx skills add` ([vercel-labs/skills](https://github.com/vercel-labs/skills)) with auto-detection of installed agents and default symlink mode, so that a repair or upgrade applied once propagates automatically to every agent that shares the canonical install + - XDG-style credential management at `~/.config/ima/{client_id, api_key}` with env-var fallback (`IMA_OPENAPI_CLIENTID` / `IMA_OPENAPI_APIKEY`) + - Bundled `scripts/diagnose.sh` for read-only health check covering install presence, credential liveness, and known upstream issues with structured `✅/⚠️/❌` report + - Bundled `scripts/install_ima_skill.sh` with version override via `--version` flag or `IMA_VERSION` env var + - Bundled `scripts/search_fanout.py` for client-side cross-knowledge-base search with priority-based KB boosting, skip-list filtering, 100-result silent-truncation detection, and permission-denied KB partitioning (typical for subscribed KBs) + - Detects and repairs ISSUE-001 (submodule SKILL.md files missing YAML frontmatter in upstream v1.1.2) with two user-selectable strategies: Strategy A (rename to `MODULE.md` and patch root references — respects upstream design intent) or Strategy B (prepend minimal frontmatter — smallest diff) + - All repair commands are idempotent, reversible (with automatic timestamped backups to `/tmp/ima-copilot-backups/`), and use `command cp`/`command mv` to bypass interactive shell aliases + - Personalization via `~/.config/ima/copilot.json` with `priority_kbs` and `skip_kbs` lists — template at `config-template/copilot.json.example` uses illustrative-only values so the skill ships with zero real KB names + - Comprehensive reference documentation in `references/` covering installation flow, API key setup, known issues (source of truth for repairs), and search best practices + - Never vendors, forks, or mirrors upstream files — every repair is a runtime instruction executed with explicit user consent + +### Changed +- Updated marketplace skills/plugins count from 43 to 44 +- Updated marketplace version from 1.40.1 to 1.41.0 + +## [1.39.0] - 2026-03-18 + +### Added +- **New Skill**: scrapling-skill v1.0.0 - Reliable Scrapling CLI installation, troubleshooting, and extraction workflows for HTML, Markdown, and text output + - Bundled `diagnose_scrapling.py` script to verify CLI health, detect missing extras, inspect Playwright browser runtime, and run real smoke tests + - Static-first workflow for choosing between `extract get`, `extract fetch`, and `stealthy-fetch` + - Verified WeChat public article extraction pattern using `#js_content` + - Verified recovery path for local TLS trust-store failures via `--no-verify` + - Bundled troubleshooting reference covering extras, browser runtime, and output validation + +### Changed +- **skill-creator** v1.5.0 → v1.5.1: Fixed `scripts/package_skill.py` so it works when invoked directly from the repository root instead of only via `python -m` +- **continue-claude-work** v1.1.0 → v1.1.1: Replaced newer Python-only type syntax in `extract_resume_context.py` so the script runs under the local `python3` environment +- Updated marketplace skills/plugins count from 42 to 43 +- Updated marketplace version from 1.38.0 to 1.39.0 +- Updated marketplace metadata description to include Scrapling CLI extraction workflows +- Updated README.md and README.zh-CN.md badges, installation commands, skill listings, use cases, quick links, and requirements +- Updated CLAUDE.md counts, version reference, and Available Skills list (added #43) + +## [1.38.0] - 2026-03-07 + +### Added +- **New Skill**: continue-claude-work v1.1.0 - Recover local `.claude` session context and continue interrupted work without `claude --resume` + - Bundled Python script (`extract_resume_context.py`) for one-call context extraction + - Compact-boundary-aware extraction using `isCompactSummary` flag (highest-signal context from session compaction summaries) + - Subagent workflow recovery — parses `subagents/` directory to report completed vs interrupted agents with last outputs + - Session end reason detection — classifies clean exit, interrupted (ctrl-c), error cascade, or abandoned + - Size-adaptive reading strategy based on file size and compaction count + - Noise filtering — skips progress/queue-operation/api_error (37-53% of session lines) + - Self-session exclusion, stale index fallback, ghost session warnings + - MEMORY.md and session-memory integration, git workspace state fusion + +### Changed +- **skill-creator** v1.4.1 → v1.5.0: SKILL.md rewrite, added eval benchmarking system (run_eval, run_loop, aggregate_benchmark), agents (analyzer, comparator, grader), eval-viewer, and improve_description script +- **transcript-fixer** v1.1.0 → v1.2.0: `--domain` defaults to all domains, added `get_domain_stats()`, cross-domain listing, and zero-match hints +- **tunnel-doctor** v1.3.0 → v1.4.0: Added Step 2C-1 for local vanity domain proxy interception, bundled `quick_diagnose.py` automated diagnostic script +- **pdf-creator** v1.0.0 → v1.1.0: Replaced Python `markdown` library with pandoc for MD→HTML conversion, removed `_ensure_list_spacing` workaround +- **github-contributor** v1.0.2 → v1.0.3: Fixed gh CLI field name (`stargazersCount` → `stargazerCount`), added Prerequisites section +- Updated marketplace skills/plugins count from 41 to 42 +- Updated marketplace version from 1.37.0 to 1.38.0 +- Updated README.md and README.zh-CN.md badges, installation commands, skill listings, use cases, quick links, and requirements +- Updated CLAUDE.md counts, version reference, and Available Skills list (added #42) ## [1.37.0] - 2026-03-02 diff --git a/CLAUDE.md b/CLAUDE.md index 93c52c7f..e15c8050 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -4,7 +4,7 @@ This file provides guidance to Claude Code (claude.ai/code) when working with co ## Repository Overview -This is a Claude Code skills marketplace containing 41 production-ready skills organized in a plugin marketplace structure. Each skill is a self-contained package that extends Claude's capabilities with specialized knowledge, workflows, and bundled resources. +This is a Claude Code skills marketplace containing 51 production-ready skills organized in a plugin marketplace structure. Most plugins expose one skill for narrow installs; suite plugins expose related skills under shared namespaces for combined installation workflows. **Essential Skill**: `skill-creator` is the most important skill in this marketplace - it's a meta-skill that enables users to create their own skills. Always recommend it first for users interested in extending Claude Code. @@ -61,13 +61,13 @@ claude plugin install skill-creator@daymade-skills ```bash # Quick validation of a skill -skill-creator/scripts/quick_validate.py /path/to/skill +cd skill-creator && uv run --with PyYAML python -m scripts.quick_validate ../skill-name # Package a skill (includes automatic validation) -skill-creator/scripts/package_skill.py /path/to/skill [output-dir] +cd skill-creator && uv run --with PyYAML python -m scripts.package_skill ../skill-name [output-dir] # Initialize a new skill from template -skill-creator/scripts/init_skill.py --path +uv run python skill-creator/scripts/init_skill.py --path ``` ### Testing Skills Locally @@ -89,10 +89,13 @@ In Claude Code, use `/plugin ...` slash commands. In your terminal, use `claude ### Git Operations -This repository uses standard git workflow: +This repository uses standard git workflow, but **always stage files by name**, +never `git add -A` / `git add .`. Multiple agents may have unstaged changes in +the same worktree — a blanket stage piggybacks their work into your commit: + ```bash git status -git add . +git add path/to/file1 path/to/file2 # specific files only git commit -m "message" git push ``` @@ -115,13 +118,28 @@ description: Clear description with activation triggers. This skill should be us --- ``` -### Privacy and Path Guidelines +### Privacy and Path Guidelines (Enforced by Pre-commit Hook) Skills for public distribution must NOT contain: - Absolute paths to user directories (`/home/username/`, `/Users/username/`) - Personal usernames, company names, product names +- Phone numbers, personal email addresses - OneDrive paths or environment-specific absolute paths -- Use relative paths within skill bundle or standard placeholders +- Use relative paths within skill bundle or standard placeholders (`/`, ``) + +**Four-layer defense system:** +1. **CLAUDE.md rules** (this section) — Claude avoids generating sensitive content +2. **Global PII Guard pre-commit hook** (`~/scripts/git-pii-guard/pre-commit`) — blocks staged PII/secrets and generated/local artifact paths +3. **Global PII Guard pre-push hook** (`~/scripts/git-pii-guard/pre-push`) — scans commits about to be pushed, catching bad local history before it hits GitHub +4. **gitleaks** (`.gitleaks.toml`) — deep scan with custom rules for this repo + +PII Guard is enabled via `~/scripts/git-pii-guard/manage.sh enable `, which sets `core.hooksPath` to `~/scripts/git-pii-guard`. +For repo-specific additions: +- `.pii-patterns` — extra content regexes +- `.pii-path-patterns` — extra forbidden path regexes +- `.pii-allowpaths` — explicit path allowlist exceptions +- `.pre-commit-config.yaml` — optional repo-local runner that wires `pre-commit` framework to the same path/content rules for contributors who prefer managed hooks +If it fires, fix the issue — do NOT use `--no-verify` to bypass. ### Content Organization @@ -134,9 +152,10 @@ Skills for public distribution must NOT contain: ## Marketplace Configuration The marketplace is configured in `.claude-plugin/marketplace.json`: - - Contains 41 plugins, each mapping to one skill +- Contains 55 plugin entries: most map to one skill, while suite plugins (`daymade-docs`, `daymade-claude-code`) map to multiple related skills - Each plugin has: name, description, version, category, keywords, skills array - Marketplace metadata: name, owner, version, homepage +- Suite plugins use `/` (a top-level directory at the repo root) as the canonical source for their member skills so suite caches contain only those skills. Single-skill plugin entries for suite members should point to the same canonical subdirectories. ### Versioning Architecture @@ -144,8 +163,8 @@ The marketplace is configured in `.claude-plugin/marketplace.json`: 1. **Marketplace Version** (`.claude-plugin/marketplace.json` → `metadata.version`) - Tracks the marketplace catalog as a whole - - Current: v1.37.0 - - Bump when: Adding/removing skills, major marketplace restructuring + - Current: v1.45.1 + - Bump when: Adding/removing skills, adding/removing suite plugins, major marketplace restructuring - Semantic versioning: MAJOR.MINOR.PATCH 2. **Individual Skill Versions** (`.claude-plugin/marketplace.json` → `plugins[].version`) @@ -179,7 +198,7 @@ This applies when you change ANY file under a skill directory: 1. **skill-creator** ⭐ - **Essential meta-skill** for creating your own skills (with init/validate/package scripts) 2. **github-ops** - GitHub operations via gh CLI and API -3. **markdown-tools** - Document conversion with WSL path handling +3. **doc-to-markdown** - DOCX/PDF/PPTX → Markdown conversion with CJK post-processing 4. **mermaid-tools** - Diagram extraction and PNG generation 5. **statusline-generator** - Claude Code statusline customization 6. **teams-channel-post-writer** - Teams communication templates @@ -212,27 +231,28 @@ This applies when you change ANY file under a skill directory: 33. **meeting-minutes-taker** - Transform meeting transcripts into structured minutes with multi-pass generation, speaker quotes, and iterative human review 34. **deep-research** - Generate format-controlled research reports with evidence mapping, citations, and multi-pass synthesis 35. **competitors-analysis** - Evidence-based competitor tracking and analysis with source citations (file:line_number format) - 36. **tunnel-doctor** - Diagnose and fix Tailscale + proxy/VPN conflicts (four layers: route, HTTP env, system proxy, SSH ProxyCommand) on macOS with WSL SSH support - 37. **windows-remote-desktop-connection-doctor** - Diagnose AVD/W365 connection quality issues with transport protocol analysis and Windows App log parsing - 38. **product-analysis** - Perform structured product audits across UX, API, architecture, and compare mode to produce prioritized optimization recommendations - 39. **financial-data-collector** - Collect real financial data for US public companies via yfinance with validation, NaN detection, and NO FALLBACK principle - 40. **excel-automation** - Create formatted Excel files, parse complex xlsm models, and control Excel windows on macOS via AppleScript - 41. **capture-screen** - Programmatically capture macOS application windows using Swift window ID discovery and screencapture workflows +36. **tunnel-doctor** - Diagnose and fix Tailscale + proxy/VPN conflicts (four layers: route, HTTP env, system proxy, SSH ProxyCommand) on macOS with WSL SSH support +37. **windows-remote-desktop-connection-doctor** - Diagnose AVD/W365 connection quality issues with transport protocol analysis and Windows App log parsing +38. **product-analysis** - Perform structured product audits across UX, API, architecture, and compare mode to produce prioritized optimization recommendations +39. **financial-data-collector** - Collect real financial data for US public companies via yfinance with validation, NaN detection, and NO FALLBACK principle +40. **excel-automation** - Create formatted Excel files, parse complex xlsm models, and control Excel windows on macOS via AppleScript +41. **capture-screen** - Programmatically capture macOS application windows using Swift window ID discovery and screencapture workflows +42. **continue-claude-work** - Recover local `.claude` session context via compact-boundary extraction, subagent workflow recovery, and session end reason detection, then continue interrupted work without `claude --resume` +43. **scrapling-skill** - Install, troubleshoot, and use Scrapling CLI for static/dynamic web extraction, WeChat article capture, and verified output validation +44. **ima-copilot** - One-stop companion and installer for the official Tencent IMA skill with zero-config three-agent installation via vercel-labs/skills, XDG credential management, read-only diagnostic, known-issue auto-repair under user consent, and personalized fan-out search with priority-based knowledge base boosting +45. **claude-export-txt-better** - Fixes broken line wrapping in Claude Code exported `.txt` conversation files; reconstructs tables, paragraphs, paths, and tool calls hard-wrapped at fixed column widths; ships with a 53-check automated validation suite +46. **douban-skill** - Exports and syncs Douban (豆瓣) book/movie/music/game collections to local CSV files via the reverse-engineered Frodo API; supports full export and RSS incremental sync with no login, cookies, or browser required +47. **wechat-article-scraper** - World-class WeChat article extraction with 6-level strategy routing, OG metadata fallback, image-paragraph association, and Sogou search discovery; supports Markdown/JSON/HTML/PDF export +48. **terraform-skill** - Operational traps for Terraform provisioners, multi-environment isolation, and zero-to-deployment reliability; covers provisioner timing races, SSH connection conflicts, DNS record duplication, volume permissions, database bootstrap gaps, Cloudflare credential errors, and init-data-only-on-first-boot pitfalls +49. **slides-creator** - Narrative-first slide deck creation guiding users through structured narrative design (ABCDEFG model), then delegating visual generation to baoyu-slide-deck. Triggers on create slides, make a presentation, generate deck, slide deck, PPT, or when user needs to turn content into visual slides +50. **debugging-network-issues** - Evidence-driven, falsification-first methodology for network/streaming/protocol-layer bugs (HTTP/2 RST_STREAM, SSE stalls, fixed-time drops, CDN/proxy/CGNAT idle timeouts). Layered isolation experiments + counter-review filter, with bundled probe scripts and a real SSE 130s case study +51. **stepfun-tts** - StepFun StepAudio 2.5 family: stepaudio-2.5-tts (Contextual TTS via instruction + inline parentheses) and stepaudio-2.5-asr (SSE endpoint, 32K context, ~100x RTF, 30-min single-call). Captures the three breaking changes from step-tts-2 (voice_label removal, /v1/audio/asr/sse endpoint, stricter censorship) with migration playbook **Recommendation**: Always suggest `skill-creator` first for users interested in creating skills or extending Claude Code. ## YouTube Downloader SOP (Internal) -Use this SOP to avoid common yt-dlp failures and confusion: - -1. Quote YouTube URLs in shell commands (zsh treats `?` as glob). Example: `'https://www.youtube.com/watch?v=VIDEO_ID'`. -2. Ensure proxy is active for both yt-dlp and PO Token providers (HTTP_PROXY/HTTPS_PROXY/ALL_PROXY). -3. If you see “Sign in to confirm you’re not a bot”, request cookie permission and use browser cookies. -4. Start the PO Token provider before downloading. Prefer Docker bgutil; fall back to browser-based WPC when Docker is unavailable or fails. -5. Use `web_safari` client when cookies are present; otherwise use `mweb` for PO tokens. -6. Keep the browser window open while WPC is minting tokens and make sure it can reach YouTube through the same proxy. -7. If you see “Only images are available” or “Requested format is not available”, treat it as PO token failure and retry after fixing provider/browser state. -8. If you see SSL EOF or fragment errors, treat it as proxy instability. Retry with progressive formats or switch to a more stable proxy. +See [youtube-downloader/references/internal-sop.md](./youtube-downloader/references/internal-sop.md) for yt-dlp troubleshooting steps (PO tokens, proxy, cookies, etc.). ## Python Development @@ -266,326 +286,48 @@ When creating a new skill: ## Adding a New Skill to Marketplace -**CRITICAL**: When adding a skill to this marketplace, you MUST update all of these files in the correct order. Missing any file will result in incomplete integration. +For the full step-by-step guide with templates and examples, see [references/new-skill-guide.md](./references/new-skill-guide.md). -### Step-by-Step Process +**Files to update** (all required): -#### 1. Refine the Skill (if needed) -```bash -# Ensure skill follows best practices -# - SKILL.md uses imperative/infinitive form -# - Third-person description in YAML frontmatter -# - Progressive disclosure (details in references/) -# - Security scan passed +| File | Locations to update | +|------|-------------------| +| `.claude-plugin/marketplace.json` | metadata.version + metadata.description + new plugin entry | +| `CHANGELOG.md` | New version entry | +| `README.md` | 7 locations: badges, description, install cmd, skill section, use case, docs link, requirements | +| `README.zh-CN.md` | 7 locations: same as above, translated | +| `CLAUDE.md` | 3 locations: overview count, marketplace config count, Available Skills list | +| `skill-name/` | The actual skill directory + packaged .zip | -cd skill-creator -python3 scripts/security_scan.py ../skill-name --verbose -``` - -#### 2. Package the Skill +**Quick workflow**: ```bash +# 1. Validate & package the skill itself cd skill-creator -python3 scripts/package_skill.py ../skill-name - -# This will: -# - Validate skill structure -# - Check security scan status -# - Create skill-name.zip in skill-creator/ -# - Move zip to skill-name/ directory -``` - -#### 3. Update CHANGELOG.md ⚠️ REQUIRED - -Add new version entry at the top (after [Unreleased]): - -```markdown -## [X.Y.0] - YYYY-MM-DD - -### Added -- **New Skill**: skill-name - Brief description - - Feature 1 - - Feature 2 - - Feature 3 - - Bundled scripts/references/assets - - Key capabilities - -### Changed -- Updated marketplace skills count from N to N+1 -- Updated marketplace version from X.(Y-1).0 to X.Y.0 -- Updated README.md badges (skills count, version) -- Updated README.md to include skill-name in skills listing -- Updated README.zh-CN.md badges (skills count, version) -- Updated README.zh-CN.md to include skill-name in skills listing -- Updated CLAUDE.md skills count from N to N+1 -- Added skill-name use case section to README.md -- Added skill-name use case section to README.zh-CN.md -- Added dependencies to requirements section (if any, both EN and ZH) -``` - -**Version numbering**: Increment MINOR version (e.g., 1.8.0 → 1.9.0) when adding a skill. - -#### 4. Update README.md ⚠️ REQUIRED - -**a. Update badges (top of file):** -```markdown -[![Skills](https://img.shields.io/badge/skills-N-blue.svg)] -[![Version](https://img.shields.io/badge/version-X.Y.0-green.svg)] -``` - -**b. Update description:** -```markdown -Professional Claude Code skills marketplace featuring N production-ready skills... -``` - -**c. Add installation command:** -```markdown -# Brief description -claude plugin install skill-name@daymade-skills -``` - -**d. Add skill section (### N. **skill-name**):** -```markdown -### N. **skill-name** - One-line Title - -Brief description paragraph. - -**When to use:** -- Use case 1 -- Use case 2 -- Use case 3 - -**Key features:** -- Feature 1 -- Feature 2 -- Feature 3 - -**Example usage:** -\`\`\`bash -# Example commands -\`\`\` - -**🎬 Live Demo** - -*Coming soon* (or add demo GIF) - -📚 **Documentation**: See [skill-name/references/](./skill-name/references/)... - -**Requirements**: Dependencies (e.g., Python 3.8+, FFmpeg, etc.) -``` - -**e. Add use case section:** -```markdown -### For [Use Case Category] -Use **skill-name** to [describe primary use case]. Combine with **other-skill** to [describe integration]. -``` - -**f. Add documentation quick link:** -```markdown -- **skill-name**: See `skill-name/references/...` for ... -``` - -**g. Update requirements section (if needed):** -```markdown -- **Tool Name** (for skill-name): `install command` -``` - -#### 5. Update CLAUDE.md ⚠️ REQUIRED - -**a. Update repository overview:** -```markdown -This is a Claude Code skills marketplace containing N production-ready skills... -``` - -**b. Update marketplace configuration:** -```markdown -The marketplace is configured in `.claude-plugin/marketplace.json`: -- Contains N plugins, each mapping to one skill -``` - -**c. Update marketplace version:** -```markdown -1. **Marketplace Version** (`.claude-plugin/marketplace.json` → `metadata.version`) - - Tracks the marketplace catalog as a whole - - Current: vX.Y.0 -``` - -**d. Add skill to Available Skills list:** -```markdown -N. **skill-name** - Brief description with key feature -``` - -#### 6. Update .claude-plugin/marketplace.json ⚠️ CRITICAL - -**MOST IMPORTANT FILE** - This file makes the skill installable! - -**a. Update metadata.description:** -```json -"description": "Professional Claude Code skills for ..., and [new skill capability]" -``` - -**b. Update metadata.version:** -```json -"version": "X.Y.0" -``` - -**c. Add new plugin entry to plugins array:** -```json -{ - "name": "skill-name", - "description": "Clear description with trigger conditions. Use when [scenarios]", - "source": "./", - "strict": false, - "version": "1.0.0", - "category": "appropriate-category", - "keywords": ["keyword1", "keyword2", "keyword3", ...], - "skills": ["./skill-name"] -} -``` - -**Categories:** `developer-tools`, `document-conversion`, `documentation`, `customization`, `communication`, `utilities`, `assets`, `design`, `productivity`, `security`, `media` - -**d. Validate JSON syntax:** -```bash -python3 -m json.tool .claude-plugin/marketplace.json > /dev/null -``` - -#### 7. Update README.zh-CN.md ⚠️ REQUIRED - -**CRITICAL**: Chinese documentation must be kept in sync with English version. - -**a. Update badges (top of file):** -```markdown -[![Skills](https://img.shields.io/badge/skills-N-blue.svg)] -[![Version](https://img.shields.io/badge/version-X.Y.0-green.svg)] -``` - -**b. Update description:** -```markdown -专业的 Claude Code 技能市场，提供 N 个生产就绪的技能，用于增强开发工作流。 -``` - -**c. Add installation command:** -```markdown -# 简短描述 -claude plugin install skill-name@daymade-skills -``` - -**d. Add skill section (### N. **skill-name** - Chinese Title):** -- Translate all content from English README -- Include: 使用场景 (When to use), 主要功能 (Key features), 示例用法 (Example usage) -- Maintain same structure as English version -- Include documentation links and requirements - -**e. Add use case section:** -```markdown -### [Use Case Category in Chinese] -使用 **skill-name** [describe use case in Chinese]. 与 **other-skill** 结合使用以 [describe integration]. -``` - -**f. Add documentation quick link:** -```markdown -- **skill-name**：参见 `skill-name/references/...` 了解 ... -``` - -**g. Update requirements section (if needed):** -```markdown -- **Tool Name**（用于 skill-name）：`install command` -``` - -**Translation tips:** -- Use professional technical Chinese -- Maintain consistency with existing translations -- Keep code examples in English (don't translate variable names, function names) -- Translate user-facing descriptions, features, and use cases - -#### 8. Verification Checklist - -Before committing, verify: - -- [ ] CHANGELOG.md has new version entry -- [ ] README.md badges updated (skills count + version) -- [ ] README.md has skill section with number -- [ ] README.md has use case section -- [ ] README.md has documentation link -- [ ] README.md requirements updated (if needed) -- [ ] README.zh-CN.md badges updated (skills count + version) ⚠️ NEW -- [ ] README.zh-CN.md has skill section with number ⚠️ NEW -- [ ] README.zh-CN.md has use case section ⚠️ NEW -- [ ] README.zh-CN.md has documentation link ⚠️ NEW -- [ ] README.zh-CN.md requirements updated (if needed) ⚠️ NEW -- [ ] README.zh-CN.md installation command added ⚠️ NEW -- [ ] CLAUDE.md skill count updated in 3 places -- [ ] CLAUDE.md has skill in Available Skills list -- [ ] marketplace.json metadata.version updated -- [ ] marketplace.json metadata.description updated -- [ ] marketplace.json has new plugin entry -- [ ] marketplace.json validates (python3 -m json.tool) -- [ ] skill-name.zip package exists -- [ ] Security scan passed - -### Common Mistakes to Avoid - -1. **Forgetting marketplace.json** ⚠️ - The most critical file! Without this, the skill cannot be installed via `claude plugin install` -2. **Forgetting Chinese documentation** ⚠️ - README.zh-CN.md must be updated in sync with README.md (6 locations) -3. **Inconsistent version numbers** - CHANGELOG, README badges (both EN and ZH), CLAUDE.md, and marketplace.json must all match -4. **Inconsistent skill counts** - README description (both EN and ZH), badges, CLAUDE.md must all have same count -5. **Missing skill number in README** - Skills must be numbered sequentially (1, 2, 3, ...) in both EN and ZH versions -6. **Invalid JSON syntax** - Always validate marketplace.json after editing -7. **Forgetting dependencies** - Update README requirements section (both EN and ZH) if skill needs external tools -8. **Incomplete Chinese translation** - Must translate all sections: description, use cases, features, use case section, docs link - -### File Update Summary Template - -When adding a skill, this is the complete file list: +uv run python -m scripts.security_scan ../skill-name --verbose +uv run --with PyYAML python -m scripts.package_skill ../skill-name + +# 2. Update all files listed above (see references/new-skill-guide.md for the +# detailed step-by-step, including 7 README locations and 3 CLAUDE.md spots) + +# 3. One-shot marketplace validation (ships with marketplace-dev skill) +cd .. && bash daymade-claude-code/marketplace-dev/scripts/check_marketplace.sh +# Runs: JSON syntax → claude plugin validate → source+skills resolution → +# reverse sync (warns when a disk SKILL.md is not registered). A WARN on +# reverse sync is the canary for orphan skills — register them or delete them. + +# 4. Stage specific files by name, never `git add -A` or `git add .` +# (a parallel agent once piggybacked another session's unstaged changes +# into its commit via `git add -A`; the fix is to stage explicitly) +git add .claude-plugin/marketplace.json CHANGELOG.md README.md README.zh-CN.md \ + CLAUDE.md skill-name/ +git commit -m "Release vX.Y.0: Add skill-name" +git push +# 5. Release +gh release create vX.Y.0 --title "Release vX.Y.0: Add skill-name" --notes "..." ``` -Files to Update: -✅ CHANGELOG.md (Add version entry) -✅ README.md (7 locations: badges, description, install, skill section, use case, docs link, requirements) -✅ README.zh-CN.md (7 locations: badges, description, install, skill section, use case, docs link, requirements) ⚠️ CRITICAL -✅ CLAUDE.md (3 locations: overview, marketplace config, available skills) -✅ .claude-plugin/marketplace.json (CRITICAL: metadata + new plugin entry) -✅ skill-name/ (The actual skill directory) -✅ skill-name/skill-name.zip (Packaged skill) -``` - -**IMPORTANT**: README.zh-CN.md is MANDATORY. Do not skip Chinese documentation updates! - -### Version Numbering Convention - -- **MAJOR.MINOR.PATCH** (Semantic Versioning) -- Increment **MINOR** when adding a new skill: 1.8.0 → 1.9.0 -- Increment **PATCH** for bug fixes or small updates: 1.9.0 → 1.9.1 -- Increment **MAJOR** for breaking changes or major restructuring: 1.9.0 → 2.0.0 - -### Quick Reference Commands - -```bash -# 1. Refine and validate skill -cd skill-creator -python3 scripts/security_scan.py ../skill-name --verbose - -# 2. Package skill -python3 scripts/package_skill.py ../skill-name - -# 3. Validate marketplace.json -cd .. -python3 -m json.tool .claude-plugin/marketplace.json > /dev/null && echo "✅ Valid" - -# 4. Check what needs committing -git status - -# 5. View specific file changes -git diff CHANGELOG.md -git diff README.md -git diff README.zh-CN.md -git diff CLAUDE.md -git diff .claude-plugin/marketplace.json -# 6. Verify Chinese documentation is in sync -grep "skills-[0-9]*" README.md README.zh-CN.md -grep "version-[0-9.]*" README.md README.zh-CN.md -``` +**Top mistakes**: Forgetting to push to GitHub, forgetting README.zh-CN.md, inconsistent version numbers across files, leaving an orphan SKILL.md on disk unregistered (caught by `check_marketplace.sh` reverse sync), using `git add -A` in a repo where multiple agents may have unstaged changes. ## Chinese User Support @@ -599,950 +341,19 @@ See README.md section "🇨🇳 中文用户指南" for details. ## Handling Third-Party Marketplace Promotion Requests -This repository is a **personal curated marketplace**, NOT a community directory or ecosystem hub. All requests to add third-party marketplace links, skill collection references, or "Community Marketplaces" sections should be declined. - -### Policy - -**DO NOT accept:** -- PRs adding "Related Resources" or "Community Marketplaces" sections linking to third-party skill collections -- Issues requesting promotion of external marketplaces -- PRs adding links to other skill repositories in README.md - -**Rationale:** -1. **Scope creep**: Shifts repository purpose from curated skills to ecosystem directory -2. **Implicit endorsement**: Listing implies quality/security review we cannot maintain -3. **Maintenance burden**: Would need to track and vet external projects over time -4. **Precedent setting**: Accepting one creates obligation to accept others - -### Response Template - -When declining, use this approach: - -```markdown -Hi @{username}, - -Thank you for your interest and for sharing {project-name}! {Brief positive acknowledgment of their project}. - -However, I'm keeping this repository focused as a **personal curated marketplace** rather than a directory of external skill collections. Adding third-party references would: - -1. Shift the repository's scope from curated skills to ecosystem directory -2. Create implicit endorsement expectations I can't maintain -3. Set precedent for similar requests (reference other declined requests if applicable) - -**What you can do instead:** - -1. **Standalone marketplace** - Your repo already works as an independent marketplace: - ``` - /plugin marketplace add {owner}/{repo} - ``` - -2. **Community channels** - Promote through: - - Claude Code GitHub discussions/issues (Anthropic's official repo) - - Developer communities (Reddit, Discord, etc.) - - Your own blog/social media - -3. **Official registry** - If/when Anthropic launches an official skill registry, that would be the appropriate place for ecosystem-wide discovery. - -Your marketplace can succeed on its own merits. Good luck with {project-name}! -``` - -### Workflow - -1. **Review the request** - Confirm it's a third-party promotion (not a legitimate contribution) -2. **Add polite comment** - Use template above, customize for their specific project -3. **Close with reason** - Use "not planned" for issues, just close for PRs -4. **Reference precedent** - Link to previously declined requests for consistency (e.g., #7, PR #5) - -### Examples - -- **Issue #7**: "Add Community Marketplaces section - Protocol Thunderdome" → Declined, closed as "not planned" -- **PR #5**: "Add Trail of Bits Security Skills to Related Resources" → Declined, closed - -## Release Workflow - -When adding a new skill or creating a marketplace release: - -### 1. Create the Skill -```bash -# Develop skill in its directory -skill-name/ -├── SKILL.md (no version history!) -├── scripts/ -└── references/ - -# Validate -./skill-creator/scripts/quick_validate.py skill-name - -# Package -./skill-creator/scripts/package_skill.py skill-name -``` - -### 2. Update Marketplace Configuration - -Edit `.claude-plugin/marketplace.json`: - -```json -{ - "metadata": { - "version": "1.x.0" // Bump minor version for new skill - }, - "plugins": [ - { - "name": "new-skill", - "version": "1.0.0", // Skill's initial version - "description": "...", - "category": "...", - "keywords": [...], - "skills": ["./new-skill"] - } - ] -} -``` - -### 3. Update Documentation - -**README.md:** -- Update badges (skills count, marketplace version) -- Add skill description and features -- Create demo GIF using cli-demo-generator -- Add use case section -- Add documentation references -- Add requirements (if applicable) - -**CLAUDE.md:** -- Update skill count in Repository Overview -- Add skill to Available Skills list -- Update Marketplace Configuration count - -### 4. Generate Demo (Optional but Recommended) - -```bash -# Use cli-demo-generator to create demo GIF -./cli-demo-generator/scripts/auto_generate_demo.py \ - -c "command1" \ - -c "command2" \ - -o demos/skill-name/demo-name.gif \ - --title "Skill Demo" \ - --theme "Dracula" -``` - -### 5. Commit and Release - -```bash -# Commit marketplace update -git add .claude-plugin/marketplace.json skill-name/ -git commit -m "Release vX.Y.0: Add skill-name - -- Add skill-name vX.Y.Z -- Update marketplace to vX.Y.0 -..." - -# Commit documentation -git add README.md CLAUDE.md demos/ -git commit -m "docs: Update README for vX.Y.0 with skill-name" - -# Push -git push - -# Create GitHub release -gh release create vX.Y.0 \ - --title "Release vX.Y.0: Add skill-name - Description" \ - --notes "$(cat <<'EOF' -## New Skill: skill-name - -Features: -- Feature 1 -- Feature 2 - -Installation: -```bash -claude plugin install skill-name@daymade-skills -``` - -Changelog: ... -EOF -)" -``` - -### Version Bumping Guide - -**Marketplace version (metadata.version):** -- **MAJOR** (2.0.0): Breaking changes, incompatible marketplace structure -- **MINOR** (1.5.0): New skill added, significant feature addition -- **PATCH** (1.4.1): Bug fixes, documentation updates, skill updates - -**Skill version (plugins[].version):** -- **MAJOR** (2.0.0): Breaking API changes for the skill -- **MINOR** (1.2.0): New features in the skill -- **PATCH** (1.1.1): Bug fixes in the skill - -### Example: v1.5.0 Release (ppt-creator) - -```bash -# 1. Created ppt-creator skill -# 2. Updated marketplace.json: 1.4.0 → 1.5.0 -# 3. Added ppt-creator plugin entry (version: 1.0.0) -# 4. Updated README.md (badges, description, demo) -# 5. Generated demo GIF with cli-demo-generator -# 6. Committed changes -# 7. Created GitHub release with gh CLI -``` +Decline all third-party marketplace promotion requests. For policy, response template, and precedents, see [references/promotion-policy.md](./references/promotion-policy.md). ## Best Practices Reference Always consult Anthropic's skill authoring best practices before creating or updating skills: https://docs.claude.com/en/docs/agents-and-tools/agent-skills/best-practices.md -- remember this release workflow in claude.md ## Plugin and Skill Architecture -This section explains the architecture of Claude Code's extension system and how different components work together. - -### Core Concepts - -#### 1. Skills - -**What**: Functional units that extend Claude's capabilities with specialized knowledge and workflows. - -**Structure**: -``` -skill-name/ -├── SKILL.md (required) # YAML frontmatter + Markdown instructions -├── scripts/ (optional) # Executable code (Python/Bash) -├── references/ (optional) # Documentation loaded as needed -└── assets/ (optional) # Templates and resources -``` - -**Loading mechanism** (Progressive Disclosure): -1. **Metadata** (~100 tokens): Always in context (name + description from YAML frontmatter) -2. **SKILL.md body** (<5k tokens): Loaded when Claude determines the skill applies -3. **Bundled resources**: Loaded only as needed by Claude - -**Location**: -- **Personal**: `~/.claude/skills/` (user-specific, not shared) -- **Project**: `.claude/skills/` (checked into git, shared with team) -- **Plugin cache**: `~/.claude/plugins/cache/{marketplace}/{plugin}/{version}/{skill}/` - -**Example**: When you ask "analyze my disk space", Claude loads the `macos-cleaner` skill's SKILL.md, then reads `references/cleanup_targets.md` as needed. - -#### 2. Plugins - -**What**: Distribution units that package one or more skills for installation via marketplaces. - -**Purpose**: Plugins enable: -- One-command installation (`claude plugin install skill-name@marketplace-name`) -- Version management -- Dependency tracking -- Marketplace distribution - -**Relationship to Skills**: -``` -Plugin (marketplace.json entry) -├── Skill 1 (./skill-name-1/) -├── Skill 2 (./skill-name-2/) -└── Skill 3 (./skill-name-3/) -``` - -**Configuration** (in `.claude-plugin/marketplace.json`): -```json -{ - "name": "my-plugin", - "description": "Use when...", - "version": "1.0.0", - "category": "utilities", - "keywords": ["keyword1", "keyword2"], - "skills": ["./skill-1", "./skill-2"] -} -``` - -**Example**: The `skill-creator` plugin contains one skill (`./skill-creator`), while a hypothetical `developer-tools` plugin might contain multiple skills like `./git-helper`, `./code-reviewer`, `./test-runner`. - -#### 3. Agents (Subagents) - -**What**: Specialized autonomous agents invoked via the `Task` tool for complex, multi-step operations. - -**Types**: -- **Bash**: Command execution specialist -- **general-purpose**: Research, search, multi-step tasks -- **Explore**: Fast codebase exploration -- **Plan**: Software architecture planning -- **skill-creator**: Meta-agent for creating skills -- **Custom**: Domain-specific agents (e.g., `test-runner`, `build-validator`) - -**When to use**: -- Tasks requiring multiple rounds of tool calls -- Open-ended exploration (finding files, searching code) -- Planning before implementation -- Autonomous execution without user intervention - -**Example**: -```python -# Instead of manually searching multiple times: -Task( - subagent_type="Explore", - description="Find error handling code", - prompt="Search the codebase for error handling patterns and list all files that handle HTTP errors" -) -``` - -#### 4. Commands - -**What**: Slash commands (e.g., `/commit`, `/review-pr`) that trigger skills. - -**Relationship**: Commands are shortcuts to invoke skills. -- `/commit` → invokes `commit` skill -- `/review-pr` → invokes `review-pr` skill - -**Configuration**: Defined in plugin's `commands/` directory or skill metadata. - -### Architecture Diagram - -``` -Marketplace (GitHub) - ↓ (git clone) -~/.claude/plugins/marketplaces/{marketplace-name}/ - ↓ (plugin install) -~/.claude/plugins/cache/{marketplace-name}/{plugin}/{version}/ - ├── skill-1/ - │ ├── SKILL.md - │ ├── scripts/ - │ └── references/ - └── skill-2/ - └── SKILL.md - ↓ (Claude loads) -Claude Code Context - ├── Metadata (always loaded) - ├── SKILL.md (loaded when relevant) - └── Resources (loaded as needed) -``` - -### Installation Flow - -#### Step 1: User initiates installation -```bash -claude plugin install macos-cleaner@daymade-skills -``` - -#### Step 2: CLI locates marketplace -```bash -# Check ~/.claude/plugins/marketplaces/daymade-skills/ -# If not exists, git clone from GitHub -``` - -#### Step 3: Read marketplace.json -```json -{ - "plugins": [ - { - "name": "macos-cleaner", - "version": "1.0.0", - "skills": ["./macos-cleaner"] - } - ] -} -``` - -#### Step 4: Download to cache -```bash -# Clone entire marketplace repo to: -~/.claude/plugins/cache/daymade-skills/macos-cleaner/1.0.0/ - -# Extract skill to: -~/.claude/plugins/cache/daymade-skills/macos-cleaner/1.0.0/macos-cleaner/ -``` - -#### Step 5: Record installation -```json -// ~/.claude/plugins/installed_plugins.json -{ - "plugins": { - "macos-cleaner@daymade-skills": [{ - "scope": "user", - "installPath": "~/.claude/plugins/cache/daymade-skills/macos-cleaner/1.0.0", - "version": "1.0.0", - "installedAt": "2026-01-11T08:03:46.593Z" - }] - } -} -``` - -#### Step 6: Claude Code loads skill -``` -When user asks: "My Mac is running out of space" - ↓ -Claude scans installed plugins metadata - ↓ -Finds "macos-cleaner" description matches - ↓ -Loads SKILL.md into context - ↓ -Executes workflow (analyze → report → confirm → cleanup) - ↓ -Loads references/scripts as needed -``` - -### Key Files and Locations - -#### Configuration Files - -| File | Location | Purpose | -|------|----------|---------| -| `marketplace.json` | `~/.claude/plugins/marketplaces/{name}/.claude-plugin/` | Defines available plugins | -| `installed_plugins.json` | `~/.claude/plugins/` | Tracks installed plugins | -| `known_marketplaces.json` | `~/.claude/plugins/` | Lists registered marketplaces | - -#### Directory Structure - -``` -~/.claude/ -├── skills/ # Personal skills (not from marketplace) -├── plugins/ -│ ├── marketplaces/ # Marketplace clones -│ │ ├── daymade-skills/ # Marketplace name -│ │ │ └── .claude-plugin/ -│ │ │ └── marketplace.json -│ │ └── anthropic-agent-skills/ -│ ├── cache/ # Installed plugins -│ │ └── daymade-skills/ -│ │ └── macos-cleaner/ -│ │ └── 1.0.0/ # Version -│ │ └── macos-cleaner/ # Skill directory -│ │ ├── SKILL.md -│ │ ├── scripts/ -│ │ └── references/ -│ ├── installed_plugins.json # Installation registry -│ └── known_marketplaces.json # Marketplace registry -``` - -### Data Flow - -#### Skill Activation -``` -User message - ↓ -Claude analyzes installed plugin metadata - ↓ -Matches description to user intent - ↓ -Loads SKILL.md (progressive disclosure) - ↓ -Executes instructions - ↓ -Loads bundled resources (scripts, references) as needed - ↓ -Generates response -``` - -#### Plugin Update -``` -Local changes to skill - ↓ -git add & commit - ↓ -git push to GitHub - ↓ -User runs: claude plugin marketplace update {marketplace-name} - ↓ -CLI pulls latest from GitHub - ↓ -Updates ~/.claude/plugins/marketplaces/{marketplace-name}/ - ↓ -User runs: claude plugin update {plugin-name@marketplace} - ↓ -Re-downloads to cache with new version number - ↓ -Updates installed_plugins.json -``` - -### Common Misconceptions - -#### ❌ Myth 1: "Updating local files immediately updates the plugin" -**Reality**: Plugins are distributed via GitHub. Local changes require `git push` before users can install updates. - -#### ❌ Myth 2: "Skills and plugins are the same thing" -**Reality**: Skills are functional units (SKILL.md + resources). Plugins are distribution packages (can contain multiple skills). - -#### ❌ Myth 3: "marketplace.json is just metadata" -**Reality**: marketplace.json is the **source of truth** for plugin discovery. Without correct configuration here, `claude plugin install` will fail with "Plugin not found". - -#### ❌ Myth 4: "Cache is just for performance" -**Reality**: Cache (`~/.claude/plugins/cache/`) is where installed plugins actually live. Deleting cache uninstalls all plugins. - -#### ❌ Myth 5: "Skills in ~/.claude/skills/ work the same as plugin skills" -**Reality**: -- `~/.claude/skills/` = Personal skills (manual management, no versioning) -- Plugin cache = Managed by CLI (versioned, updateable, shareable) - -### Best Practices - -#### For Skill Authors - -1. **Clear metadata**: Description should clearly state "Use when..." to help Claude match user intent -2. **Progressive disclosure**: Keep SKILL.md lean, move details to `references/` -3. **Test locally first**: Copy to `~/.claude/skills/` for testing before packaging -4. **Version properly**: Use semver (MAJOR.MINOR.PATCH) in marketplace.json -5. **Document bundled resources**: All scripts and references should be mentioned in SKILL.md - -#### For Marketplace Maintainers - -1. **Git workflow**: Always `git push` after updating marketplace.json -2. **Validate JSON**: Run `python -m json.tool marketplace.json` before committing -3. **Update cache**: Remind users to run `claude plugin marketplace update` after releases -4. **Version consistency**: Marketplace version ≠ plugin versions (they track independently) - -#### For Users - -1. **Update marketplaces**: Run `claude plugin marketplace update {name}` periodically -2. **Check installed plugins**: Inspect `~/.claude/plugins/installed_plugins.json` -3. **Clear cache on issues**: `rm -rf ~/.claude/plugins/cache/{marketplace-name}` then reinstall -4. **Understand scopes**: - - `--scope user`: Only you (default) - - `--scope project`: Shared with team via `.claude/plugins/` - - `--scope local`: Gitignored, local only +For full architecture documentation (core concepts, installation flow, data flow, common misconceptions, best practices), see [references/plugin-architecture.md](./references/plugin-architecture.md). ## Plugin and Skill Troubleshooting -This section provides systematic debugging steps for common plugin and skill installation issues. - -### Understanding the Architecture First - -**CRITICAL**: Before troubleshooting, understand that Claude Code's plugin system is **GitHub-based**, not local-file-based. - -``` -GitHub Repository (source of truth) - ↓ (git clone / git pull) -~/.claude/plugins/marketplaces/{marketplace-name}/ - ↓ (claude plugin install) -~/.claude/plugins/cache/{marketplace-name}/{plugin}/{version}/ - ↓ (Claude Code loads) -Active skill in Claude's context -``` - -**Key insight**: Local file changes are NOT visible to `claude plugin install` until pushed to GitHub. - -### Common Error 1: "Plugin not found in marketplace" - -**Error message**: -``` -Installing plugin "skill-name@marketplace-name"... -✘ Failed to install plugin: Plugin "skill-name" not found in marketplace "marketplace-name" -``` - -**Root causes** (in order of likelihood): - -#### Cause 1.1: Local changes not pushed to GitHub ⭐ **MOST COMMON** - -**Symptoms**: -- `git status` shows modified files or untracked directories -- marketplace.json updated locally but install fails -- All documentation updated but plugin not found - -**Diagnosis**: -```bash -# Check if you have uncommitted changes -git status - -# Check last commit vs remote -git log origin/main..HEAD - -# Verify GitHub has latest marketplace.json -# Open: https://github.com/{owner}/{repo}/blob/main/.claude-plugin/marketplace.json -``` - -**Solution**: -```bash -# 1. Commit all changes -git add -A -git commit -m "Release vX.Y.Z: Add {skill-name}" - -# 2. Push to GitHub -git push - -# 3. Clear local marketplace cache -rm -rf ~/.claude/plugins/cache/{marketplace-name} - -# 4. Update marketplace from GitHub -claude plugin marketplace update {marketplace-name} - -# 5. Retry installation -claude plugin install {skill-name}@{marketplace-name} -``` - -**Why this happens**: You updated marketplace.json locally, but Claude CLI pulls from GitHub, not your local filesystem. - -#### Cause 1.2: marketplace.json configuration error - -**Symptoms**: -- Git is up-to-date but install still fails -- Other plugins from same marketplace install fine - -**Diagnosis**: -```bash -# 1. Check marketplace.json syntax -python3 -m json.tool .claude-plugin/marketplace.json > /dev/null - -# 2. Verify plugin entry exists -cat .claude-plugin/marketplace.json | jq '.plugins[] | select(.name == "skill-name")' - -# 3. Check spelling matches exactly -# Plugin name in marketplace.json MUST match install command -``` - -**Common mistakes**: -```json -// ❌ Wrong: name mismatch -{ - "name": "macos_cleaner", // Underscore - "skills": ["./macos-cleaner"] // Dash -} - -// ✅ Correct: consistent naming -{ - "name": "macos-cleaner", - "skills": ["./macos-cleaner"] -} -``` - -**Solution**: Fix marketplace.json, then commit and push. - -#### Cause 1.3: Skill directory missing - -**Symptoms**: -- marketplace.json has entry -- Git is up-to-date -- Plugin installs but skills don't load - -**Diagnosis**: -```bash -# Check if skill directory exists -ls -la {skill-name}/ - -# Verify SKILL.md exists -ls -la {skill-name}/SKILL.md -``` - -**Solution**: Ensure skill directory and SKILL.md are committed to git. - -### Common Error 2: Marketplace cache is stale - -**Symptoms**: -- GitHub has latest changes -- Install finds plugin but gets old version -- Newly added plugins not visible - -**Diagnosis**: -```bash -# Check cache timestamp -ls -la ~/.claude/plugins/cache/{marketplace-name}/ - -# Compare with last git push -git log -1 --format="%ai" -``` - -**Solution**: -```bash -# Option 1: Update marketplace cache -claude plugin marketplace update {marketplace-name} - -# Option 2: Clear cache and re-fetch -rm -rf ~/.claude/plugins/cache/{marketplace-name} -claude plugin marketplace update {marketplace-name} -``` - -### Common Error 3: JSON syntax error - -**Error message**: -``` -Error parsing marketplace manifest -``` - -**Diagnosis**: -```bash -# Validate JSON syntax -python3 -m json.tool .claude-plugin/marketplace.json - -# Check for common issues: -# - Missing commas -# - Trailing commas (in arrays/objects) -# - Unescaped quotes in strings -# - Missing closing braces -``` - -**Solution**: Fix JSON syntax, validate, commit, push. - -### Systematic Debugging Process - -When facing ANY plugin/skill issue, follow this checklist: - -#### Step 1: Verify marketplace registration - -```bash -# List registered marketplaces -claude plugin marketplace list - -# Expected output should include your marketplace -``` - -If missing, register: -```bash -claude plugin marketplace add https://github.com/{owner}/{repo} -``` - -#### Step 2: Check Git status - -```bash -# Are there uncommitted changes? -git status - -# Is local ahead of remote? -git log origin/main..HEAD - -# Push if needed -git push -``` - -#### Step 3: Verify GitHub has latest - -```bash -# Open in browser or use gh CLI -gh browse .claude-plugin/marketplace.json - -# Or check with curl -curl https://raw.githubusercontent.com/{owner}/{repo}/main/.claude-plugin/marketplace.json | jq '.plugins[] | .name' -``` - -#### Step 4: Clear and update cache - -```bash -# Remove stale cache -rm -rf ~/.claude/plugins/cache/{marketplace-name} - -# Re-fetch from GitHub -claude plugin marketplace update {marketplace-name} -``` - -#### Step 5: Validate configuration - -```bash -# Check marketplace.json is valid JSON -python3 -m json.tool .claude-plugin/marketplace.json > /dev/null && echo "✅ Valid JSON" - -# Check plugin entry exists -cat .claude-plugin/marketplace.json | jq '.plugins[] | select(.name == "skill-name")' || echo "❌ Plugin not found" -``` - -#### Step 6: Inspect installation state - -```bash -# Check if plugin is installed -cat ~/.claude/plugins/installed_plugins.json | jq -r '.plugins | keys[]' | grep "skill-name" - -# If installed, check details -cat ~/.claude/plugins/installed_plugins.json | jq '.plugins["skill-name@marketplace-name"]' - -# Verify files exist -ls ~/.claude/plugins/cache/{marketplace-name}/{skill-name}/{version}/ -``` - -### Debugging Commands Reference - -| Purpose | Command | -|---------|---------| -| List marketplaces | `claude plugin marketplace list` | -| Update marketplace | `claude plugin marketplace update {name}` | -| Install plugin | `claude plugin install {plugin}@{marketplace}` | -| Uninstall plugin | `claude plugin uninstall {plugin}@{marketplace}` | -| Update plugin | `claude plugin update {plugin}@{marketplace}` | -| Validate manifest | `claude plugin validate {path}` | -| Check installed plugins | `cat ~/.claude/plugins/installed_plugins.json \| jq '.plugins \| keys'` | -| Inspect plugin details | `cat ~/.claude/plugins/installed_plugins.json \| jq '.plugins["{plugin}@{marketplace}"]'` | -| Clear marketplace cache | `rm -rf ~/.claude/plugins/cache/{marketplace-name}` | -| Verify JSON syntax | `python3 -m json.tool {file.json}` | - -### Understanding File Locations - -```bash -# Marketplace clones (git repositories) -~/.claude/plugins/marketplaces/{marketplace-name}/ - -# Installed plugin cache -~/.claude/plugins/cache/{marketplace-name}/{plugin-name}/{version}/ - -# Installation registry -~/.claude/plugins/installed_plugins.json - -# Personal skills (not from marketplace) -~/.claude/skills/ - -# Project-scoped skills (shared with team) -.claude/skills/ -``` - -### Common Pitfalls - -#### Pitfall 1: Confusing local skills with plugin skills - -```bash -# ❌ Wrong: Copying skill to personal directory doesn't make it installable -cp -r my-skill ~/.claude/skills/my-skill # Works, but not managed by plugin system - -# ✅ Correct: Install via marketplace for version management -claude plugin install my-skill@my-marketplace -``` - -#### Pitfall 2: Forgetting to push after updating marketplace.json - -```bash -# ❌ Wrong workflow -vim .claude-plugin/marketplace.json # Add new plugin -git add .claude-plugin/marketplace.json -git commit -m "Add plugin" -# FORGOT TO PUSH! -claude plugin install new-plugin@my-marketplace # ❌ Fails: not found - -# ✅ Correct workflow -vim .claude-plugin/marketplace.json -git add -A -git commit -m "Add new plugin" -git push # ← CRITICAL STEP -claude plugin marketplace update my-marketplace -claude plugin install new-plugin@my-marketplace # ✅ Works -``` - -#### Pitfall 3: Expecting instant propagation - -```bash -# ❌ Wrong expectation -git push # Push changes -claude plugin install new-plugin@my-marketplace # ❌ Fails: cache is stale - -# ✅ Correct: Update cache first -git push -claude plugin marketplace update my-marketplace # Fetch latest from GitHub -claude plugin install new-plugin@my-marketplace # ✅ Works -``` - -#### Pitfall 4: Inconsistent naming - -```json -// marketplace.json -{ - "name": "my_plugin", // Underscore - "skills": ["./my-plugin"] // Dash - will cause confusion -} -``` - -```bash -# User tries to install -claude plugin install my-plugin@marketplace # ❌ Not found (name has underscore) -claude plugin install my_plugin@marketplace # ✅ Works, but confusing -``` - -**Best practice**: Use consistent kebab-case for both plugin name and skill directory. - -### Real-World Example: macos-cleaner Installation Issue - -**Scenario**: After creating macos-cleaner skill and updating all documentation, `claude plugin install macos-cleaner@daymade-skills` failed with "Plugin not found". - -**Investigation**: -```bash -# 1. Check git status -git status -# Output: 16 modified/untracked files - -# 2. Check marketplace cache -ls -la ~/.claude/plugins/cache/daymade-skills/ -# Output: Last modified Dec 20 (weeks old!) - -# 3. Check GitHub -# marketplace.json on GitHub: version 1.20.0 (old) -# Local marketplace.json: version 1.21.0 (new) -``` - -**Root cause**: Local changes weren't pushed to GitHub. CLI was pulling from GitHub, not local files. - -**Solution**: -```bash -# 1. Commit and push -git add -A -git commit -m "Release v1.21.0: Add macos-cleaner" -git push - -# 2. Update marketplace -claude plugin marketplace update daymade-skills - -# 3. Install -claude plugin install macos-cleaner@daymade-skills -# ✔ Successfully installed plugin: macos-cleaner@daymade-skills -``` - -**Verification**: -```bash -cat ~/.claude/plugins/installed_plugins.json | jq '.plugins["macos-cleaner@daymade-skills"]' -# Output: Installation details with correct version - -ls ~/.claude/plugins/cache/daymade-skills/macos-cleaner/1.0.0/ -# Output: All skill files present -``` - -**Lesson**: Always remember the GitHub-based architecture. Local changes are invisible until pushed. - -### Advanced: Manual Cache Inspection - -If automated commands don't reveal the issue, manually inspect: - -```bash -# 1. Check marketplace clone -cat ~/.claude/plugins/marketplaces/{marketplace-name}/.claude-plugin/marketplace.json | jq '.metadata.version' - -# 2. Check what's in cache -ls -R ~/.claude/plugins/cache/{marketplace-name}/ - -# 3. Compare with GitHub -curl -s https://raw.githubusercontent.com/{owner}/{repo}/main/.claude-plugin/marketplace.json | jq '.metadata.version' - -# 4. Check installation record -cat ~/.claude/plugins/installed_plugins.json | jq '.plugins' | grep -i "{skill-name}" -``` - -### When All Else Fails - -1. **Complete cache reset**: - ```bash - rm -rf ~/.claude/plugins/cache/* - claude plugin marketplace update {marketplace-name} - ``` - -2. **Re-register marketplace**: - ```bash - # Remove marketplace - # (No direct command, manual edit ~/.claude/plugins/known_marketplaces.json) - - # Re-add - claude plugin marketplace add https://github.com/{owner}/{repo} - ``` - -3. **Check Claude Code version**: - ```bash - claude --version - # Plugins require Claude Code v2.0.12+ - ``` - -4. **Enable verbose logging** (if available): - ```bash - CLAUDE_DEBUG=1 claude plugin install {plugin}@{marketplace} - ``` - -### Getting Help - -If you're still stuck: - -1. **Check GitHub issues**: https://github.com/anthropics/claude-code/issues -2. **Verify marketplace.json**: Run `claude plugin validate .claude-plugin/marketplace.json` -3. **Compare with working marketplace**: Study anthropic's official marketplace structure -4. **Document your debugging**: Include output from all diagnostic commands above +For systematic debugging steps (common errors, debugging process, pitfalls, real-world examples), see [references/plugin-troubleshooting.md](./references/plugin-troubleshooting.md). -For this project specifically, refer to: -- [Plugin and Skill Architecture](#plugin-and-skill-architecture) - Architecture overview (this document) -- [skill-creator/SKILL.md](./skill-creator/SKILL.md) - Skill creation guide -- [CONTRIBUTING.md](./CONTRIBUTING.md) - Development workflow +**Quick fix for most issues**: Commit → push → `claude plugin marketplace update daymade-skills` → retry install. diff --git a/COMPETITOR_MATRIX.md b/COMPETITOR_MATRIX.md new file mode 100644 index 00000000..eed46137 --- /dev/null +++ b/COMPETITOR_MATRIX.md @@ -0,0 +1,137 @@ +# 竞品功能对比矩阵 v91 + +## 评估标准 + +- ✅ 已实现且对标 +- ⚠️ 已实现但有差距 +- ❌ 未实现 +- ❓ 未知（需测试） + +## 核心功能矩阵 + +| 功能类别 | 功能点 | WeChat Scraper | Omnivore | Wallabag | Matter | Readwise | 优先级 | +|---------|-------|----------------|----------|----------|--------|----------|--------| +| **抓取** | 微信文章抓取 | ⚠️ 6级策略但无规则库 | ❌ | ❌ | ❌ | ❌ | P0 | +| | 通用网页抓取 | ⚠️ 基础实现 | ✅ Readability | ✅ Graby | ✅ | ❌ | P0 | +| | 站点规则(ftr-site-config) | ✅ Round 95 已完成 | ✅ | ✅ | ❌ | ❌ | P1 | +| | 反爬处理 | ⚠️ 基础 | ✅ 代理池 | ⚠️ 简单 | ✅ | ❌ | P1 | +| | 图片下载 | ✅ | ✅ | ✅ | ✅ | ❌ | P1 | +| | 视频抓取 | ❌ | ⚠️ 有限 | ❌ | ✅ | ❌ | P2 | +| | 抓取成功率 | 🔄 测试中 | ~95% | ~90% | ~95% | N/A | P0 | +| **阅读** | 沉浸式阅读器 | ✅ | ✅ | ✅ | ✅ | ✅ | P0 | +| | 主题切换 | ❌ | ✅ | ✅ | ✅ | ✅ | P1 | +| | 字体调节 | ⚠️ 基础 | ✅ | ✅ | ✅ | ✅ | P1 | +| | 进度同步 | ✅ | ✅ | ⚠️ | ✅ | ✅ | P0 | +| | 离线阅读 | ⚠️ PWA缓存 | ✅ | ✅ | ✅ | ✅ | P1 | +| | 全文搜索 | ⚠️ pgvector但未验证 | ✅ | ✅ | ✅ | ✅ | P0 | +| **批注** | 文本高亮 | ✅ | ✅ | ⚠️ | ✅ | ✅ | P0 | +| | 添加评论 | ✅ | ✅ | ❌ | ✅ | ✅ | P0 | +| | 标签系统 | ✅ | ✅ | ✅ | ⚠️ | ✅ | P1 | +| | 定位准确率 | 🔄 测试中 | ~99% | N/A | ~95% | N/A | P0 | +| | 批注导出 | ⚠️ Markdown | ✅ | ✅ | ✅ | ✅ | P1 | +| | 社交批注 | ❌ | ✅ | ❌ | ⚠️ | ❌ | P2 | +| **TTS** | 文本朗读 | ✅ Round 98 Edge TTS | ❌ | ❌ | ✅ | ❌ | P1 | +| | 语速调节 | ⚠️ | N/A | N/A | ✅ | N/A | P1 | +| | 离线TTS | ⚠️ Web Speech | N/A | N/A | ✅ | N/A | P2 | +| **同步** | 云同步 | ✅ Supabase | ✅ | ✅ | ✅ | ✅ | P0 | +| | 多设备 | ✅ | ✅ | ✅ | ✅ | ✅ | P0 | +| | 冲突解决 | ⚠️ 简单 | ✅ | ⚠️ | ✅ | ✅ | P1 | +| | 同步速度 | ❓ 未测 | <1s | ~3s | <1s | <1s | P1 | +| **导出** | Markdown | ✅ | ✅ | ✅ | ✅ | ✅ | P0 | +| | PDF | ✅ | ✅ | ✅ | ❌ | ❌ | P1 | +| | Notion | ✅ | ❌ | ❌ | ❌ | ❌ | P1 | +| | Obsidian | ✅ | ❌ | ❌ | ❌ | ❌ | P1 | +| | Readwise | ⚠️ | ❌ | ❌ | ❌ | ✅ 原生 | P2 | +| | Kindle推送 | ❌ | ❌ | ⚠️ | ❌ | ❌ | P2 | +| **发现** | 邮件订阅 | ❌ | ✅ | ❌ | ❌ | ✅ | P2 | +| | 微信转发收藏 | ✅ MiniApp+Official | ❌ | ❌ | ✅ | ❌ | P0 | +| | 浏览器插件 | ✅ | ✅ | ✅ | ✅ | ✅ | P0 | +| | 移动端App | ⚠️ PWA | ✅ 原生 | ❌ | ✅ 原生 | ✅ | P1 | +| **工程化** | 单元测试 | 🔄 ~5% | ✅ 80%+ | ✅ 70%+ | ✅ | ✅ | P0 | +| | E2E测试 | 🔄 Stagehand | ✅ | ✅ | ✅ | ✅ | P0 | +| | CI/CD | ✅ GitHub Actions | ✅ | ✅ | ✅ | ✅ | P1 | +| | 错误监控 | ⚠️ Sentry配置 | ✅ | ✅ | ✅ | ✅ | P1 | +| | 性能监控 | ⚠️ Health check | ✅ | ✅ | ⚠️ | ✅ | P1 | +| | 文档完善度 | ⚠️ | ✅ | ✅ | ✅ | ✅ | P2 | + +## 关键差距分析 + +### P0（关键差距） + +1. **抓取成功率未知** + - 没有基准测试数据 + - 需要建立测试集（100篇各类型文章） + +2. **批注定位准确率未测** + - 核心功能没有量化指标 + - 需要自动化测试验证 + +3. **工程化严重不足** + - 测试覆盖率 5% vs 竞品 70-80% + - 无 CI/CD 流水线 + +4. **~~微信生态集成~~** ✅ Round 92 已完成 + - ~~Cubox/Matter 支持微信转发即收藏~~ + - ~~我们需要复制链接 → 粘贴 → 抓取（3步 vs 1步）~~ + - 已实现：小程序转发 + 公众号消息双轨集成 + +### P1（重要差距） + +1. **TTS朗读** + - Matter 的核心差异化 + - 刚实现，未验证 + +2. **站点规则(ftr-site-config)** + - Omnivore/Wallabag 使用社区规则库 + - 我们自研策略，维护成本高 + +3. **主题/字体定制** + - 竞品都支持，我们缺失 + +### P2（次要差距） + +1. 视频抓取 +2. 邮件订阅 +3. Kindle推送 + +## 行动计划 + +### 阶段1：验证现有功能（2周） +- [x] ~~建立100篇文章测试集~~ ✅ Round 93 已完成 +- [x] ~~测量抓取成功率~~ 🔄 Round 93 已建立测试框架 +- [x] ~~测量批注定位准确率~~ 🔄 Round 93 已建立测试框架 +- [ ] 完成E2E测试覆盖核心流程 + +### 阶段2：补齐关键差距（4周） +- [ ] 提升测试覆盖率至60% +- [ ] 集成 ftr-site-config 规则库 +- [ ] 验证TTS功能 +- [x] ~~CI/CD流水线~~ ✅ Round 94 已完成 + +### 阶段3：差异化功能（4周） +- [x] ~~微信转发集成方案~~ ✅ Round 92 已完成 +- [ ] 主题系统 +- [ ] 性能优化 + +## 测试数据收集 + +需要收集的指标： + +``` +抓取成功率 = 成功抓取数 / 总尝试数 × 100% +目标: ≥ 95% + +批注定位准确率 = 正确定位数 / 总批注数 × 100% +目标: ≥ 98% + +API响应时间 (p95) +目标: ≤ 200ms + +同步冲突率 = 冲突次数 / 总同步数 × 100% +目标: ≤ 0.1% +``` + +--- + +最后更新: Round 91 +下次更新: Round 100 (完成验证后) diff --git a/EOF b/EOF new file mode 100644 index 00000000..e69de29b diff --git a/QUICKSTART.md b/QUICKSTART.md index ec254187..2f3abb85 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -122,7 +122,7 @@ claude plugin marketplace add https://github.com/daymade/claude-code-skills # In Claude Code use `/plugin ...`; in your terminal use `claude plugin ...` # Step 2: Install skills you need claude plugin install github-ops@daymade-skills -claude plugin install markdown-tools@daymade-skills +claude plugin install doc-to-markdown@daymade-skills # ... add more as needed # Step 3: Restart Claude Code @@ -136,7 +136,7 @@ This table is a quick starter list. See [README.md](./README.md) for the full ca |-------|-------------|-------------| | **skill-creator** ⭐ | Create your own skills | Building custom workflows | | **github-ops** | GitHub operations | Managing PRs, issues, workflows | -| **markdown-tools** | Document conversion | Converting docs to markdown | +| **doc-to-markdown** | Document conversion | Converting docs to markdown | | **mermaid-tools** | Diagram generation | Creating PNG diagrams | | **statusline-generator** | Statusline customization | Customizing Claude Code UI | | **teams-channel-post-writer** | Teams communication | Writing professional posts | diff --git a/QUICKSTART.zh-CN.md b/QUICKSTART.zh-CN.md index a188f556..7b865a96 100644 --- a/QUICKSTART.zh-CN.md +++ b/QUICKSTART.zh-CN.md @@ -122,7 +122,7 @@ claude plugin marketplace add https://github.com/daymade/claude-code-skills # 在 Claude Code 内使用 `/plugin ...`，在终端中使用 `claude plugin ...` # 步骤 2：安装你需要的技能 claude plugin install github-ops@daymade-skills -claude plugin install markdown-tools@daymade-skills +claude plugin install doc-to-markdown@daymade-skills # ... 根据需要添加更多 # 步骤 3：重启 Claude Code @@ -136,7 +136,7 @@ claude plugin install markdown-tools@daymade-skills |-------|-------------|-------------| | **skill-creator** ⭐ | 创建你自己的技能 | 构建自定义工作流 | | **github-ops** | GitHub 操作 | 管理 PR、问题、工作流 | -| **markdown-tools** | 文档转换 | 将文档转换为 markdown | +| **doc-to-markdown** | 文档转换 | 将文档转换为 markdown | | **mermaid-tools** | 图表生成 | 创建 PNG 图表 | | **statusline-generator** | 状态栏定制 | 自定义 Claude Code UI | | **teams-channel-post-writer** | Teams 通信 | 编写专业帖子 | diff --git a/README.md b/README.md index 015a5f52..f4104e96 100644 --- a/README.md +++ b/README.md @@ -6,15 +6,15 @@ [![简体中文](https://img.shields.io/badge/语言-简体中文-red)](./README.zh-CN.md) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -[![Skills](https://img.shields.io/badge/skills-41-blue.svg)](https://github.com/daymade/claude-code-skills) -[![Version](https://img.shields.io/badge/version-1.37.0-green.svg)](https://github.com/daymade/claude-code-skills) +[![Skills](https://img.shields.io/badge/skills-51-blue.svg)](https://github.com/daymade/claude-code-skills) +[![Version](https://img.shields.io/badge/version-1.51.0-green.svg)](https://github.com/daymade/claude-code-skills) [![Claude Code](https://img.shields.io/badge/Claude%20Code-2.0.13+-purple.svg)](https://claude.com/code) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](./CONTRIBUTING.md) [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://github.com/daymade/claude-code-skills/graphs/commit-activity) -Professional Claude Code skills marketplace featuring 41 production-ready skills for enhanced development workflows. +Professional Claude Code skills marketplace featuring 51 production-ready skills for enhanced development workflows. ## 📑 Table of Contents @@ -38,13 +38,35 @@ Professional Claude Code skills marketplace featuring 41 production-ready skills The `skill-creator` is the **meta-skill** that enables you to build, validate, and package your own Claude Code skills. It's the most important tool in this marketplace because it empowers you to extend Claude Code with your own specialized workflows. -### Why skill-creator First? +### Why This skill-creator? -- **🎯 Foundation**: Learn how skills work by creating your own -- **🛠️ Complete Toolkit**: Initialization, validation, and packaging scripts included -- **📖 Best Practices**: Learn from production-ready examples -- **🚀 Quick Start**: Generate skill templates in seconds -- **✅ Quality Assurance**: Built-in validation ensures your skills meet standards +This is a **production-hardened fork** of [Anthropic's official skill-creator](https://github.com/anthropics/skills/tree/main/skills/skill-creator), born from building real skills and hitting every wall the official version doesn't warn you about. + +**The official skill-creator tells you _what_ to build. Ours also tells you _what not to try_ — and why.** + +| You're trying to... | Official | This Fork | +|---------------------|----------|-----------| +| Research before building | "Check available MCPs" (5 lines) | 8-channel search protocol with decision matrix: Adopt / Extend / Build | +| Create a skill interactively | Prose-based instructions | 9 structured AskUserQuestion checkpoints — user never loses context | +| Avoid common mistakes | No guidance | Cache edit warnings, prerequisite checks, security scan gate | +| Know the architecture options | Not mentioned | Inline vs Fork decision guide with examples (choosing wrong silently breaks your skill) | +| Validate before shipping | Basic YAML check | Expanded validator (all frontmatter fields, path reference integrity, whitespace issues) | +| Catch security issues | No tooling | `security_scan.py` with gitleaks integration — hard gate before packaging | +| Learn from real failures | No failure cases | Battle-tested methodology with documented failure patterns and gotchas | + +**Quality comparison** (independent audit, 8 dimensions): + +| Dimension | Official | This Fork | +|-----------|----------|-----------| +| Actionability | 7 | 9 | +| Error Prevention | 5 | 9 | +| Prior Art Research | 4 | 9 | +| Counter Review Process | 4 | 8 | +| Real-World Lessons | 3 | 8 | +| User Experience | 4 | 9 | +| **Total (out of 80)** | **42** | **65** | + +> Full methodology: [skill-creator/references/skill-development-methodology.md](./skill-creator/references/skill-development-methodology.md) ### Quick Install @@ -140,13 +162,50 @@ In Claude Code, use `/plugin ...` slash commands. In your terminal, use `claude claude plugin install skill-creator@daymade-skills ``` +**Documentation Suite** (shared namespace for document workflows): +```bash +claude plugin install daymade-docs@daymade-skills +``` + +This suite exposes related skills under one namespace, including: + +```text +/daymade-docs:doc-to-markdown +/daymade-docs:mermaid-tools +/daymade-docs:pdf-creator +/daymade-docs:ppt-creator +/daymade-docs:docs-cleaner +/daymade-docs:meeting-minutes-taker +``` + +Single-skill plugins remain available for narrower installs and independent updates. Documentation skills live under `daymade-docs/`, so both the suite and the individual documentation plugins install from the same canonical source while keeping plugin caches narrow. + +**Claude Code Operations Suite** (shared namespace for Claude Code power-user workflows): +```bash +claude plugin install daymade-claude-code@daymade-skills +``` + +This suite bundles the skills that extend Claude Code itself — session recovery, CLAUDE.md tuning, troubleshooting, statusline configuration, export repair, and marketplace development: + +```text +/daymade-claude-code:claude-code-history-files-finder +/daymade-claude-code:continue-claude-work +/daymade-claude-code:claude-skills-troubleshooting +/daymade-claude-code:claude-md-progressive-disclosurer +/daymade-claude-code:statusline-generator +/daymade-claude-code:claude-export-txt-better +/daymade-claude-code:marketplace-dev +``` + +Installed names render as `daymade-claude-code:` instead of the repeating `:` form you get from the individual single-skill plugins. + **Install Other Skills:** ```bash # GitHub operations claude plugin install github-ops@daymade-skills # Document conversion -claude plugin install markdown-tools@daymade-skills +claude plugin install doc-to-markdown@daymade-skills # Diagram generation claude plugin install mermaid-tools@daymade-skills @@ -237,6 +296,24 @@ claude plugin install excel-automation@daymade-skills # Programmatic macOS screenshot capture workflows claude plugin install capture-screen@daymade-skills + +# Resume interrupted Claude work from local session artifacts +claude plugin install continue-claude-work@daymade-skills + +# Scrapling CLI extraction and troubleshooting +claude plugin install scrapling-skill@daymade-skills + +# Tencent IMA knowledge base companion and installer +claude plugin install ima-copilot@daymade-skills + +# Fix broken line wrapping in Claude Code exported .txt conversations +claude plugin install claude-export-txt-better@daymade-skills + +# Export Douban (豆瓣) book/movie/music/game collections to CSV +claude plugin install douban-skill@daymade-skills + +# Terraform operational traps and multi-environment reliability patterns +claude plugin install terraform-skill@daymade-skills ``` Each skill can be installed independently - choose only what you need! @@ -288,7 +365,7 @@ Comprehensive GitHub operations using gh CLI and GitHub API. --- -### 2. **markdown-tools** - Document Conversion Suite +### 2. **doc-to-markdown** - Document Conversion Suite Converts documents to markdown with Windows/WSL path handling and PDF image extraction. @@ -307,7 +384,7 @@ Converts documents to markdown with Windows/WSL path handling and PDF image extr **🎬 Live Demo** -![Markdown Tools Demo](./demos/markdown-tools/convert-docs.gif) +![Markdown Tools Demo](./demos/doc-to-markdown/convert-docs.gif) --- @@ -820,7 +897,7 @@ python3 scripts/analyze_sessions.py stats /path/to/session.jsonl --show-files *Coming soon* -📚 **Documentation**: See [claude-code-history-files-finder/references/](./claude-code-history-files-finder/references/) for: +📚 **Documentation**: See [daymade-claude-code/claude-code-history-files-finder/references/](./daymade-claude-code/claude-code-history-files-finder/references/) for: - `session_file_format.md` - JSONL structure and extraction patterns - `workflow_examples.md` - Detailed recovery and analysis workflows @@ -907,23 +984,24 @@ Create professional PDF documents from markdown with proper Chinese typography u - Ensuring correct Chinese font rendering **Key features:** -- WeasyPrint + Markdown conversion pipeline -- Built-in Chinese font fallbacks +- pandoc + WeasyPrint conversion pipeline (dual backend: WeasyPrint or headless Chrome) +- Built-in Chinese/Japanese/Korean (CJK) font fallbacks with auto CJK code-block rendering +- Theme system (default for formal docs, warm-terra for training materials) - A4 layout defaults with print-friendly margins - Batch conversion scripts **Example usage:** ```bash -uv run --with weasyprint --with markdown scripts/md_to_pdf.py input.md output.pdf +uv run --with weasyprint scripts/md_to_pdf.py input.md output.pdf ``` **🎬 Live Demo** *Coming soon* -📚 **Documentation**: See [pdf-creator/SKILL.md](./pdf-creator/SKILL.md) for setup and workflow details. +📚 **Documentation**: See [daymade-docs/pdf-creator/SKILL.md](./daymade-docs/pdf-creator/SKILL.md) for setup and workflow details. -**Requirements**: Python 3.8+, `weasyprint`, `markdown` +**Requirements**: Python 3.8+, `pandoc` (system install), `weasyprint` (or Chrome as fallback backend) --- @@ -951,7 +1029,7 @@ Optimize user CLAUDE.md files using progressive disclosure to reduce context blo *Coming soon* -📚 **Documentation**: See [claude-md-progressive-disclosurer/SKILL.md](./claude-md-progressive-disclosurer/SKILL.md). +📚 **Documentation**: See [claude-md-progressive-disclosurer/SKILL.md](./daymade-claude-code/claude-md-progressive-disclosurer/SKILL.md). --- @@ -1402,7 +1480,7 @@ python3 scripts/enable_all_plugins.py daymade-skills *Coming soon* -📚 **Documentation**: See [claude-skills-troubleshooting/SKILL.md](./claude-skills-troubleshooting/SKILL.md) for complete troubleshooting workflow and architecture guidance. +📚 **Documentation**: See [claude-skills-troubleshooting/SKILL.md](./daymade-claude-code/claude-skills-troubleshooting/SKILL.md) for complete troubleshooting workflow and architecture guidance. **Requirements**: None (uses Claude Code built-in Python) @@ -1437,7 +1515,7 @@ claude plugin install meeting-minutes-taker@daymade-skills *Coming soon* -📚 **Documentation**: See [meeting-minutes-taker/SKILL.md](./meeting-minutes-taker/SKILL.md) for complete workflow and template guidance. +📚 **Documentation**: See [daymade-docs/meeting-minutes-taker/SKILL.md](./daymade-docs/meeting-minutes-taker/SKILL.md) for complete workflow and template guidance. **Requirements**: None @@ -1749,6 +1827,301 @@ claude plugin install capture-screen@daymade-skills --- +### 42. **continue-claude-work** - Resume Interrupted Claude Work + +Recover actionable context from local `~/.claude` session artifacts and continue implementation without reopening the old interactive session. Uses a bundled Python script for intelligent context extraction. + +**When to use:** +- A user provides a Claude session ID and wants the task continued +- You need to inspect local `.claude` JSONL files instead of running `claude --resume` +- A previous session was interrupted and the next concrete step must be reconstructed +- A multi-agent workflow was interrupted and you need to know which subagents completed + +**Key features:** +- Compact-boundary-aware extraction — reads Claude's own session compaction summaries as highest-signal context +- Subagent workflow recovery — reports completed vs. interrupted subagents with last outputs +- Session end reason detection — classifies clean exit, interrupted (ctrl-c), error cascade, or abandoned +- Size-adaptive strategy — different reading approaches for small (<500KB) vs. large (>5MB) sessions +- Noise filtering — skips progress/queue-operation/api_error messages (37-53% of session lines) +- Self-session exclusion, stale index fallback, MEMORY.md integration, git workspace state + +**Example usage:** +```bash +# Install the skill +claude plugin install continue-claude-work@daymade-skills + +# Then ask Claude to resume from local artifacts +"continue work from session 123e4567-e89b-12d3-a456-426614174000" +"don't resume, just read the .claude files and continue" +"check what I was working on in the last session and keep going" +``` + +📚 **Documentation**: See [continue-claude-work/SKILL.md](./daymade-claude-code/continue-claude-work/SKILL.md). + +**Requirements**: Python 3.8+, `git` for workspace reconciliation. + +--- + +### 43. **scrapling-skill** - Reliable Scrapling CLI Workflows + +Install, troubleshoot, and use Scrapling CLI with a verified static-first workflow for extracting HTML, Markdown, or text from webpages. Includes a diagnostic script for broken extras installs, Playwright browser runtime checks, and smoke tests against real URLs. + +**When to use:** +- Users mention Scrapling, `uv tool install scrapling`, or `scrapling extract` +- You need to choose between static and browser-backed fetching +- You need to extract article bodies from WeChat public pages (`mp.weixin.qq.com`) +- A Scrapling install works partially but fails on missing extras, browser runtime, or TLS verification + +**Key features:** +- Bundled `diagnose_scrapling.py` script for CLI, browser runtime, and live URL smoke tests +- Verified default path: start with `extract get`, escalate to `extract fetch` only when needed +- WeChat extraction pattern using `#js_content` for clean article Markdown +- Troubleshooting guidance for missing `click`, Playwright runtime setup, and `curl: (60)` trust-store failures +- Output validation workflow using file size and content checks instead of exit-code assumptions + +**Example usage:** +```bash +# Install the skill +claude plugin install scrapling-skill@daymade-skills + +# Then ask Claude to work through Scrapling for you +"Install Scrapling CLI and verify the setup" +"Extract this WeChat article into Markdown with Scrapling" +"Decide whether this page needs static or browser-backed fetching" +``` + +**🎬 Live Demo** + +*Coming soon* + +📚 **Documentation**: See [scrapling-skill/SKILL.md](./scrapling-skill/SKILL.md) and [scrapling-skill/references/troubleshooting.md](./scrapling-skill/references/troubleshooting.md). + +**Requirements**: Python 3.6+, `uv`, Scrapling CLI, and Playwright browser runtime for browser-backed fetches. + +--- + +### 44. **ima-copilot** - Tencent IMA Companion & Installer + +One-stop wrapper for the official Tencent IMA skill (`ima.qq.com`). Installs upstream `ima-skill` to Claude Code, Codex, and OpenClaw via `npx skills add`, guides API key setup, detects and repairs known upstream issues under user consent, and implements a personalized fan-out search strategy that floats priority knowledge bases to the top. + +**When to use:** +- Users mention IMA, 腾讯 IMA, ima.qq.com, or need to install the official ima-skill +- Users report `Skipped loading skill(s) due to invalid SKILL.md` warnings related to ima-skill +- You need to search across IMA knowledge bases with KB-priority boosting +- You need to configure or rotate IMA API credentials +- Upstream ima-skill ships a known issue (e.g., missing YAML frontmatter in submodule files) + +**Key features:** +- Zero-config installation to Claude Code / Codex / OpenClaw via [vercel-labs/skills](https://github.com/vercel-labs/skills) with auto-detection and default symlink mode (fix or upgrade once, every agent sees it) +- XDG-style credential management at `~/.config/ima/{client_id, api_key}` with env-var fallback +- `scripts/diagnose.sh` read-only health check (install presence, credential liveness, known issues) +- `scripts/search_fanout.py` client-side cross-KB search with priority lists, subset-skip lists, and 100-hit silent-truncation detection +- Wrapper-only architecture: never vendors upstream files, never forks — every repair is a runtime instruction executed with explicit consent and automatic timestamped backups +- Two user-selectable repair strategies for the frontmatter issue (rename to `MODULE.md` or prepend minimal frontmatter) +- Personalization via `~/.config/ima/copilot.json` with illustrative-only template values + +**Example usage:** +```bash +# Install the skill +claude plugin install ima-copilot@daymade-skills + +# Then ask Claude to drive the flow +"Install ima-skill and configure my IMA API key" +"Run diagnose on my ima-skill and fix whatever is broken" +"Search my IMA knowledge bases for embedding model comparisons, priority to my curated KB" +``` + +**🎬 Live Demo** + +*Coming soon* + +📚 **Documentation**: See [ima-copilot/SKILL.md](./ima-copilot/SKILL.md) and [ima-copilot/references/known_issues.md](./ima-copilot/references/known_issues.md). + +**Requirements**: Node.js 18+ (for `npx skills`), `curl`, `unzip`, Python 3.6+. IMA OpenAPI credentials from [https://ima.qq.com/agent-interface](https://ima.qq.com/agent-interface). + +--- + +### 45. **claude-export-txt-better** - Fix Claude Code Export Formatting + +Reconstruct broken line wrapping in Claude Code exported `.txt` conversation files. Rebuilds tables, paragraphs, paths, and tool calls that were hard-wrapped at fixed column widths, and ships with an automated 53-check validation suite (file-agnostic, catches over- and under-merging regressions). + +**When to use:** +- Users have a Claude Code export file where tables, paths, or tool output got mangled by line wrapping +- Users mention "fix export", "fix conversation", "make export readable" +- Users reference a file matching `YYYY-MM-DD-HHMMSS-*.txt` +- Users want to post-process `/export` output before sharing or archiving it + +**Key features:** +- Deterministic Python script (`fix-claude-export.py`) with `--stats` mode for before/after metrics +- 53-check automated validator (`validate-claude-export-fix.py`) that catches regressions +- Evals directory with real fixture cases +- No external dependencies beyond `uv` and Python 3.8+ + +**Example usage:** +```bash +# Fix and show stats +uv run daymade-claude-code/claude-export-txt-better/scripts/fix-claude-export.py broken.txt --stats + +# Custom output path +uv run daymade-claude-code/claude-export-txt-better/scripts/fix-claude-export.py broken.txt -o fixed.txt + +# Validate the fix +uv run daymade-claude-code/claude-export-txt-better/scripts/validate-claude-export-fix.py broken.txt fixed.txt +``` + +**🎬 Live Demo** + +*Coming soon* + +📚 **Documentation**: See [claude-export-txt-better/SKILL.md](./daymade-claude-code/claude-export-txt-better/SKILL.md) and the bundled `evals/` fixtures. + +**Requirements**: Python 3.8+, `uv` package manager. + +--- + +### 46. **douban-skill** - Douban Collection Export & Sync + +Export and sync Douban (豆瓣) book / movie / music / game collections to local CSV files via the reverse-engineered Frodo API. Full export covers all history; RSS incremental sync keeps daily updates current. No login, no cookies, no browser — just a user ID and it works. + +**When to use:** +- Users want to back up their Douban reading/watching/listening/gaming history +- Users mention 豆瓣, douban, 读书记录, 观影记录, 书影音 +- Users need incremental sync of recent Douban activity +- Users want CSV output compatible with Excel (UTF-8 BOM) + +**Key features:** +- Full export of all 4 categories (books/movies/music/games) via Frodo API +- RSS incremental sync for daily updates (last ~10 items per feed) +- Pre-flight user-ID validation (fail-fast on wrong ID) +- UTF-8 BOM CSV output, Excel-compatible, cross-platform +- Bundled troubleshooting log documenting 7 tested scraping approaches and why each failed (Douban PoW challenges block every web-scraping approach — only Frodo API works) +- `.gitleaks.toml` allowlist for the public Android APK credentials + +**Example usage:** +```bash +# Full export of user's collections +uv run douban-skill/scripts/douban-frodo-export.py + +# Incremental RSS sync (last ~10 items per category) +uv run douban-skill/scripts/douban-rss-sync.py +``` + +**🎬 Live Demo** + +*Coming soon* + +📚 **Documentation**: See [douban-skill/SKILL.md](./douban-skill/SKILL.md) and [douban-skill/references/troubleshooting.md](./douban-skill/references/troubleshooting.md) for the complete failure log of rejected approaches. + +**Requirements**: Python 3.8+, `uv` package manager. No login or cookies required. + +--- + +### 47. **terraform-skill** - Terraform Operational Traps + +Failure patterns from real Terraform deployments — every item caused an actual incident. Organized as *exact error → root cause → copy-paste fix*. Covers provisioner timing races, SSH connection conflicts, multi-environment isolation, DNS record duplication, volume permissions, database bootstrap gaps, snapshot cross-contamination, Cloudflare credential format errors, hardcoded domains in Caddyfiles/compose, and init-data-only-on-first-boot pitfalls. + +**When to use:** +- Writing `null_resource` provisioners or `remote-exec` blocks that SSH into fresh instances +- Setting up multi-environment (prod/staging/dev) Terraform with shared modules +- Debugging containers that are Restarting/unhealthy after `terraform apply` +- Hitting "docker: not found" in remote-exec, rsync connection drops in local-exec, or TLS cert errors +- Troubleshooting drift or provisioner failures during re-runs +- Configuring Caddy/gateway resources with Cloudflare credentials + +**Key features:** +- Copy-paste `.hcl` snippets for each trap, not abstract advice +- Coverage spanning cloud-init, Docker, file provisioners, DNS, TLS, snapshots, and cross-env contamination +- Every pattern tagged with the exact symptom so grep finds it fast + +**Example usage:** +```bash +# Trigger the skill naturally during Terraform work +"I'm getting 'docker: not found' in my null_resource provisioner after apply" +"My rsync local-exec is failing with 'connection unexpectedly closed'" +"Help me write a multi-env Terraform setup without snapshot cross-contamination" +``` + +**🎬 Live Demo** + +*Coming soon* + +📚 **Documentation**: See [terraform-skill/SKILL.md](./terraform-skill/SKILL.md) and the bundled `references/` for detailed remediation patterns. + +**Requirements**: None (Terraform-adjacent knowledge only; no runtime dependencies). + +--- + +### 48. **slides-creator** - Narrative-First Slide Deck Creation + +Guides users through structured narrative design (ABCDEFG model), then delegates visual generation to `baoyu-slide-deck`. Focuses on what machines can't do — narrative co-design with humans. + +**When to use:** +- Creating presentations, slide decks, or PPTs from user content +- Turning articles, transcripts, or notes into visual slides +- Designing narrative arcs for talks and workshops + +**Key features:** +- Phase 0: Source material collection (user's own words first) +- Phase 1: Narrative structure discussion using ABCDEFG model +- Phase 2: Content structuring for machine-readable input +- Phase 3-5: Delegates visual generation to baoyu-slide-deck +- Phase 6: Post-processing with directory reorganization and speaker notes extraction + +**Example usage:** +```bash +# Trigger the skill naturally +"Help me turn my article into a slide deck" +"Create a presentation from my talk transcript" +"I need a 20-minute deck for a workshop" +``` + +**Requirements**: baoyu-slide-deck skill for visual generation. + +--- + +### 49. **debugging-network-issues** - Evidence-Driven Network Investigation + +Falsification-first methodology for network, streaming, and protocol-layer bugs where the obvious cause is probably wrong. Built from a real 5-hour SSE incident where assumption-stacking wasted hours that a 10-minute layered experiment would have resolved. + +**When to use:** +- Connection resets (`ECONNRESET`, HTTP/2 `RST_STREAM`, `INTERNAL_ERROR`) +- SSE / long-polling stalls or fixed-time drops (60s, 100s, 130s) +- CDN / proxy / CGNAT idle-timeout incidents +- Any "works sometimes / fails after N seconds" pattern +- Multi-hop systems (client → CDN → LB → reverse proxy → app → upstream) where a symptom could plausibly come from several layers + +**Key features:** +- Layered isolation experiments: run the same logical request through three or more paths differing by exactly one hop +- Env-gated runtime instrumentation patterns (no production-code mutation) +- Counter-review four-question filter to challenge single-cause assumptions +- Bundled probe scripts (`layered-isolation-probe.sh`, `mock-idle-upstream.py`) +- Real case study: SSE RST_STREAM at 130s caused by CGNAT idle timeout + +**Requirements**: None (methodology + portable shell/Python probes). + +--- + +### 50. **stepfun-tts** - StepFun StepAudio 2.5 TTS + ASR + +Generate Chinese / Japanese speech and transcribe long audio with StepFun's StepAudio 2.5 family. Captures the three non-obvious pitfalls that cost hours otherwise: `voice_label` removal, the `/v1/audio/asr/sse` endpoint, and stricter censorship. + +**When to use:** +- Chinese / Japanese TTS with emotional and prosody control +- Long audio transcription (up to ~30 minutes single-call, 32K context, ~100x RTF) +- Migration from `step-tts-2` to `stepaudio-2.5-tts` (`voice_label` → `instruction` breaking change) +- Hitting StepFun censorship blocks or endpoint mismatches + +**Key features:** +- `stepaudio-2.5-tts` with `instruction` (≤200 chars natural-language mood) + inline `()` prosody +- `stepaudio-2.5-asr` SSE streaming with base64 audio (avoids the misleading "model not supported" error) +- Bundled `tts_generate.py` (with `--batch `), `asr_transcribe.py`, `ab_compare.sh` +- API key resolution: `$STEPFUN_API_KEY` → `${CLAUDE_PLUGIN_DATA}/config.json` fallback +- Censorship rewrite playbook in `references/migration_from_v2.md` + +**Requirements**: StepFun API key (https://platform.stepfun.com/). + +--- + ## 🎬 Interactive Demo Gallery Want to see all demos in one place with click-to-enlarge functionality? Check out our [interactive demo gallery](./demos/index.html) or browse the [demos directory](./demos/). @@ -1759,7 +2132,7 @@ Want to see all demos in one place with click-to-enlarge functionality? Check ou Use **github-ops** to streamline PR creation, issue management, and API operations. ### For Documentation -Combine **markdown-tools** for document conversion and **mermaid-tools** for diagram generation to create comprehensive documentation. Use **llm-icon-finder** to add brand icons. +Combine **doc-to-markdown** for document conversion and **mermaid-tools** for diagram generation to create comprehensive documentation. Use **llm-icon-finder** to add brand icons. ### For Research & Analysis Use **deep-research** to produce format-controlled research reports with evidence tables and citations. Combine with **fact-checker** to validate claims or with **twitter-reader** for social-source collection. @@ -1780,7 +2153,7 @@ Use **repomix-unmixer** to extract and validate repomix-packed skills or reposit Use **skill-creator** (see [Essential Skill](#-essential-skill-skill-creator) section above) to build, validate, and package your own Claude Code skills following best practices. ### For Presentations & Business Communication -Use **ppt-creator** to generate professional slide decks with data visualizations, structured storytelling, and complete PPTX output for pitches, reviews, and keynotes. +Use **ppt-creator** to generate professional slide decks with data visualizations, structured storytelling, and complete PPTX output for pitches, reviews, and keynotes. Use **slides-creator** for narrative-first slide design — it guides you through the ABCDEFG storytelling framework, collects your original content first, then delegates visual generation to baoyu-slide-deck. Perfect when you have existing articles, transcripts, or talks that need to become visual slides. ### For Video Quality Analysis Use **video-comparer** to analyze compression results, evaluate codec performance, and generate interactive comparison reports. Combine with **youtube-downloader** to compare different quality downloads. @@ -1812,6 +2185,12 @@ Use **prompt-optimizer** to transform vague feature requests into precise EARS s ### For Session History & File Recovery Use **claude-code-history-files-finder** to recover deleted files from previous Claude Code sessions, search for specific implementations across conversation history, or track file evolution over time. Essential for recovering accidentally deleted code or finding that feature implementation you remember but can't locate. +### For Resuming Interrupted Claude Sessions +Use **continue-claude-work** to recover the last actionable request from local `~/.claude` artifacts and continue implementation without reopening the original session. Combine with **claude-code-history-files-finder** when you need broader cross-session search, statistics, or deleted-file recovery. + +### For Web Extraction & WeChat Articles +Use **scrapling-skill** to install and validate Scrapling CLI, choose between static and browser-backed fetching, and extract clean Markdown from sites like `mp.weixin.qq.com`. Combine with **deep-research** to turn extracted sources into structured reports or with **docs-cleaner** to normalize captured article content. + ### For Documentation Maintenance Use **docs-cleaner** to consolidate redundant documentation while preserving valuable content. Perfect for cleaning up documentation sprawl after rapid development phases or merging overlapping docs into authoritative sources. @@ -1831,7 +2210,7 @@ Use **iOS-APP-developer** to configure XcodeGen projects, resolve SPM dependency Use **macos-cleaner** to intelligently analyze and reclaim disk space on macOS with safety-first approach. Unlike one-click cleaners that blindly delete, macos-cleaner explains what each file is, categorizes by risk level (🟢/🟡/🔴), and requires explicit confirmation before any deletion. Perfect for developers dealing with Docker/Homebrew/npm/pip cache bloat, users wanting to understand storage consumption, or anyone who values transparency over automation. Combines script-based precision with optional Mole visual tool integration for hybrid workflow. ### For Twitter/X Content Research -Use **twitter-reader** to fetch tweet content without JavaScript rendering or authentication. Perfect for documenting social media discussions, archiving threads, analyzing tweet content, or gathering reference material from Twitter/X. Combine with **markdown-tools** to convert fetched content into other formats, or with **repomix-safe-mixer** to package research collections securely. +Use **twitter-reader** to fetch tweet content without JavaScript rendering or authentication. Perfect for documenting social media discussions, archiving threads, analyzing tweet content, or gathering reference material from Twitter/X. Combine with **doc-to-markdown** to convert fetched content into other formats, or with **repomix-safe-mixer** to package research collections securely. ### For Skill Quality & Open-Source Contributions Use **skill-reviewer** to validate your own skills against best practices before publishing, or to review and improve others' skill repositories. Combine with **github-contributor** to find high-impact open-source projects, create professional PRs, and build your contributor reputation. Perfect for developers who want to contribute to the Claude Code ecosystem or any GitHub project systematically. @@ -1851,6 +2230,24 @@ Use **windows-remote-desktop-connection-doctor** to diagnose Azure Virtual Deskt ### For Plugin & Skill Troubleshooting Use **claude-skills-troubleshooting** to diagnose and resolve Claude Code plugin and skill configuration issues. Debug why plugins appear installed but don't show in available skills, understand the installed_plugins.json vs settings.json enabledPlugins architecture, and batch-enable missing plugins from a marketplace. Essential for marketplace maintainers debugging installation issues, developers troubleshooting skill activation, or anyone confused by the GitHub #17832 auto-enable bug. +### For Tencent IMA Knowledge Base Workflows +Use **ima-copilot** to install the official Tencent IMA skill across Claude Code / Codex / OpenClaw, configure API credentials, detect and repair known upstream issues, and run personalized fan-out searches across all your IMA knowledge bases with priority-based boosting. The wrapper architecture means upstream upgrades never collide with your fixes — every repair is a runtime instruction, not a shipped patch. Perfect for IMA power users who switch between multiple coding agents, or for anyone who has hit the "Skipped loading skill(s) due to invalid SKILL.md" warning. + +### For Post-Processing Claude Code Exports +Use **claude-export-txt-better** to clean up `/export` output before archiving or sharing. The default export format hard-wraps tables, paths, and tool-call blocks at fixed column widths, which breaks readability in any viewer wider than 80 columns. The skill reconstructs the original structure and validates the fix with 53 automated checks so regressions are caught immediately. + +### For Personal Data Backup (Douban) +Use **douban-skill** to back up your Douban 书影音 (book/movie/music/game) history to CSV. Douban has no official export — the public API was shut down in 2018 and all web scraping is blocked by PoW challenges. This skill uses the same Frodo API as the official Android app, so it just works without any login or cookies. Ships with a full failure log of 7 rejected scraping approaches, saving you hours of wasted effort. + +### For Terraform & IaC Troubleshooting +Use **terraform-skill** when your `terraform apply` fails at a provisioner step, when fresh instances hit "docker: not found", or when multi-environment setups accidentally share snapshots. Every pattern in the skill is an *exact error → root cause → copy-paste fix* triple drawn from real incidents. Perfect for anyone who has lost a weekend to timing races in cloud-init, rsync connection drops in local-exec, or hardcoded domains in Caddyfiles. + +### For Network, Streaming & Protocol-Layer Debugging +Use **debugging-network-issues** when symptoms do not match the obvious cause: HTTP/2 `RST_STREAM`, SSE stalls at exactly 60s/100s/130s, "works sometimes but not always" failures, or anything that looks like an idle-timeout incident through CDN / proxy / CGNAT chains. The skill replaces assumption-stacking with **layered isolation experiments** — running the same logical request through three or more paths that differ by one hop — plus a counter-review pattern for shipping fixes only after the hypothesis has been falsified, not just confirmed. + +### For Chinese TTS & Long-Audio Transcription (StepFun) +Use **stepfun-tts** for Chinese / Japanese voice synthesis with emotional control via `instruction` + inline `()` prosody, or for transcribing up to 30-minute audio in a single call (32K context, ~100x RTF). Captures the three breaking changes that ambush new StepAudio 2.5 users: `voice_label` removal, the `/v1/audio/asr/sse` endpoint mismatch, and stricter censorship rules. Combine with **transcript-fixer** for ASR post-processing or with **meeting-minutes-taker** to turn long recordings into structured minutes. + ## 📚 Documentation Each skill includes: @@ -1862,9 +2259,9 @@ Each skill includes: ### Quick Links - **github-ops**: See `github-ops/references/api_reference.md` for API documentation -- **markdown-tools**: See `markdown-tools/references/conversion-examples.md` for conversion scenarios -- **mermaid-tools**: See `mermaid-tools/references/setup_and_troubleshooting.md` for setup guide -- **statusline-generator**: See `statusline-generator/references/color_codes.md` for customization +- **doc-to-markdown**: See `daymade-docs/doc-to-markdown/references/conversion-examples.md` for conversion scenarios +- **mermaid-tools**: See `daymade-docs/mermaid-tools/references/setup_and_troubleshooting.md` for setup guide +- **statusline-generator**: See `daymade-claude-code/statusline-generator/references/color_codes.md` for customization - **teams-channel-post-writer**: See `teams-channel-post-writer/references/writing-guidelines.md` for quality standards - **repomix-unmixer**: See `repomix-unmixer/references/repomix-format.md` for format specifications - **skill-creator**: See `skill-creator/SKILL.md` for complete skill creation workflow @@ -1872,18 +2269,18 @@ Each skill includes: - **cli-demo-generator**: See `cli-demo-generator/references/vhs_syntax.md` for VHS syntax and `cli-demo-generator/references/best_practices.md` for demo guidelines - **cloudflare-troubleshooting**: See `cloudflare-troubleshooting/references/api_overview.md` for API documentation - **ui-designer**: See `ui-designer/SKILL.md` for design system extraction workflow -- **ppt-creator**: See `ppt-creator/references/WORKFLOW.md` for 9-stage creation process and `ppt-creator/references/ORCHESTRATION_OVERVIEW.md` for automation +- **ppt-creator**: See `daymade-docs/ppt-creator/references/WORKFLOW.md` for 9-stage creation process and `daymade-docs/ppt-creator/references/ORCHESTRATION_OVERVIEW.md` for automation - **youtube-downloader**: See `youtube-downloader/SKILL.md` for usage examples and troubleshooting - **repomix-safe-mixer**: See `repomix-safe-mixer/references/common_secrets.md` for detected credential patterns - **video-comparer**: See `video-comparer/references/video_metrics.md` for quality metrics interpretation and `video-comparer/references/configuration.md` for customization options - **transcript-fixer**: See `transcript-fixer/references/workflow_guide.md` for step-by-step workflows and `transcript-fixer/references/team_collaboration.md` for collaboration patterns - **qa-expert**: See `qa-expert/references/master_qa_prompt.md` for autonomous execution (100x speedup) and `qa-expert/references/google_testing_standards.md` for AAA pattern and OWASP testing - **prompt-optimizer**: See `prompt-optimizer/references/ears_syntax.md` for EARS transformation patterns, `prompt-optimizer/references/domain_theories.md` for theory catalog, and `prompt-optimizer/references/examples.md` for complete transformations -- **claude-code-history-files-finder**: See `claude-code-history-files-finder/references/session_file_format.md` for JSONL structure and `claude-code-history-files-finder/references/workflow_examples.md` for recovery workflows -- **docs-cleaner**: See `docs-cleaner/SKILL.md` for consolidation workflows +- **claude-code-history-files-finder**: See `daymade-claude-code/claude-code-history-files-finder/references/session_file_format.md` for JSONL structure and `daymade-claude-code/claude-code-history-files-finder/references/workflow_examples.md` for recovery workflows +- **docs-cleaner**: See `daymade-docs/docs-cleaner/SKILL.md` for consolidation workflows - **deep-research**: See `deep-research/references/research_report_template.md` for report structure and `deep-research/references/source_quality_rubric.md` for source triage -- **pdf-creator**: See `pdf-creator/SKILL.md` for PDF conversion and font setup -- **claude-md-progressive-disclosurer**: See `claude-md-progressive-disclosurer/SKILL.md` for CLAUDE.md optimization workflow +- **pdf-creator**: See `daymade-docs/pdf-creator/SKILL.md` for PDF conversion and font setup +- **claude-md-progressive-disclosurer**: See `daymade-claude-code/claude-md-progressive-disclosurer/SKILL.md` for CLAUDE.md optimization workflow - **skills-search**: See `skills-search/SKILL.md` for CCPM CLI commands and registry operations - **promptfoo-evaluation**: See `promptfoo-evaluation/references/promptfoo_api.md` for evaluation patterns - **iOS-APP-developer**: See `iOS-APP-developer/references/xcodegen-full.md` for XcodeGen options and project.yml details @@ -1892,24 +2289,33 @@ Each skill includes: - **skill-reviewer**: See `skill-reviewer/references/evaluation_checklist.md` for complete evaluation criteria, `skill-reviewer/references/pr_template.md` for PR templates, and `skill-reviewer/references/marketplace_template.json` for marketplace configuration - **github-contributor**: See `github-contributor/references/pr_checklist.md` for PR quality checklist, `github-contributor/references/project_evaluation.md` for project evaluation criteria, and `github-contributor/references/communication_templates.md` for issue/PR templates - **i18n-expert**: See `i18n-expert/SKILL.md` for complete i18n setup workflow, key architecture guidance, and audit procedures -- **claude-skills-troubleshooting**: See `claude-skills-troubleshooting/SKILL.md` for plugin troubleshooting workflow and architecture +- **claude-skills-troubleshooting**: See `daymade-claude-code/claude-skills-troubleshooting/SKILL.md` for plugin troubleshooting workflow and architecture - **fact-checker**: See `fact-checker/SKILL.md` for fact-checking workflow and claim verification process - **competitors-analysis**: See `competitors-analysis/SKILL.md` for evidence-based analysis workflow and `competitors-analysis/references/profile_template.md` for competitor profile template - **windows-remote-desktop-connection-doctor**: See `windows-remote-desktop-connection-doctor/references/windows_app_log_analysis.md` for log parsing patterns and `windows-remote-desktop-connection-doctor/references/avd_transport_protocols.md` for transport protocol details - **product-analysis**: See `product-analysis/SKILL.md` for workflow and `product-analysis/references/synthesis_methodology.md` for cross-agent weighting and recommendation logic - **excel-automation**: See `excel-automation/SKILL.md` for create/parse/control workflows and `excel-automation/references/formatting-reference.md` for formatting standards - **capture-screen**: See `capture-screen/SKILL.md` for CGWindowID-based screenshot workflows on macOS +- **continue-claude-work**: See `daymade-claude-code/continue-claude-work/SKILL.md` for local artifact recovery, drift checks, and resume workflow +- **scrapling-skill**: See `scrapling-skill/SKILL.md` for the CLI workflow and `scrapling-skill/references/troubleshooting.md` for verified Scrapling failure modes +- **ima-copilot**: See `ima-copilot/SKILL.md` for the wrapper architecture and routing, `ima-copilot/references/installation_flow.md` for the install deep dive, `ima-copilot/references/known_issues.md` for the issue registry and repair commands, and `ima-copilot/references/search_best_practices.md` for the fan-out strategy and 100-result truncation details +- **claude-export-txt-better**: See `daymade-claude-code/claude-export-txt-better/SKILL.md` for the workflow, `daymade-claude-code/claude-export-txt-better/scripts/fix-claude-export.py` for the reconstruction algorithm, and `daymade-claude-code/claude-export-txt-better/evals/` for real regression fixtures +- **douban-skill**: See `douban-skill/SKILL.md` for the export workflow and `douban-skill/references/troubleshooting.md` for the complete log of 7 tested scraping approaches and why each failed +- **terraform-skill**: See `terraform-skill/SKILL.md` for the full catalogue of operational traps organised by exact error → root cause → copy-paste fix +- **slides-creator**: See `slides-creator/SKILL.md` for the narrative-first workflow, `slides-creator/references/narrative-design-guide.md` for the ABCDEFG model, and `slides-creator/references/content-creation-first-law.md` for the universal content creation principle +- **debugging-network-issues**: See `debugging-network-issues/SKILL.md` for the falsification-first workflow, `debugging-network-issues/references/layered-isolation-experiment.md` for the multi-hop isolation pattern, and `debugging-network-issues/references/case-sse-rst-130s.md` for the real production case study +- **stepfun-tts**: See `stepfun-tts/SKILL.md` for the TTS+ASR decision tree and `stepfun-tts/references/migration_from_v2.md` for the `voice_label` → `instruction` migration playbook plus the censorship rewrite list ## 🛠️ Requirements - **Claude Code** 2.0.13 or higher - **Python 3.6+** (for scripts in multiple skills) - **gh CLI** (for github-ops) -- **markitdown** (for markdown-tools) +- **markitdown** (for doc-to-markdown) - **mermaid-cli** (for mermaid-tools) - **yt-dlp** (for youtube-downloader): `brew install yt-dlp` or `pip install yt-dlp` - **FFmpeg/FFprobe** (for video-comparer): `brew install ffmpeg`, `apt install ffmpeg`, or `winget install ffmpeg` -- **weasyprint, markdown** (for pdf-creator) +- **pandoc + weasyprint** (for pdf-creator): `brew install pandoc` + `pip install weasyprint` (or use Chrome as backend) - **VHS** (for cli-demo-generator): `brew install vhs` - **Jina.ai API key** (for twitter-reader): Free tier available at https://jina.ai/ - **asciinema** (optional, for cli-demo-generator interactive recording) @@ -1924,6 +2330,10 @@ Each skill includes: - **Codex CLI** (optional, for product-analysis multi-model mode) - **uv + openpyxl** (for excel-automation): `uv run --with openpyxl ...` - **macOS** (for capture-screen and excel-automation AppleScript control workflows) +- **Python 3.8+** (for continue-claude-work): bundled script for session extraction (no external dependencies) +- **uv + Scrapling CLI** (for scrapling-skill): `uv tool install 'scrapling[shell]'` and `scrapling install` for browser-backed fetches +- **Node.js 18+ + curl + unzip** (for ima-copilot): `npx skills` is fetched on demand from the npm registry; IMA OpenAPI credentials from [https://ima.qq.com/agent-interface](https://ima.qq.com/agent-interface) +- **StepFun API key** (for stepfun-tts): Available at [https://platform.stepfun.com/](https://platform.stepfun.com/) → API Keys ## ❓ FAQ @@ -2006,4 +2416,4 @@ If you find these skills useful, please: **Built with ❤️ using the skill-creator skill for Claude Code** -Last updated: 2026-01-22 | Marketplace version 1.23.0 +Last updated: 2026-04-11 | Marketplace version 1.46.0 diff --git a/README.zh-CN.md b/README.zh-CN.md index 5404eaaf..9494a450 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -6,15 +6,15 @@ [![简体中文](https://img.shields.io/badge/语言-简体中文-red)](./README.zh-CN.md) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -[![Skills](https://img.shields.io/badge/skills-41-blue.svg)](https://github.com/daymade/claude-code-skills) -[![Version](https://img.shields.io/badge/version-1.37.0-green.svg)](https://github.com/daymade/claude-code-skills) +[![Skills](https://img.shields.io/badge/skills-51-blue.svg)](https://github.com/daymade/claude-code-skills) +[![Version](https://img.shields.io/badge/version-1.51.0-green.svg)](https://github.com/daymade/claude-code-skills) [![Claude Code](https://img.shields.io/badge/Claude%20Code-2.0.13+-purple.svg)](https://claude.com/code) [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](./CONTRIBUTING.md) [![Maintenance](https://img.shields.io/badge/Maintained%3F-yes-green.svg)](https://github.com/daymade/claude-code-skills/graphs/commit-activity) -专业的 Claude Code 技能市场，提供 41 个生产就绪的技能，用于增强开发工作流。 +专业的 Claude Code 技能市场，提供 51 个生产就绪的技能，用于增强开发工作流。 ## 📑 目录 @@ -38,13 +38,35 @@ `skill-creator` 是一个**元技能**，它使你能够构建、验证和打包自己的 Claude Code 技能。它是这个市场中最重要的工具，因为它赋予你用自己的专业工作流扩展 Claude Code 的能力。 -### 为什么首选 skill-creator？ +### 为什么选这个 skill-creator？ -- **🎯 基础工具**：通过创建自己的技能来学习技能的工作原理 -- **🛠️ 完整工具包**：包含初始化、验证和打包脚本 -- **📖 最佳实践**：从生产就绪的示例中学习 -- **🚀 快速启动**：在几秒钟内生成技能模板 -- **✅ 质量保证**：内置验证确保你的技能符合标准 +这是 [Anthropic 官方 skill-creator](https://github.com/anthropics/skills/tree/main/skills/skill-creator) 的**生产强化版 fork**——从真实 skill 开发中踩过的坑里长出来的。 + +**官方告诉你"做什么"。我们还告诉你"别试什么"——以及为什么。** + +| 你想要... | 官方版 | 本 Fork | +|----------|--------|---------| +| 造之前先调研 | "Check available MCPs"（5 行） | 8 渠道搜索协议 + Adopt/Extend/Build 决策矩阵 | +| 交互式创建 skill | 纯文字指令 | 9 个结构化 AskUserQuestion 检查点——用户永远不丢上下文 | +| 避免常见错误 | 无指引 | 缓存编辑警告、前置依赖检查、安全扫描门禁 | +| 了解架构选项 | 未提及 | Inline vs Fork 决策指南（选错会静默破坏你的 skill） | +| 发布前验证 | 基本 YAML 检查 | 扩展验证器（全部 frontmatter 字段、路径引用完整性、空白字符问题） | +| 安全审查 | 无工具 | `security_scan.py` + gitleaks 集成——打包前硬门禁 | +| 从真实失败中学习 | 无失败案例 | 实战方法论 + 文档化的失败模式和踩坑记录 | + +**质量对比**（独立审计，8 个维度）： + +| 维度 | 官方版 | 本 Fork | +|------|--------|---------| +| 可操作性 | 7 | 9 | +| 错误预防 | 5 | 9 | +| 前置调研 | 4 | 9 | +| 对抗性审查 | 4 | 8 | +| 实战经验 | 3 | 8 | +| 用户体验 | 4 | 9 | +| **总分（/80）** | **42** | **65** | + +> 完整方法论：[skill-creator/references/skill-development-methodology.md](./skill-creator/references/skill-development-methodology.md) ### 快速安装 @@ -140,13 +162,50 @@ Marketplace 名称是 `daymade-skills`（来自 marketplace.json），安装插 claude plugin install skill-creator@daymade-skills ``` +**文档套件**（为文档工作流提供统一命名空间）： +```bash +claude plugin install daymade-docs@daymade-skills +``` + +这个套件会在同一个命名空间下暴露相关技能： + +```text +/daymade-docs:doc-to-markdown +/daymade-docs:mermaid-tools +/daymade-docs:pdf-creator +/daymade-docs:ppt-creator +/daymade-docs:docs-cleaner +/daymade-docs:meeting-minutes-taker +``` + +单技能插件仍然保留，适合更窄的安装范围和独立更新。文档技能的 canonical source 位于 `daymade-docs/`，因此套件和单个文档插件都从同一份源安装，同时保持 plugin cache 边界收窄。 + +**Claude Code 操作套件**（为 Claude Code 本体扩展工作流提供统一命名空间）： +```bash +claude plugin install daymade-claude-code@daymade-skills +``` + +一次安装即可获得扩展 Claude Code 本体的全部 power-user 技能——会话恢复、CLAUDE.md 调优、故障诊断、statusline 配置、导出修复与 marketplace 开发： + +```text +/daymade-claude-code:claude-code-history-files-finder +/daymade-claude-code:continue-claude-work +/daymade-claude-code:claude-skills-troubleshooting +/daymade-claude-code:claude-md-progressive-disclosurer +/daymade-claude-code:statusline-generator +/daymade-claude-code:claude-export-txt-better +/daymade-claude-code:marketplace-dev +``` + +安装后调用显示为 `daymade-claude-code:`，避免了单技能插件 `:` 的重复形式。 + **安装其他技能：** ```bash # GitHub 操作 claude plugin install github-ops@daymade-skills # 文档转换 -claude plugin install markdown-tools@daymade-skills +claude plugin install doc-to-markdown@daymade-skills # 图表生成 claude plugin install mermaid-tools@daymade-skills @@ -240,6 +299,24 @@ claude plugin install excel-automation@daymade-skills # macOS 程序化窗口截图工作流 claude plugin install capture-screen@daymade-skills + +# 基于本地会话产物续做中断的 Claude 工作 +claude plugin install continue-claude-work@daymade-skills + +# Scrapling CLI 抽取与故障排查 +claude plugin install scrapling-skill@daymade-skills + +# 腾讯 IMA 知识库伴侣与安装器 +claude plugin install ima-copilot@daymade-skills + +# 修复 Claude Code 导出 .txt 文件的断行问题 +claude plugin install claude-export-txt-better@daymade-skills + +# 导出豆瓣书影音游戏收藏到 CSV +claude plugin install douban-skill@daymade-skills + +# Terraform 实操陷阱与多环境可靠性模式 +claude plugin install terraform-skill@daymade-skills ``` 每个技能都可以独立安装 - 只选择你需要的！ @@ -313,7 +390,7 @@ CC-Switch 支持以下中国 AI 服务提供商： --- -### 2. **markdown-tools** - 文档转换套件 +### 2. **doc-to-markdown** - 文档转换套件 将文档转换为 markdown，支持 Windows/WSL 路径处理和 PDF 图片提取。 @@ -332,7 +409,7 @@ CC-Switch 支持以下中国 AI 服务提供商： **🎬 实时演示** -![Markdown 工具演示](./demos/markdown-tools/convert-docs.gif) +![Markdown 工具演示](./demos/doc-to-markdown/convert-docs.gif) --- @@ -561,7 +638,7 @@ CC-Switch 支持以下中国 AI 服务提供商： *即将推出* -📚 **文档**：参见 [ppt-creator/references/WORKFLOW.md](./ppt-creator/references/WORKFLOW.md) 了解 9 阶段创建流程 +📚 **文档**：参见 [daymade-docs/ppt-creator/references/WORKFLOW.md](./daymade-docs/ppt-creator/references/WORKFLOW.md) 了解 9 阶段创建流程 --- @@ -863,7 +940,7 @@ python3 scripts/analyze_sessions.py stats /path/to/session.jsonl --show-files *即将推出* -📚 **文档**：参见 [claude-code-history-files-finder/references/](./claude-code-history-files-finder/references/)： +📚 **文档**：参见 [daymade-claude-code/claude-code-history-files-finder/references/](./daymade-claude-code/claude-code-history-files-finder/references/)： - `session_file_format.md` - JSONL 结构和提取模式 - `workflow_examples.md` - 详细的恢复和分析工作流 @@ -964,7 +1041,7 @@ uv run --with weasyprint --with markdown scripts/md_to_pdf.py input.md output.pd *即将推出* -📚 **文档**：参见 [pdf-creator/SKILL.md](./pdf-creator/SKILL.md) 了解设置与工作流。 +📚 **文档**：参见 [daymade-docs/pdf-creator/SKILL.md](./daymade-docs/pdf-creator/SKILL.md) 了解设置与工作流。 **要求**：Python 3.8+，`weasyprint`、`markdown` @@ -993,7 +1070,7 @@ uv run --with weasyprint --with markdown scripts/md_to_pdf.py input.md output.pd *即将推出* -📚 **文档**：参见 [claude-md-progressive-disclosurer/SKILL.md](./claude-md-progressive-disclosurer/SKILL.md)。 +📚 **文档**：参见 [claude-md-progressive-disclosurer/SKILL.md](./daymade-claude-code/claude-md-progressive-disclosurer/SKILL.md)。 --- @@ -1444,7 +1521,7 @@ python3 scripts/enable_all_plugins.py daymade-skills *即将推出* -📚 **文档**：参见 [claude-skills-troubleshooting/SKILL.md](./claude-skills-troubleshooting/SKILL.md) 了解完整的故障排除工作流程和架构指导。 +📚 **文档**：参见 [claude-skills-troubleshooting/SKILL.md](./daymade-claude-code/claude-skills-troubleshooting/SKILL.md) 了解完整的故障排除工作流程和架构指导。 **要求**：无（使用 Claude Code 内置 Python） @@ -1479,7 +1556,7 @@ claude plugin install meeting-minutes-taker@daymade-skills *即将推出* -📚 **文档**：参见 [meeting-minutes-taker/SKILL.md](./meeting-minutes-taker/SKILL.md) 了解完整的工作流程和模板指导。 +📚 **文档**：参见 [daymade-docs/meeting-minutes-taker/SKILL.md](./daymade-docs/meeting-minutes-taker/SKILL.md) 了解完整的工作流程和模板指导。 **要求**：无 @@ -1791,6 +1868,301 @@ claude plugin install capture-screen@daymade-skills --- +### 42. **continue-claude-work** - 续做中断的 Claude 工作 + +从本地 `~/.claude` 会话产物中恢复可执行上下文，并在不重新打开旧交互会话的前提下继续实现工作。内置 Python 脚本实现智能上下文提取。 + +**使用场景：** +- 用户提供 Claude 会话 ID，希望继续上次的任务 +- 需要直接检查本地 `.claude` JSONL 文件，而不是运行 `claude --resume` +- 上一次会话被中断，需要重建下一步具体动作 +- 多 agent 工作流被中断，需要了解哪些 subagent 已完成 + +**主要功能：** +- Compact-boundary 感知提取 — 读取 Claude 自身的会话压缩摘要作为最高信噪比上下文 +- Subagent 工作流恢复 — 报告已完成与被中断的 subagent 及其最后输出 +- 会话结束原因检测 — 区分正常退出、中断（ctrl-c）、错误级联、废弃会话 +- 大小自适应策略 — 对小型（<500KB）和大型（>5MB）会话采用不同读取方式 +- 噪声过滤 — 跳过 progress/queue-operation/api_error 消息（占会话行数的 37-53%） +- 自会话排除、过期索引回退、MEMORY.md 集成、git 工作区状态 + +**示例用法：** +```bash +# 安装技能 +claude plugin install continue-claude-work@daymade-skills + +# 然后让 Claude 基于本地产物续做 +"continue work from session 123e4567-e89b-12d3-a456-426614174000" +"不用真的 resume，去 .claude 里找上下文继续做" +"查看上次会话做了什么，然后继续" +``` + +📚 **文档**：参见 [continue-claude-work/SKILL.md](./daymade-claude-code/continue-claude-work/SKILL.md)。 + +**要求**：Python 3.8+，用于工作区核对的 `git`。 + +--- + +### 43. **scrapling-skill** - 可靠的 Scrapling CLI 工作流 + +围绕 Scrapling CLI 提供经过验证的安装、排障与网页抽取工作流，用于从网页输出 HTML、Markdown 或纯文本。内置诊断脚本，可检查 extras 安装问题、Playwright 浏览器运行时，以及真实 URL 的烟测结果。 + +**使用场景：** +- 用户提到 Scrapling、`uv tool install scrapling` 或 `scrapling extract` +- 需要判断应该使用静态抓取还是浏览器抓取 +- 需要从微信公众号页面（`mp.weixin.qq.com`）提取正文 +- Scrapling 安装看似成功，但在 extras、浏览器运行时或 TLS 校验上失败 + +**主要功能：** +- 内置 `diagnose_scrapling.py`，检查 CLI、浏览器运行时与真实 URL 烟测 +- 经过验证的默认路径：先用 `extract get`，只有必要时再升级到 `extract fetch` +- 针对微信公众号文章的 `#js_content` 提取模式 +- 覆盖缺少 `click`、Playwright 运行时缺失、`curl: (60)` 证书问题等真实故障 +- 用文件大小和内容验证结果，而不是只看退出码 + +**示例用法：** +```bash +# 安装技能 +claude plugin install scrapling-skill@daymade-skills + +# 然后让 Claude 代你跑 Scrapling +"安装 Scrapling CLI 并验证配置" +"用 Scrapling 把这篇微信公众号文章提取成 Markdown" +"判断这个页面应不应该走浏览器抓取" +``` + +**🎬 实时演示** + +*即将推出* + +📚 **文档**：参见 [scrapling-skill/SKILL.md](./scrapling-skill/SKILL.md) 和 [scrapling-skill/references/troubleshooting.md](./scrapling-skill/references/troubleshooting.md)。 + +**要求**：Python 3.6+、`uv`、Scrapling CLI；如需浏览器抓取，还需要 Playwright 浏览器运行时。 + +--- + +### 44. **ima-copilot** - 腾讯 IMA 伴侣与安装器 + +围绕官方腾讯 IMA skill（`ima.qq.com`）的一站式包装层。通过 `npx skills add` 把官方 `ima-skill` 一键安装到 Claude Code、Codex、OpenClaw 三个平台；引导用户配置 API 凭据；在用户授权下检测并修复上游已知问题；提供按知识库置顶的个人化扇出搜索策略。 + +**使用场景：** +- 用户提到 IMA、腾讯 IMA、ima.qq.com，或需要安装官方 ima-skill +- 用户遇到 `Skipped loading skill(s) due to invalid SKILL.md` 这类 ima-skill 加载告警 +- 需要跨 IMA 知识库搜索并把某些精选库置顶 +- 需要配置或轮换 IMA API 凭据 +- 上游 ima-skill 发布了带 bug 的新版（例如子模块 SKILL.md 缺少 YAML frontmatter） + +**主要功能：** +- 通过 [vercel-labs/skills](https://github.com/vercel-labs/skills) 实现对 Claude Code / Codex / OpenClaw 三个目标的零配置安装，自动探测已安装的 agent，默认走 symlink 模式——修一次或升级一次，所有共享同一 canonical install 的 agent 自动同步 +- 凭据管理走 XDG 风格：`~/.config/ima/{client_id, api_key}`，同时支持 `IMA_OPENAPI_CLIENTID` / `IMA_OPENAPI_APIKEY` 环境变量兜底 +- 内置只读诊断脚本 `scripts/diagnose.sh`，用结构化 `✅/⚠️/❌` 报告覆盖安装状态、凭据 liveness、以及所有已知上游问题 +- 内置 `scripts/search_fanout.py`，实现客户端跨知识库扇出搜索，支持优先库置顶、子集库过滤、100 条静默截断检测，以及订阅只读库的权限差异分组 +- 严格的包装层架构：永不 vendor 上游文件，永不 fork，每一次修复都是运行时指令 + 明确用户授权 + 自动带时间戳的 `/tmp` 备份 +- 针对 frontmatter 缺失问题提供两种可选修复策略：A 策略（把子模块改名为 `MODULE.md` 并 patch 根 SKILL.md 引用，尊重上游设计意图）或 B 策略（仅追加最小 frontmatter，最小化差异） +- 个人化偏好通过 `~/.config/ima/copilot.json` 声明，仓库只提供示例模板 `config-template/copilot.json.example`，不预设任何真实知识库名 + +**示例用法：** +```bash +# 安装技能 +claude plugin install ima-copilot@daymade-skills + +# 然后让 Claude 代你跑完整个流程 +"装一下 ima-skill 并配置我的 IMA API key" +"对我的 ima-skill 做一次诊断，有问题就修" +"在我的 IMA 知识库里搜 embedding 模型对比，精选库置顶" +``` + +**🎬 实时演示** + +*即将推出* + +📚 **文档**：参见 [ima-copilot/SKILL.md](./ima-copilot/SKILL.md) 和 [ima-copilot/references/known_issues.md](./ima-copilot/references/known_issues.md)。 + +**要求**：Node.js 18+（`npx skills` 在按需拉取）、`curl`、`unzip`、Python 3.6+；从 [https://ima.qq.com/agent-interface](https://ima.qq.com/agent-interface) 获取 IMA OpenAPI 凭据。 + +--- + +### 45. **claude-export-txt-better** - 修复 Claude Code 导出文件的断行 + +重建 Claude Code 导出的 `.txt` 对话文件中被硬换行切坏的表格、段落、路径和工具调用输出。附带 53 项自动校验套件（文件无关，能捕捉 over-/under-merge 回归）。 + +**使用场景：** +- 用户的 Claude Code 导出文件被固定列宽换行搞坏了表格、路径或工具输出 +- 用户提到"修复导出""修复对话""让导出可读" +- 用户有匹配 `YYYY-MM-DD-HHMMSS-*.txt` 的文件 +- 用户想在分享或归档前后处理 `/export` 的输出 + +**主要功能：** +- 确定性的 Python 脚本（`fix-claude-export.py`），带 `--stats` 模式查看前后指标 +- 53 项自动校验器（`validate-claude-export-fix.py`），捕捉回归 +- evals 目录带真实 fixture 案例 +- 零外部依赖，只需 `uv` 和 Python 3.8+ + +**示例用法：** +```bash +# 修复并显示统计 +uv run daymade-claude-code/claude-export-txt-better/scripts/fix-claude-export.py broken.txt --stats + +# 自定义输出路径 +uv run daymade-claude-code/claude-export-txt-better/scripts/fix-claude-export.py broken.txt -o fixed.txt + +# 校验修复结果 +uv run daymade-claude-code/claude-export-txt-better/scripts/validate-claude-export-fix.py broken.txt fixed.txt +``` + +**🎬 实时演示** + +*即将推出* + +📚 **文档**：参见 [claude-export-txt-better/SKILL.md](./daymade-claude-code/claude-export-txt-better/SKILL.md) 和打包在内的 `evals/` fixture。 + +**要求**：Python 3.8+、`uv` 包管理器。 + +--- + +### 46. **douban-skill** - 豆瓣收藏导出与同步 + +通过逆向的 Frodo API 导出和同步豆瓣书影音游戏收藏到本地 CSV 文件。全量导出覆盖所有历史，RSS 增量同步保持每日更新。无需登录、无需 cookies、无需浏览器——只要一个用户 ID 就能跑通。 + +**使用场景：** +- 用户想备份自己的豆瓣读书/观影/听歌/游戏历史 +- 用户提到 豆瓣、douban、读书记录、观影记录、书影音 +- 用户需要增量同步最近的豆瓣动态 +- 用户想要 Excel 兼容的 CSV 输出（UTF-8 BOM） + +**主要功能：** +- 全量导出全部 4 类（书/影/音/游）通过 Frodo API +- RSS 增量同步每日更新（每个 feed 最近约 10 条） +- 预检查用户 ID 有效性（错误 ID 立刻失败） +- UTF-8 BOM CSV 输出，Excel 兼容，跨平台 +- 内置故障日志，记录 7 种被测试过的抓取方案以及每种为什么失败（豆瓣的 PoW 挑战封锁所有网页抓取——只有 Frodo API 可行） +- `.gitleaks.toml` allowlist 处理公开的 Android APK 凭据 + +**示例用法：** +```bash +# 全量导出用户收藏 +uv run douban-skill/scripts/douban-frodo-export.py + +# RSS 增量同步（每类最近 10 条左右） +uv run douban-skill/scripts/douban-rss-sync.py +``` + +**🎬 实时演示** + +*即将推出* + +📚 **文档**：参见 [douban-skill/SKILL.md](./douban-skill/SKILL.md) 和 [douban-skill/references/troubleshooting.md](./douban-skill/references/troubleshooting.md) 查看所有被拒方案的完整故障日志。 + +**要求**：Python 3.8+、`uv` 包管理器。无需登录或 cookies。 + +--- + +### 47. **terraform-skill** - Terraform 实操陷阱 + +来自真实 Terraform 部署的失败模式——每一条都对应一次真实事故。组织为*确切报错 → 根本原因 → 复制粘贴修复*。覆盖 provisioner 时序竞争、SSH 连接冲突、多环境隔离、DNS 记录重复、数据卷权限、数据库 bootstrap 缺口、快照跨环境污染、Cloudflare 凭据格式错误、Caddyfile/compose 里的硬编码域名，以及 init-data-only-on-first-boot 陷阱。 + +**使用场景：** +- 写 `null_resource` provisioner 或 `remote-exec` SSH 到新实例 +- 做多环境（prod/staging/dev）Terraform + 共享模块 +- 调试 `terraform apply` 后一直 Restarting/unhealthy 的容器 +- 遇到 remote-exec 的 "docker: not found"、local-exec 的 rsync connection drops、或 TLS 证书错误 +- 重跑时遇到 drift 或 provisioner 失败 +- 配置 Caddy/网关资源和 Cloudflare 凭据 + +**主要功能：** +- 每个陷阱都有可复制粘贴的 `.hcl` 片段，不是抽象建议 +- 覆盖 cloud-init、Docker、file provisioner、DNS、TLS、快照、跨环境污染 +- 每个模式都标了确切症状，方便 grep 快速定位 + +**示例用法：** +```bash +# 在 Terraform 工作中自然触发这个 skill +"我的 null_resource provisioner apply 后报 'docker: not found'" +"我的 rsync local-exec 报 'connection unexpectedly closed'" +"帮我写一个多环境 Terraform setup，避免快照跨环境污染" +``` + +**🎬 实时演示** + +*即将推出* + +📚 **文档**：参见 [terraform-skill/SKILL.md](./terraform-skill/SKILL.md) 和打包在内的 `references/` 查看详细修复模式。 + +**要求**：无（只需要 Terraform 相关知识；无运行时依赖）。 + +--- + +### 48. **slides-creator** - 叙事优先的幻灯片创建 + +引导用户完成结构化叙事设计（ABCDEFG 模型），然后将视觉生成委托给 `baoyu-slide-deck`。专注于机器做不到的事——与人类的叙事共创。 + +**使用场景：** +- 从用户内容创建演示文稿、幻灯片或 PPT +- 将文章、转录稿或笔记转化为视觉幻灯片 +- 为演讲和工作坊设计叙事弧线 + +**主要功能：** +- Phase 0：源材料收集（优先使用用户自己的文字） +- Phase 1：使用 ABCDEFG 模型进行叙事结构讨论 +- Phase 2：机器可读输入的内容结构化 +- Phase 3-5：将视觉生成委托给 baoyu-slide-deck +- Phase 6：目录重组和讲者备注提取的后处理 + +**示例用法：** +```bash +# 自然触发 skill +"帮我把我的文章做成幻灯片" +"从我的演讲转录稿创建演示文稿" +"我需要一个 20 分钟的工作坊演示" +``` + +**要求**：需要 baoyu-slide-deck skill 进行视觉生成。 + +--- + +### 49. **debugging-network-issues** - 证据驱动的网络问题排查 + +针对网络、流式、协议层 bug 的"先证伪、再下结论"方法论。源自一次真实的 5 小时 SSE 生产事故——堆假设浪费的几个小时，10 分钟分层实验就能解决。 + +**使用场景：** +- 连接重置（`ECONNRESET`、HTTP/2 `RST_STREAM`、`INTERNAL_ERROR`） +- SSE / 长轮询挂起或定时断开（60s、100s、130s） +- CDN / 代理 / CGNAT 空闲超时事件 +- "时灵时不灵 / N 秒后必断"模式 +- 多跳系统（client → CDN → LB → reverse proxy → app → upstream）症状可能来自多层 + +**主要功能：** +- 分层隔离实验：让同一逻辑请求走三条以上、每条仅差一跳的路径 +- 环境变量门控的运行时埋点（不污染生产代码） +- 反审查四问过滤器，挑战单因果假设 +- 内置探针脚本（`layered-isolation-probe.sh`、`mock-idle-upstream.py`） +- 真实案例：CGNAT 130s 空闲超时导致的 SSE RST_STREAM + +**要求**：无（方法论 + 可移植的 shell/Python 探针）。 + +--- + +### 50. **stepfun-tts** - 阶跃 StepAudio 2.5 TTS + ASR + +用 StepFun 阶跃的 StepAudio 2.5 系列做中文 / 日语语音合成与长音频转写。封装了三个会浪费时间的非显然坑：`voice_label` 移除、`/v1/audio/asr/sse` 端点、更严的审查。 + +**使用场景：** +- 带情感和韵律控制的中 / 日语 TTS +- 长音频转写（单次最长 ~30 分钟、32K context、~100x RTF） +- 从 `step-tts-2` 迁移到 `stepaudio-2.5-tts`（`voice_label` → `instruction` 是破坏性变更） +- 遇到 StepFun 审查拦截或端点错误 + +**主要功能：** +- `stepaudio-2.5-tts`：用 `instruction`（≤200 字自然语言情绪）+ 文中 `()` 行内韵律 +- `stepaudio-2.5-asr`：SSE 流式 + base64 音频（避开误导性的 "model not supported" 错误） +- 内置 `tts_generate.py`（含 `--batch `）、`asr_transcribe.py`、`ab_compare.sh` +- API key 解析顺序：`$STEPFUN_API_KEY` → `${CLAUDE_PLUGIN_DATA}/config.json` 兜底 +- `references/migration_from_v2.md` 给出审查拦截的改写策略 + +**要求**：StepFun API key（https://platform.stepfun.com/）。 + +--- + ## 🎬 交互式演示画廊 想要在一个地方查看所有演示并具有点击放大功能？访问我们的[交互式演示画廊](./demos/index.html)或浏览[演示目录](./demos/)。 @@ -1801,7 +2173,7 @@ claude plugin install capture-screen@daymade-skills 使用 **github-ops** 简化 PR 创建、问题管理和 API 操作。 ### 文档处理 -结合 **markdown-tools** 进行文档转换和 **mermaid-tools** 进行图表生成，创建全面的文档。使用 **llm-icon-finder** 添加品牌图标。 +结合 **doc-to-markdown** 进行文档转换和 **mermaid-tools** 进行图表生成，创建全面的文档。使用 **llm-icon-finder** 添加品牌图标。 ### 调研与分析 使用 **deep-research** 生成格式可控的调研报告，包含证据表与引用。与 **fact-checker** 结合用于验证关键结论，或与 **twitter-reader** 结合收集社媒资料。 @@ -1822,7 +2194,7 @@ claude plugin install capture-screen@daymade-skills 使用 **skill-creator**（参见上面的[必备技能](#-必备技能skill-creator)部分）构建、验证和打包你自己的 Claude Code 技能，遵循最佳实践。 ### 演示文稿与商务沟通 -使用 **ppt-creator** 生成具有数据可视化、结构化叙事和完整 PPTX 输出的专业幻灯片，用于推介、评审和主题演讲。 +使用 **ppt-creator** 生成具有数据可视化、结构化叙事和完整 PPTX 输出的专业幻灯片，用于推介、评审和主题演讲。使用 **slides-creator** 进行叙事优先的幻灯片设计——它引导你完成 ABCDEFG 叙事框架，优先收集你的原始内容，然后将视觉生成委托给 baoyu-slide-deck。非常适合需要将现有文章、转录稿或演讲转化为视觉幻灯片的场景。 ### 视频质量分析 使用 **video-comparer** 分析压缩结果、评估编解码器性能并生成交互式比较报告。与 **youtube-downloader** 结合使用以比较不同质量的下载。 @@ -1854,6 +2226,12 @@ claude plugin install capture-screen@daymade-skills ### 会话历史与文件恢复 使用 **claude-code-history-files-finder** 从之前的 Claude Code 会话中恢复已删除的文件、在对话历史中搜索特定实现，或跟踪文件随时间的演变。对于恢复意外删除的代码或查找你记得但找不到的功能实现至关重要。 +### 续做中断的 Claude 会话 +使用 **continue-claude-work** 从本地 `~/.claude` 产物中恢复最后一个可执行请求，并在不重新打开原始会话的情况下继续实现。若还需要跨会话搜索、统计分析或恢复已删除文件，可与 **claude-code-history-files-finder** 配合使用。 + +### 网页提取与微信公众号文章 +使用 **scrapling-skill** 安装并验证 Scrapling CLI，判断应使用静态抓取还是浏览器抓取，并从 `mp.weixin.qq.com` 等页面提取干净的 Markdown。可与 **deep-research** 配合，将抓取内容整理为结构化报告，或与 **docs-cleaner** 配合清理抽取后的文章内容。 + ### 文档维护 使用 **docs-cleaner** 在保留有价值内容的同时整合冗余文档。非常适合在快速开发阶段后清理文档扩散或将重叠的文档合并为权威来源。 @@ -1867,7 +2245,7 @@ claude plugin install capture-screen@daymade-skills 使用 **iOS-APP-developer** 配置 XcodeGen 项目，处理 SPM 依赖、签名与部署问题。 ### Twitter/X 内容研究 -使用 **twitter-reader** 无需 JavaScript 渲染或身份验证即可获取推文内容。非常适合记录社交媒体讨论、归档话题、分析推文内容或从 Twitter/X 收集参考资料。与 **markdown-tools** 结合可将获取的内容转换为其他格式，或与 **repomix-safe-mixer** 结合安全地打包研究集合。 +使用 **twitter-reader** 无需 JavaScript 渲染或身份验证即可获取推文内容。非常适合记录社交媒体讨论、归档话题、分析推文内容或从 Twitter/X 收集参考资料。与 **doc-to-markdown** 结合可将获取的内容转换为其他格式，或与 **repomix-safe-mixer** 结合安全地打包研究集合。 ### macOS 系统维护与磁盘空间恢复 使用 **macos-cleaner** 以安全优先的方式智能分析和恢复 macOS 上的磁盘空间。与盲目删除的一键清理工具不同，macos-cleaner 解释每个文件是什么、按风险级别分类（🟢/🟡/🔴），并在任何删除前需要明确确认。非常适合处理 Docker/Homebrew/npm/pip 缓存膨胀的开发者、希望了解存储空间消耗的用户，或任何重视透明度而非自动化的人。结合基于脚本的精度和可选的 Mole 可视化工具集成以实现混合工作流。 @@ -1893,6 +2271,24 @@ claude plugin install capture-screen@daymade-skills ### 插件与技能故障排除 使用 **claude-skills-troubleshooting** 诊断和解决 Claude Code 插件和技能配置问题。调试为什么插件显示已安装但未显示在可用技能列表中、了解 installed_plugins.json 与 settings.json enabledPlugins 架构，以及批量启用市场中缺失的插件。非常适合市场维护者调试安装问题、开发者调试技能激活，或任何对 GitHub #17832 自动启用 bug 感到困惑的人。 +### 腾讯 IMA 知识库工作流 +使用 **ima-copilot** 把官方腾讯 IMA skill 一键装到 Claude Code / Codex / OpenClaw 三个平台，引导配置 API 凭据，在用户授权下检测与修复上游已知问题，并在所有 IMA 知识库上跑带优先级置顶的个人化扇出搜索。因为整个架构是包装层而不是 fork，上游升级永远不会和你的修复冲突——每一次修复都是运行时指令，不是 shipped patch。特别适合同时使用多个 coding agent 的 IMA 重度用户，或遇到过 "Skipped loading skill(s) due to invalid SKILL.md" 告警的人。 + +### Claude Code 导出后处理 +使用 **claude-export-txt-better** 在归档或分享前清理 `/export` 的输出。默认导出格式在固定列宽硬换行，表格、路径、工具调用块一打开就散架。这个 skill 重建原始结构，并用 53 项自动校验立刻抓到回归。 + +### 个人数据备份（豆瓣） +使用 **douban-skill** 把豆瓣书影音历史备份到 CSV。豆瓣没有官方导出——2018 年公共 API 就关停了，所有网页抓取都被 PoW 挑战卡住。这个 skill 用官方 Android app 同一套 Frodo API，不需要登录也不需要 cookies。内置 7 个被拒方案的完整故障日志，省掉你几个小时的弯路。 + +### Terraform 与 IaC 故障排查 +使用 **terraform-skill** 当 `terraform apply` 在 provisioner 步骤失败、新实例遇到 "docker: not found"、或多环境 setup 意外共享快照时。Skill 里每一条都是*确切报错 → 根本原因 → 复制粘贴修复*三元组，来自真实事故。特别适合曾经被 cloud-init 的时序竞争、local-exec 里 rsync 连接断开、或者 Caddyfile 里硬编码域名搞掉一个周末的人。 + +### 网络、流式与协议层调试 +使用 **debugging-network-issues** 应对症状和"显然原因"对不上的场景：HTTP/2 `RST_STREAM`、SSE 在 60s/100s/130s 整点卡死、"时灵时不灵"故障、或 CDN / 代理 / CGNAT 链路上的空闲超时事件。Skill 用**分层隔离实验**（同一逻辑请求走三条以上、每条仅差一跳的路径）替代假设堆叠，再加一套反审查模式——只在假设被**证伪**而不是单纯被"证实"之后才上 fix。 + +### 中文 TTS 与长音频转写（StepFun 阶跃） +使用 **stepfun-tts** 进行中 / 日语语音合成（通过 `instruction` + 行内 `()` 控制情绪与韵律），或单次最长 30 分钟的长音频转写（32K context、~100x RTF）。封装了让 StepAudio 2.5 新用户必踩的三个破坏性变更：`voice_label` 移除、`/v1/audio/asr/sse` 端点错位、更严的审查规则。可与 **transcript-fixer** 组合做 ASR 后处理，或与 **meeting-minutes-taker** 把长录音变成结构化纪要。 + ## 📚 文档 每个技能包括： @@ -1904,9 +2300,9 @@ claude plugin install capture-screen@daymade-skills ### 快速链接 - **github-ops**：参见 `github-ops/references/api_reference.md` 了解 API 文档 -- **markdown-tools**：参见 `markdown-tools/references/conversion-examples.md` 了解转换场景 -- **mermaid-tools**：参见 `mermaid-tools/references/setup_and_troubleshooting.md` 了解设置指南 -- **statusline-generator**：参见 `statusline-generator/references/color_codes.md` 了解自定义 +- **doc-to-markdown**：参见 `daymade-docs/doc-to-markdown/references/conversion-examples.md` 了解转换场景 +- **mermaid-tools**：参见 `daymade-docs/mermaid-tools/references/setup_and_troubleshooting.md` 了解设置指南 +- **statusline-generator**：参见 `daymade-claude-code/statusline-generator/references/color_codes.md` 了解自定义 - **teams-channel-post-writer**：参见 `teams-channel-post-writer/references/writing-guidelines.md` 了解质量标准 - **repomix-unmixer**：参见 `repomix-unmixer/references/repomix-format.md` 了解格式规范 - **skill-creator**：参见 `skill-creator/SKILL.md` 了解完整的技能创建工作流 @@ -1914,18 +2310,18 @@ claude plugin install capture-screen@daymade-skills - **cli-demo-generator**：参见 `cli-demo-generator/references/vhs_syntax.md` 了解 VHS 语法和 `cli-demo-generator/references/best_practices.md` 了解演示指南 - **cloudflare-troubleshooting**：参见 `cloudflare-troubleshooting/references/api_overview.md` 了解 API 文档 - **ui-designer**：参见 `ui-designer/SKILL.md` 了解完整的设计系统提取工作流 -- **ppt-creator**：参见 `ppt-creator/references/WORKFLOW.md` 了解 9 阶段创建流程和 `ppt-creator/references/ORCHESTRATION_OVERVIEW.md` 了解自动化 +- **ppt-creator**：参见 `daymade-docs/ppt-creator/references/WORKFLOW.md` 了解 9 阶段创建流程和 `daymade-docs/ppt-creator/references/ORCHESTRATION_OVERVIEW.md` 了解自动化 - **youtube-downloader**：参见 `youtube-downloader/SKILL.md` 了解使用示例和故障排除 - **repomix-safe-mixer**：参见 `repomix-safe-mixer/references/common_secrets.md` 了解检测到的凭据模式 - **video-comparer**：参见 `video-comparer/references/video_metrics.md` 了解质量指标解释和 `video-comparer/references/configuration.md` 了解自定义选项 - **transcript-fixer**：参见 `transcript-fixer/references/workflow_guide.md` 了解分步工作流和 `transcript-fixer/references/team_collaboration.md` 了解协作模式 - **qa-expert**：参见 `qa-expert/references/master_qa_prompt.md` 了解自主执行（100 倍加速）和 `qa-expert/references/google_testing_standards.md` 了解 AAA 模式和 OWASP 测试 - **prompt-optimizer**：参见 `prompt-optimizer/references/ears_syntax.md` 了解 EARS 转换模式、`prompt-optimizer/references/domain_theories.md` 了解理论目录和 `prompt-optimizer/references/examples.md` 了解完整转换示例 -- **claude-code-history-files-finder**：参见 `claude-code-history-files-finder/references/session_file_format.md` 了解 JSONL 结构和 `claude-code-history-files-finder/references/workflow_examples.md` 了解恢复工作流 -- **docs-cleaner**：参见 `docs-cleaner/SKILL.md` 了解整合工作流 +- **claude-code-history-files-finder**：参见 `daymade-claude-code/claude-code-history-files-finder/references/session_file_format.md` 了解 JSONL 结构和 `daymade-claude-code/claude-code-history-files-finder/references/workflow_examples.md` 了解恢复工作流 +- **docs-cleaner**：参见 `daymade-docs/docs-cleaner/SKILL.md` 了解整合工作流 - **deep-research**：参见 `deep-research/references/research_report_template.md` 了解报告结构，并参见 `deep-research/references/source_quality_rubric.md` 了解来源分级标准 -- **pdf-creator**：参见 `pdf-creator/SKILL.md` 了解 PDF 转换与字体设置 -- **claude-md-progressive-disclosurer**：参见 `claude-md-progressive-disclosurer/SKILL.md` 了解 CLAUDE.md 优化工作流 +- **pdf-creator**：参见 `daymade-docs/pdf-creator/SKILL.md` 了解 PDF 转换与字体设置 +- **claude-md-progressive-disclosurer**：参见 `daymade-claude-code/claude-md-progressive-disclosurer/SKILL.md` 了解 CLAUDE.md 优化工作流 - **skills-search**：参见 `skills-search/SKILL.md` 了解 CCPM CLI 命令和注册表操作 - **promptfoo-evaluation**：参见 `promptfoo-evaluation/references/promptfoo_api.md` 了解评测模式 - **iOS-APP-developer**：参见 `iOS-APP-developer/references/xcodegen-full.md` 了解 XcodeGen 选项与 project.yml 细节 @@ -1934,20 +2330,29 @@ claude plugin install capture-screen@daymade-skills - **skill-reviewer**：参见 `skill-reviewer/references/evaluation_checklist.md` 了解完整评估标准、`skill-reviewer/references/pr_template.md` 了解 PR 模板、`skill-reviewer/references/marketplace_template.json` 了解 marketplace 配置 - **github-contributor**：参见 `github-contributor/references/pr_checklist.md` 了解 PR 质量清单、`github-contributor/references/project_evaluation.md` 了解项目评估标准、`github-contributor/references/communication_templates.md` 了解 issue/PR 沟通模板 - **i18n-expert**：参见 `i18n-expert/SKILL.md` 了解完整的 i18n 设置工作流程、键架构指导和审计程序 -- **claude-skills-troubleshooting**：参见 `claude-skills-troubleshooting/SKILL.md` 了解插件故障排除工作流程和架构 +- **claude-skills-troubleshooting**：参见 `daymade-claude-code/claude-skills-troubleshooting/SKILL.md` 了解插件故障排除工作流程和架构 - **fact-checker**：参见 `fact-checker/SKILL.md` 了解事实核查工作流程和声明验证过程 - **competitors-analysis**：参见 `competitors-analysis/SKILL.md` 了解证据驱动的分析工作流程和 `competitors-analysis/references/profile_template.md` 了解竞品档案模板 - **windows-remote-desktop-connection-doctor**：参见 `windows-remote-desktop-connection-doctor/references/windows_app_log_analysis.md` 了解日志解析模式和 `windows-remote-desktop-connection-doctor/references/avd_transport_protocols.md` 了解传输协议详情 - **product-analysis**：参见 `product-analysis/SKILL.md` 了解工作流，参见 `product-analysis/references/synthesis_methodology.md` 了解跨代理加权与推荐逻辑 - **excel-automation**：参见 `excel-automation/SKILL.md` 了解创建/解析/控制工作流，参见 `excel-automation/references/formatting-reference.md` 了解格式规范 - **capture-screen**：参见 `capture-screen/SKILL.md` 了解基于 CGWindowID 的 macOS 截图流程 +- **continue-claude-work**：参见 `daymade-claude-code/continue-claude-work/SKILL.md` 了解本地会话产物恢复、漂移检查与续做流程 +- **scrapling-skill**：参见 `scrapling-skill/SKILL.md` 了解 CLI 工作流，参见 `scrapling-skill/references/troubleshooting.md` 了解已验证的 Scrapling 故障模式 +- **ima-copilot**：参见 `ima-copilot/SKILL.md` 了解包装层架构与路由规则，参见 `ima-copilot/references/installation_flow.md` 了解安装流程细节，参见 `ima-copilot/references/known_issues.md` 了解已知问题清单与修复命令，参见 `ima-copilot/references/search_best_practices.md` 了解扇出搜索策略与 100 条截断处理 +- **claude-export-txt-better**：参见 `daymade-claude-code/claude-export-txt-better/SKILL.md` 了解工作流，参见 `daymade-claude-code/claude-export-txt-better/scripts/fix-claude-export.py` 了解重建算法，参见 `daymade-claude-code/claude-export-txt-better/evals/` 查看真实回归 fixture +- **douban-skill**：参见 `douban-skill/SKILL.md` 了解导出工作流，参见 `douban-skill/references/troubleshooting.md` 查看 7 种被测抓取方案及失败原因的完整日志 +- **terraform-skill**：参见 `terraform-skill/SKILL.md` 查看按确切报错 → 根本原因 → 复制粘贴修复组织的实操陷阱完整目录 +- **slides-creator**：参见 `slides-creator/SKILL.md` 了解叙事优先工作流，参见 `slides-creator/references/narrative-design-guide.md` 了解 ABCDEFG 模型，参见 `slides-creator/references/content-creation-first-law.md` 了解通用内容创作原则 +- **debugging-network-issues**：参见 `debugging-network-issues/SKILL.md` 了解证伪优先工作流，参见 `debugging-network-issues/references/layered-isolation-experiment.md` 了解多跳隔离模式，参见 `debugging-network-issues/references/case-sse-rst-130s.md` 查看真实生产案例 +- **stepfun-tts**：参见 `stepfun-tts/SKILL.md` 了解 TTS+ASR 决策树，参见 `stepfun-tts/references/migration_from_v2.md` 查看 `voice_label` → `instruction` 迁移手册和审查改写清单 ## 🛠️ 系统要求 - **Claude Code** 2.0.13 或更高版本 - **Python 3.6+**（用于多个技能中的脚本） - **gh CLI**（用于 github-ops） -- **markitdown**（用于 markdown-tools） +- **markitdown**（用于 doc-to-markdown） - **mermaid-cli**（用于 mermaid-tools） - **VHS**（用于 cli-demo-generator）：`brew install vhs` - **asciinema**（可选，用于 cli-demo-generator 交互式录制） @@ -1963,6 +2368,10 @@ claude plugin install capture-screen@daymade-skills - **Mole**（可选，用于 macos-cleaner 可视化清理）：从 https://github.com/tw93/Mole 下载 - **uv + openpyxl**（用于 excel-automation）：`uv run --with openpyxl ...` - **macOS**（用于 capture-screen 与 excel-automation 的 AppleScript 控制流程） +- **Python 3.8+**（用于 continue-claude-work）：内置脚本进行会话提取（无外部依赖） +- **uv + Scrapling CLI**（用于 scrapling-skill）：`uv tool install 'scrapling[shell]'`，浏览器抓取前运行 `scrapling install` +- **Node.js 18+ + curl + unzip**（用于 ima-copilot）：`npx skills` 按需从 npm registry 拉取；IMA OpenAPI 凭据从 [https://ima.qq.com/agent-interface](https://ima.qq.com/agent-interface) 获取 +- **StepFun API key**（用于 stepfun-tts）：在 [https://platform.stepfun.com/](https://platform.stepfun.com/) → API Keys 获取 ## ❓ 常见问题 @@ -2045,4 +2454,4 @@ claude plugin install skill-name@daymade-skills **使用 skill-creator 技能为 Claude Code 精心打造 ❤️** -最后更新：2026-01-22 | 市场版本 1.23.0 +最后更新：2026-04-11 | 市场版本 1.46.0 diff --git a/TECHNICAL_DEBT.md b/TECHNICAL_DEBT.md new file mode 100644 index 00000000..dcdac309 --- /dev/null +++ b/TECHNICAL_DEBT.md @@ -0,0 +1,242 @@ +# 技术债务清单 + +## 当前债务状况 + +| 债务类型 | 严重程度 | 预计修复时间 | 阻塞发布 | +|---------|---------|-------------|----------| +| 测试覆盖不足 | 🔴 极高 | 4周 | ✅ 是 | +| 缺少CI/CD | 🔴 极高 | 1周 | ✅ 是 | +| 错误处理不完善 | 🟠 高 | 2周 | ✅ 是 | +| 性能未优化 | 🟠 高 | 2周 | ❌ 否 | +| 文档不完整 | 🟡 中 | 1周 | ❌ 否 | +| 代码重复 | 🟡 中 | 1周 | ❌ 否 | + +## 详细债务清单 + +### 🔴 极高优先级 + +#### 1. 测试覆盖不足（5% → 目标80%） + +**现状:** +- 总源文件: 2,999 +- 测试文件: 5 +- 覆盖率: ~5% + +**缺失测试:** +- [ ] `inbox-engine.ts` - 核心逻辑无测试 +- [ ] `annotation-engine.ts` - 核心逻辑无测试 +- [ ] `sync-engine.ts` - 核心逻辑无测试 +- [ ] `tts-engine.ts` - 新增无测试 +- [ ] `reading-agent-sdk.ts` - 复杂逻辑无测试 +- [ ] 所有API routes - 无集成测试 + +**风险:** +- 重构困难 +- 回归bug无法发现 +- 无法 confident 地发布 + +**修复方案:** +```bash +# 优先级1: 核心引擎单元测试 +- inbox-engine.test.ts +- annotation-engine.test.ts +- sync-engine.test.ts + +# 优先级2: API集成测试 +- api/articles.test.ts +- api/scrape.test.ts +- api/sync.test.ts + +# 优先级3: E2E测试 +- scrape-to-read.spec.ts (已完成) +- reading-annotations.spec.ts (已完成) +- tts-stagehand.spec.ts (已完成) +``` + +#### 2. 缺少CI/CD流水线 + +**现状:** +- 无自动化测试 +- 无自动化部署 +- 无代码质量检查 + +**需要建立:** +- [ ] GitHub Actions workflow + - [ ] Lint check + - [ ] Type check + - [ ] Unit tests + - [ ] E2E tests + - [ ] Security scan + - [ ] Deploy to staging + - [ ] Deploy to production + +**参考实现:** +Omnivore 的 `.github/workflows/` 配置 + +#### 3. 错误处理不完善 + +**现状:** +- 大量 `throw error` 未处理 +- 用户看到的是原始错误信息 +- 无错误边界(Error Boundaries) + +**需要修复:** +- [ ] 统一错误处理中间件 +- [ ] 用户友好的错误提示 +- [ ] Sentry上报所有未捕获错误 +- [ ] React Error Boundaries + +### 🟠 高优先级 + +#### 4. 性能未优化 + +**已知问题:** +- [ ] 首屏加载无骨架屏 +- [ ] 大文章渲染卡顿 +- [ ] 图片懒加载不完善 +- [ ] 无虚拟滚动 + +**待测量指标:** +``` +LCP (Largest Contentful Paint): 目标 < 2.5s +FID (First Input Delay): 目标 < 100ms +CLS (Cumulative Layout Shift): 目标 < 0.1 +TTI (Time to Interactive): 目标 < 3.8s +``` + +#### 5. 数据库查询未优化 + +**潜在问题:** +- [ ] 缺少复合索引 +- [ ] N+1查询问题 +- [ ] 大数据表无分区 + +**需要:** +- [ ] 查询性能分析 +- [ ] EXPLAIN ANALYZE 所有慢查询 +- [ ] 连接池配置优化 + +### 🟡 中优先级 + +#### 6. 文档不完整 + +**缺失:** +- [ ] API文档 (OpenAPI/Swagger) +- [ ] 架构决策记录 (ADR) +- [ ] 部署指南 +- [ ] 贡献者指南 + +#### 7. 代码重复 + +**发现重复:** +- [ ] 日期格式化函数 (多处) +- [ ] API错误处理逻辑 (多处) +- [ ] 类型定义分散 + +## 还债计划 + +### Sprint 1: 测试基础 (Week 1-2) + +```bash +# 目标: 测试覆盖率 5% → 30% + +Week 1: +- [ ] 设置 Vitest + React Testing Library +- [ ] inbox-engine.ts 单元测试 +- [ ] annotation-engine.ts 单元测试 + +Week 2: +- [ ] sync-engine.ts 单元测试 +- [ ] tts-engine.ts 单元测试 +- [ ] 核心组件单元测试 +``` + +### Sprint 2: CI/CD + 错误处理 (Week 3-4) + +```bash +# 目标: 自动化流水线 + 错误监控 + +Week 3: +- [ ] GitHub Actions workflow +- [ ] 自动化测试触发 +- [ ] Sentry生产环境配置 + +Week 4: +- [ ] 错误边界组件 +- [ ] 统一错误处理 +- [ ] 用户友好错误提示 +``` + +### Sprint 3: 性能优化 (Week 5-6) + +```bash +# 目标: Web Vitals 达标 + +Week 5: +- [ ] 性能基线测量 +- [ ] 图片优化 +- [ ] 代码分割 + +Week 6: +- [ ] 虚拟滚动 (大文章) +- [ ] 骨架屏 +- [ ] 缓存策略 +``` + +### Sprint 4: 清理 (Week 7-8) + +```bash +# 目标: 代码质量提升 + +Week 7: +- [ ] 重构重复代码 +- [ ] 类型定义集中化 +- [ ] 数据库索引优化 + +Week 8: +- [ ] 文档完善 +- [ ] 架构图更新 +- [ ] 技术债务复盘 +``` + +## 债务预防 + +### 代码审查清单 + +- [ ] 新功能必须包含测试 +- [ ] 复杂逻辑必须有注释 +- [ ] API变更必须更新文档 +- [ ] 性能敏感代码必须有基准测试 + +### 自动化防护 + +```yaml +# .github/workflows/quality.yml +- name: Test Coverage + run: | + npm run test:coverage + # 失败如果覆盖率 < 60% + +- name: Bundle Size + run: | + npm run analyze + # 失败如果 bundle > 500KB + +- name: Performance Budget + run: | + lhci autorun + # 失败如果 Web Vitals 不达标 +``` + +## 债务追踪 + +| 日期 | 债务项 | 状态 | 备注 | +|-----|-------|------|------| +| 2025-04-12 | 测试覆盖 | 🟡 进行中 | 已规划E2E | +| 2025-04-12 | CI/CD | 🔴 未开始 | 优先级P0 | +| 2025-04-12 | 错误处理 | 🔴 未开始 | 优先级P0 | + +--- + +最后更新: Round 91 +下次审查: Round 95 diff --git a/asr-transcribe-to-text/.security-scan-passed b/asr-transcribe-to-text/.security-scan-passed new file mode 100644 index 00000000..d93e23f4 --- /dev/null +++ b/asr-transcribe-to-text/.security-scan-passed @@ -0,0 +1,4 @@ +Security scan passed +Scanned at: 2026-03-22T23:58:59.508059 +Tool: gitleaks + pattern-based validation +Content hash: 5dbbc175b8bfd6c6c8ab97f4112bf1f48eb489ef2a55375f88266deade644cd4 diff --git a/asr-transcribe-to-text/SKILL.md b/asr-transcribe-to-text/SKILL.md new file mode 100644 index 00000000..77994c56 --- /dev/null +++ b/asr-transcribe-to-text/SKILL.md @@ -0,0 +1,250 @@ +--- +name: asr-transcribe-to-text +description: Transcribes audio and video files to text using Qwen3-ASR. Supports two modes — local MLX inference on macOS Apple Silicon (no API key, 15-27x realtime) and remote API via vLLM/OpenAI-compatible endpoints. Auto-detects platform and recommends the best path. Triggers when the user wants to transcribe recordings, convert audio/video to text, do speech-to-text, or mentions ASR, Qwen ASR, 转录, 语音转文字, 录音转文字. Also triggers for meeting recordings, lectures, interviews, podcasts, screen recordings, or any audio/video file the user wants converted to text. +argument-hint: [audio-or-video-file-path ...] +--- + +# ASR Transcribe to Text + +Transcribe audio/video files to text using Qwen3-ASR. Two inference paths: + +| Mode | When | Speed | Cost | +|------|------|-------|------| +| **Local MLX** | macOS Apple Silicon | 15-27x realtime | Free | +| **Remote API** | Any platform, or when local unavailable | Depends on GPU | API/self-hosted | + +Configuration persists in `${CLAUDE_PLUGIN_DATA}/config.json`. + +## Step 0: Detect Platform and Load Config + +```bash +cat "${CLAUDE_PLUGIN_DATA}/config.json" 2>/dev/null +``` + +**If config exists**, read values and proceed to Step 1. + +**If config does not exist**, auto-detect platform first: + +```bash +python3 -c " +import sys, platform +is_mac_arm = sys.platform == 'darwin' and platform.machine() in ('arm64', 'aarch64') +print(f'Platform: {sys.platform} {platform.machine()}') +print(f'Apple Silicon: {is_mac_arm}') +if is_mac_arm: + print('RECOMMEND: local-mlx') +else: + print('RECOMMEND: remote-api') +" +``` + +Then use **AskUserQuestion** with platform-aware defaults: + +For **macOS Apple Silicon** (recommended: local): +``` +ASR setup — your Mac has Apple Silicon, so local transcription is recommended. + +Q1: Transcription mode? + A) Local MLX — runs on your Mac's GPU, no API key needed, 15-27x realtime (Recommended) + B) Remote API — send audio to a server (vLLM, Tailscale workstation, etc.) + +Q2: Does your network have an HTTP proxy that might intercept traffic? + A) Yes — bypass proxy for ASR traffic (Recommended if using Shadowrocket/Clash) + B) No — direct connection +``` + +For **other platforms** (recommended: remote): +``` +ASR setup — local MLX requires macOS Apple Silicon. Using remote API mode. + +Q1: ASR Endpoint URL? + A) http://workstation-4090-wsl:8002/v1/audio/transcriptions (Qwen3-ASR vLLM via Tailscale) + B) http://localhost:8002/v1/audio/transcriptions (Local server) + C) Custom URL + +Q2: Proxy bypass needed? + A) Yes (Recommended for Shadowrocket/Clash/corporate proxy) + B) No +``` + +Save config: +```bash +mkdir -p "${CLAUDE_PLUGIN_DATA}" +python3 -c " +import json +config = { + 'mode': 'MODE', # 'local-mlx' or 'remote-api' + 'model': 'MODEL_ID', # local: 'mlx-community/Qwen3-ASR-1.7B-8bit', remote: 'Qwen/Qwen3-ASR-1.7B' + 'max_tokens': 200000, # local only, critical for long audio + 'endpoint': 'URL', # remote only + 'noproxy': True, + 'max_timeout': 900 # remote only +} +with open('${CLAUDE_PLUGIN_DATA}/config.json', 'w') as f: + json.dump(config, f, indent=2) +print('Config saved.') +" +``` + +## Step 1: Extract Audio (if input is video) + +For video files (mp4, mov, mkv, avi, webm), extract as 16kHz mono WAV: + +```bash +ffmpeg -i INPUT_VIDEO -vn -acodec pcm_s16le -ar 16000 -ac 1 OUTPUT.wav -y +``` + +Audio files (wav, mp3, m4a, flac, ogg) can be used directly. Get duration: +```bash +ffprobe -v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 INPUT_FILE +``` + +**Cleanup**: After transcription succeeds, delete extracted WAV files to save disk space. + +## Step 2: Transcribe + +### Path A: Local MLX (macOS Apple Silicon) + +Use the bundled script — it handles model loading, chunking, and the critical `max_tokens` parameter: + +```bash +uv run ${CLAUDE_PLUGIN_ROOT}/scripts/transcribe_local_mlx.py \ + INPUT_AUDIO [INPUT_AUDIO2 ...] \ + --output-dir OUTPUT_DIR +``` + +The script loads the model once and transcribes all files sequentially (no GPU contention). For details on performance, model compatibility, and the max_tokens truncation issue, see `references/local_mlx_guide.md`. + +**Critical**: The upstream `mlx-audio` default `max_tokens=8192` silently truncates audio longer than ~40 minutes. The bundled script defaults to `200000`. If calling `model.generate()` directly, always pass `max_tokens=200000`. + +### Path B: Remote API + +**Health check first** (skip if already verified this session): +```bash +python3 -c " +import json, subprocess, sys +with open('${CLAUDE_PLUGIN_DATA}/config.json') as f: + cfg = json.load(f) +base = cfg['endpoint'].rsplit('/audio/', 1)[0] +noproxy = ['--noproxy', '*'] if cfg.get('noproxy', True) else [] +result = subprocess.run( + ['curl', '-s', '--max-time', '10'] + noproxy + [f'{base}/models'], + capture_output=True, text=True +) +if result.returncode != 0 or not result.stdout.strip(): + print(f'HEALTH CHECK FAILED: {base}/models', file=sys.stderr) + sys.exit(1) +print(f'Service healthy: {base}') +" +``` + +Read config and send via curl: + +```bash +python3 -c " +import json, subprocess, sys, os, tempfile +with open('${CLAUDE_PLUGIN_DATA}/config.json') as f: + cfg = json.load(f) +noproxy = ['--noproxy', '*'] if cfg.get('noproxy', True) else [] +timeout = str(cfg.get('max_timeout', 900)) +audio_file = 'AUDIO_FILE_PATH' +output_json = tempfile.mktemp(suffix='.json', prefix='asr_') + +result = subprocess.run( + ['curl', '-s', '--max-time', timeout] + noproxy + [ + cfg['endpoint'], + '-F', f'file=@{audio_file}', + '-F', f'model={cfg[\"model\"]}', + '-o', output_json + ], capture_output=True, text=True +) + +with open(output_json) as f: + data = json.load(f) +if 'text' not in data: + print(f'ERROR: {json.dumps(data)[:300]}', file=sys.stderr) + sys.exit(1) +text = data['text'] +print(f'Transcribed: {len(text)} chars', file=sys.stderr) +print(text) +os.unlink(output_json) +" > OUTPUT.txt +``` + +**If remote health check fails**, diagnose in order: +1. Network: `ping -c 1 HOST` or `tailscale status | grep HOST` +2. Service: `tailscale ssh USER@HOST "curl -s localhost:PORT/v1/models"` +3. Proxy: retry with `--noproxy '*'` toggled + +## Step 3: Verify Output + +After transcription, check for truncation — the most common failure mode: + +1. Confirm output is not empty +2. Check character count is plausible (~400 chars/min for Chinese, ~200 words/min for English) +3. Check the **ending** — does it trail off mid-sentence? If so, `max_tokens` was exhausted +4. Show user the first and last ~200 characters as preview + +If truncated or wrong, use **AskUserQuestion**: +``` +Transcription may be truncated: +- Expected: ~[N] chars for [M] minutes of audio +- Got: [actual] chars ([pct]% of expected) +- Last line: "[last 100 chars...]" + +Options: +A) Retry with higher max_tokens (current: [N], try: [N*2]) +B) Switch mode — try [local/remote] instead +C) Save as-is — the output looks complete to me +D) Abort +``` + +## Step 4: Fallback — Overlap-Merge (Remote API Only) + +If single remote request fails (timeout, OOM), fall back to chunked transcription: + +```bash +python3 ${CLAUDE_PLUGIN_ROOT}/scripts/overlap_merge_transcribe.py \ + --config "${CLAUDE_PLUGIN_DATA}/config.json" \ + INPUT_AUDIO OUTPUT.txt +``` + +Splits into 18-minute chunks with 2-minute overlap, merges using punctuation-stripped fuzzy matching. See `references/overlap_merge_strategy.md` for algorithm details. + +For local MLX mode, overlap-merge is unnecessary — the bundled script handles chunking internally with `max_tokens=200000`. + +## Step 5: Recommend Transcript Correction + +ASR output always contains recognition errors — homophones, garbled technical terms, broken sentences. After successful transcription, **proactively suggest** running the `transcript-fixer` skill on the output: + +``` +Transcription complete: [N] chars saved to [output_path]. + +ASR output typically contains recognition errors (homophones, garbled terms, broken sentences). +Would you like me to run /transcript-fixer to clean up the text? + +Options: +A) Yes — run transcript-fixer on the output now (Recommended) +B) No — the raw transcription is good enough for my needs +C) Later — I'll run it myself when ready +``` + +If the user chooses A, invoke the `transcript-fixer` skill with the output file path. The two skills form a natural pipeline: **transcribe → correct → review**. + +## Reconfigure + +```bash +rm "${CLAUDE_PLUGIN_DATA}/config.json" +``` + +Then re-run Step 0. + +## Bundled Resources + +**Scripts:** +- `transcribe_local_mlx.py` — Local MLX transcription (macOS ARM64, PEP 723 deps) +- `overlap_merge_transcribe.py` — Chunked transcription with overlap merge (remote API fallback) + +**References:** +- `local_mlx_guide.md` — Performance benchmarks, max_tokens truncation, model compatibility +- `overlap_merge_strategy.md` — Why naive chunking fails, fuzzy merge algorithm diff --git a/asr-transcribe-to-text/references/local_mlx_guide.md b/asr-transcribe-to-text/references/local_mlx_guide.md new file mode 100644 index 00000000..6ee442ac --- /dev/null +++ b/asr-transcribe-to-text/references/local_mlx_guide.md @@ -0,0 +1,58 @@ +# Local MLX Transcription Guide + +## Platform Requirements + +- macOS on Apple Silicon (M1/M2/M3/M4/M5+) +- Python 3.10+ +- `uv` package manager +- ~3GB disk for model weights (first download) + +## Recommended Configuration + +| Setting | Value | Why | +|---------|-------|-----| +| Model | `mlx-community/Qwen3-ASR-1.7B-8bit` | 8-bit quantized, fast inference, good quality | +| max_tokens | `200000` | Default 8192 silently truncates audio >40min | +| Audio format | WAV 16kHz mono PCM | Best compatibility with ASR models | + +## Performance Benchmarks (M5 Pro 48GB, April 2026) + +| Audio Length | Inference Time | Speed | Chars | Tokens | +|-------------|---------------|-------|-------|--------| +| 1 min | 3.7s | 16x realtime | 295 | ~180 | +| 5 min | 11.1s | 27x realtime | 1,633 | ~980 | +| 15 min | 50.5s | 17.8x realtime | 5,074 | ~3,045 | +| 123 min | 502s (8m22s) | 14.7x realtime | 40,347 | 24,337 | +| 96 min | 409s (6m48s) | 14.1x realtime | 30,018 | 18,214 | + +Model load: ~4s (cached), ~130s (first download). + +## Critical: max_tokens Truncation + +The `model.generate()` method in mlx-audio has `max_tokens=8192` as default. This is a **global budget shared across all audio chunks**, not per-chunk. When exhausted, remaining chunks are silently skipped. + +For 123 minutes of Chinese speech: +- Required: ~24,000 tokens +- Default budget: 8,192 tokens +- Result: only first ~40 minutes transcribed, rest silently dropped + +Always pass `max_tokens=200000` for any audio longer than 20 minutes. + +## Model Weight Compatibility + +Two MLX packages exist for Qwen3-ASR. Their weight formats are **incompatible**: + +| Package | Use with | Weight Format | +|---------|----------|--------------| +| `mlx-audio` (Blaizzy) | `mlx-community/Qwen3-ASR-1.7B-8bit` | mlx-audio quantization (audio_tower quantized) | +| `mlx-qwen3-asr` (moona3k) | `Qwen/Qwen3-ASR-1.7B` | Own loader (audio_tower NOT quantized) | + +Crossing these produces "Missing 297 parameters" error. This skill uses `mlx-audio`. + +## Alternatives Not Recommended + +| Approach | Issue | +|----------|-------| +| PyTorch MPS (qwen-asr package) | 97.77% time in GPU↔CPU sync, RTF 5.5-24.5x | +| whisper.cpp large-v3-turbo | High Chinese error rate | +| Official qwen-asr on macOS | Designed for CUDA only | diff --git a/asr-transcribe-to-text/references/overlap_merge_strategy.md b/asr-transcribe-to-text/references/overlap_merge_strategy.md new file mode 100644 index 00000000..e005be0e --- /dev/null +++ b/asr-transcribe-to-text/references/overlap_merge_strategy.md @@ -0,0 +1,56 @@ +# Overlap-Merge Strategy: Why and How + +## The Problem with Naive Chunking + +When ASR transcribes audio in chunks, each chunk's last sentence gets **forcibly truncated**. The model closes the sentence at the chunk boundary even if the speaker is mid-sentence. + +**Real example from testing (5-minute chunks):** + +| Version | Text at boundary | +|---------|-----------------| +| 5min chunk ending | "...靠的就。" (truncated) | +| Continuous 10min | "...靠的就是其中一次战略会。" (complete) | +| Next 5min chunk start | "如果不这么啃，这个业务..." (picks up but gap exists) | + +Concatenating these chunks produces: "靠的就。如果不这么啃..." — losing "是其中一次战略会" entirely. + +## Why Exact String Matching Fails + +The same 2-minute audio segment transcribed in two different contexts (end of chunk A vs start of chunk B) produces **different punctuation**: + +- Chunk A: "可能三年AI的进化" +- Chunk B: "可能。三年AI的进化" + +The words are identical, but punctuation differs because the model's sentence boundary decisions depend on surrounding context. Exact string matching finds zero overlap. + +## The Solution: Punctuation-Stripped Fuzzy Matching + +1. Strip all punctuation from both the tail of chunk A and the head of chunk B +2. Find the longest common substring in the stripped versions +3. Use chunk B's version at the merge point (because chunk A's ending is truncated while chunk B has the complete sentence) + +## Optimal Parameters (Empirically Determined) + +| Parameter | Value | Rationale | +|-----------|-------|-----------| +| Chunk duration | 18 min (1080s) | Safe margin under 20min paper benchmark; 4090 24GB handles much more | +| Overlap duration | 2 min (120s) | ~800 chars overlap region; enough for reliable fuzzy matching | +| Min match length | 15 chars | Filters false positives while catching real overlaps | +| Search window | 600 chars | Covers the overlap region with margin | + +## When to Use This Fallback + +Only use overlap-merge when the single full-length request fails. Reasons it might fail: +- Audio longer than ~2 hours (untested territory, may OOM on 24GB VRAM) +- GPU memory pressure from other processes +- Network timeout (curl max-time exceeded) + +For audio under 1 hour, always try single request first — it's faster, simpler, and produces the best quality. + +## Empirical Comparison (55-minute AI course recording) + +| Strategy | Segments | Boundaries | Total chars | Quality | +|----------|----------|------------|-------------|---------| +| 12x5min direct concat | 12 | 11 cuts | 23,060 | 11 truncated sentences | +| 4x18min overlap merge | 4 | 3 merges | 22,781 | Zero truncation | +| 1x55min single request | 1 | 0 | 22,889 | Perfect (best) | diff --git a/asr-transcribe-to-text/scripts/overlap_merge_transcribe.py b/asr-transcribe-to-text/scripts/overlap_merge_transcribe.py new file mode 100644 index 00000000..b7305756 --- /dev/null +++ b/asr-transcribe-to-text/scripts/overlap_merge_transcribe.py @@ -0,0 +1,200 @@ +# /// script +# requires-python = ">=3.9" +# /// +""" +Overlap-merge transcription for long audio files. + +Splits audio into 18-minute chunks with 2-minute overlap, transcribes each chunk +via a configurable ASR endpoint, then merges using punctuation-stripped fuzzy +matching to eliminate sentence truncation at boundaries. + +Usage: + python3 scripts/overlap_merge_transcribe.py INPUT_AUDIO OUTPUT.txt --config CONFIG.json + python3 scripts/overlap_merge_transcribe.py INPUT_AUDIO OUTPUT.txt --endpoint URL --model MODEL +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile + + +def get_duration(audio_path: str) -> float: + result = subprocess.run( + ["ffprobe", "-v", "error", "-show_entries", "format=duration", + "-of", "default=noprint_wrappers=1:nokey=1", audio_path], + capture_output=True, text=True + ) + return float(result.stdout.strip()) + + +def split_audio(audio_path: str, chunk_dir: str, chunk_duration: int, overlap: int) -> list[tuple[int, int, str]]: + """Split audio into overlapping chunks. Returns list of (start_sec, duration_sec, chunk_path).""" + total = int(get_duration(audio_path)) + chunks = [] + start = 0 + + while start < total: + duration = min(chunk_duration, total - start) + chunk_path = os.path.join(chunk_dir, f"chunk_{len(chunks):02d}.mp3") + + subprocess.run( + ["ffmpeg", "-i", audio_path, "-ss", str(start), "-t", str(duration), + "-acodec", "copy", chunk_path, "-y"], + capture_output=True + ) + chunks.append((start, duration, chunk_path)) + print(f" Chunk {len(chunks)-1}: {start//60}:{start%60:02d} - {(start+duration)//60}:{(start+duration)%60:02d}", file=sys.stderr) + + start += duration - overlap + if start + duration >= total and duration == chunk_duration: + start = total - duration # ensure last chunk covers the end + if start <= chunks[-1][0]: + break + + return chunks + + +def transcribe(audio_path: str, endpoint: str, model: str, noproxy: bool = True) -> str: + """Send audio to ASR endpoint and return text.""" + noproxy_args = ["--noproxy", "*"] if noproxy else [] + result = subprocess.run( + ["curl", "-s", "--max-time", "600"] + noproxy_args + [ + endpoint, + "-F", f"file=@{audio_path}", + "-F", f"model={model}" + ], + capture_output=True, text=True + ) + data = json.loads(result.stdout) + return data["text"] + + +def strip_punct(text: str) -> str: + """Remove all punctuation, keep only CJK chars, letters, and digits.""" + return re.sub(r'[^\w\u4e00-\u9fff]', '', text) + + +def fuzzy_merge(text_a: str, text_b: str, search_chars: int = 600, min_match: int = 15) -> str: + """ + Merge two overlapping transcription segments using punctuation-stripped fuzzy matching. + + The ASR model produces slightly different punctuation for the same audio segment + across different runs, so exact string matching fails. By stripping punctuation + before matching, we find the true overlap region reliably. + + Uses text_b's version at the merge point because text_a truncates its final sentence + while text_b has the complete version. + """ + tail_a_clean = strip_punct(text_a[-search_chars:]) + text_b_clean = strip_punct(text_b) + + best_match_len = 0 + best_b_clean_end = 0 + + # Search for longest matching substring (punctuation-stripped) + for start in range(len(tail_a_clean)): + substr = tail_a_clean[start:start + min_match] + if len(substr) < min_match: + break + pos = text_b_clean.find(substr) + if pos >= 0: + # Extend the match as far as possible + match_len = min_match + while (start + match_len < len(tail_a_clean) + and pos + match_len < len(text_b_clean) + and tail_a_clean[start + match_len] == text_b_clean[pos + match_len]): + match_len += 1 + + if match_len > best_match_len: + best_match_len = match_len + best_b_clean_end = pos + match_len + best_a_clean_start = start + + if best_match_len >= min_match: + # Map clean positions back to raw text positions + # For text_a: find where the match starts in raw text + a_offset = len(text_a) - search_chars + clean_count = 0 + a_cut_pos = len(text_a) + for idx, ch in enumerate(text_a[-search_chars:]): + if strip_punct(ch): + clean_count += 1 + if clean_count > best_a_clean_start: + a_cut_pos = a_offset + idx + break + + # For text_b: find where the match ends in raw text + clean_count = 0 + b_start_pos = 0 + for idx, ch in enumerate(text_b): + if strip_punct(ch): + clean_count += 1 + if clean_count >= best_b_clean_end: + b_start_pos = idx + 1 + break + + print(f" Merged: {best_match_len} chars matched (punct-stripped)", file=sys.stderr) + return text_a[:a_cut_pos] + text_b[b_start_pos:] + else: + print(f" Warning: no overlap found ({best_match_len} chars), concatenating directly", file=sys.stderr) + return text_a + text_b + + +def main(): + parser = argparse.ArgumentParser(description="Overlap-merge ASR transcription") + parser.add_argument("input", help="Input audio/video file") + parser.add_argument("output", help="Output text file") + parser.add_argument("--config", help="Path to config.json (from CLAUDE_PLUGIN_DATA)") + parser.add_argument("--endpoint", default="http://workstation-4090-wsl:8002/v1/audio/transcriptions", help="ASR endpoint URL") + parser.add_argument("--model", default="Qwen/Qwen3-ASR-1.7B", help="Model name") + parser.add_argument("--noproxy", action="store_true", default=True, help="Use --noproxy with curl") + parser.add_argument("--chunk-duration", type=int, default=1080, help="Chunk duration in seconds (default: 1080 = 18min)") + parser.add_argument("--overlap", type=int, default=120, help="Overlap duration in seconds (default: 120 = 2min)") + args = parser.parse_args() + + # Load config from file if provided, otherwise use CLI args + if args.config and os.path.exists(args.config): + with open(args.config) as f: + cfg = json.load(f) + args.endpoint = cfg.get("endpoint", args.endpoint) + args.model = cfg.get("model", args.model) + args.noproxy = cfg.get("noproxy", args.noproxy) + + print(f"Input: {args.input}", file=sys.stderr) + total_duration = get_duration(args.input) + print(f"Duration: {total_duration:.0f}s ({total_duration/60:.1f}min)", file=sys.stderr) + + with tempfile.TemporaryDirectory() as chunk_dir: + # Split + print(f"\nSplitting into {args.chunk_duration}s chunks with {args.overlap}s overlap...", file=sys.stderr) + chunks = split_audio(args.input, chunk_dir, args.chunk_duration, args.overlap) + print(f"Created {len(chunks)} chunks\n", file=sys.stderr) + + # Transcribe each chunk + texts = [] + for i, (start, dur, path) in enumerate(chunks): + print(f"Transcribing chunk {i} ({start//60}:{start%60:02d})...", end=" ", file=sys.stderr, flush=True) + text = transcribe(path, args.endpoint, args.model, args.noproxy) + texts.append(text) + print(f"{len(text)} chars", file=sys.stderr) + + # Merge + print(f"\nMerging {len(texts)} segments...", file=sys.stderr) + merged = texts[0] + for i in range(1, len(texts)): + print(f" Merging chunk {i-1} + {i}:", file=sys.stderr) + merged = fuzzy_merge(merged, texts[i]) + + # Save + with open(args.output, "w", encoding="utf-8") as f: + f.write(merged) + + print(f"\nDone! {len(merged)} chars saved to {args.output}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/asr-transcribe-to-text/scripts/transcribe_local_mlx.py b/asr-transcribe-to-text/scripts/transcribe_local_mlx.py new file mode 100644 index 00000000..1d661e4d --- /dev/null +++ b/asr-transcribe-to-text/scripts/transcribe_local_mlx.py @@ -0,0 +1,80 @@ +# /// script +# requires-python = ">=3.10" +# dependencies = ["mlx-audio>=0.3.1"] +# /// +""" +Local ASR transcription using mlx-audio + Qwen3-ASR on Apple Silicon. + +Usage: + uv run scripts/transcribe_local_mlx.py INPUT_AUDIO [INPUT_AUDIO2 ...] [--output-dir DIR] + +CRITICAL: max_tokens defaults to 200000. The upstream mlx-audio default (8192) +silently truncates audio longer than ~40 minutes. This was discovered empirically: +123 minutes of Chinese speech requires ~24,000 tokens. 8192 only covers the first +~40 minutes before the token budget is exhausted and remaining chunks are skipped. +""" + +import argparse +import os +import platform +import sys +import time + + +def check_platform(): + if sys.platform != "darwin" or platform.machine() not in ("arm64", "aarch64"): + print("ERROR: Local MLX transcription requires macOS on Apple Silicon (M1+).", file=sys.stderr) + print("Use the remote API mode instead.", file=sys.stderr) + sys.exit(1) + + +def main(): + parser = argparse.ArgumentParser(description="Transcribe audio/video using local MLX Qwen3-ASR") + parser.add_argument("inputs", nargs="+", help="Audio/video file paths") + parser.add_argument("--output-dir", default=None, help="Output directory (default: same as input)") + parser.add_argument("--model", default="mlx-community/Qwen3-ASR-1.7B-8bit", + help="HuggingFace model ID (default: mlx-community/Qwen3-ASR-1.7B-8bit)") + parser.add_argument("--max-tokens", type=int, default=200000, + help="Max tokens for generation (default: 200000, covers ~3 hours of speech)") + args = parser.parse_args() + + check_platform() + + from mlx_audio.stt.generate import load_model + + print(f"Loading model {args.model}...", file=sys.stderr, flush=True) + t0 = time.time() + model = load_model(args.model) + load_time = time.time() - t0 + print(f"Model loaded in {load_time:.1f}s", file=sys.stderr, flush=True) + + for audio_path in args.inputs: + if not os.path.exists(audio_path): + print(f"SKIP: {audio_path} not found", file=sys.stderr) + continue + + name = os.path.splitext(os.path.basename(audio_path))[0] + out_dir = args.output_dir or os.path.dirname(audio_path) or "." + output_path = os.path.join(out_dir, f"{name}.txt") + + print(f"\nTranscribing: {os.path.basename(audio_path)}", file=sys.stderr, flush=True) + t1 = time.time() + + result = model.generate(audio_path, max_tokens=args.max_tokens, verbose=True) + + elapsed = time.time() - t1 + text = result.text if hasattr(result, "text") else str(result) + gen_tokens = result.generation_tokens if hasattr(result, "generation_tokens") else "N/A" + + with open(output_path, "w", encoding="utf-8") as f: + f.write(text) + + print(f"Done: {elapsed:.1f}s, {len(text)} chars, {gen_tokens} tokens → {output_path}", + file=sys.stderr, flush=True) + + total = time.time() - t0 + print(f"\nAll done. Total: {total:.1f}s", file=sys.stderr, flush=True) + + +if __name__ == "__main__": + main() diff --git a/bilibili-content/SKILL.md b/bilibili-content/SKILL.md new file mode 100644 index 00000000..dc42d337 --- /dev/null +++ b/bilibili-content/SKILL.md @@ -0,0 +1,142 @@ +--- +name: bilibili-content +description: "Extract Bilibili video subtitles (AI-generated → uploader-uploaded → local Whisper ASR fallback) and top comments. Output as study notes, summaries, threads, or blog posts. Use when the user shares a Bilibili link, wants a video summarized, or needs transcripts. 提取B站视频字幕（AI生成→投稿者上传→Whisper ASR兜底）和热门评论，输出学习笔记、摘要、博客等格式。用户发B站链接时使用。" +--- + +# Bilibili Content Tool · B站内容提取工具 + +提取 B站视频字幕和热门评论，输出为摘要、学习笔记、博客等格式。 + +Extract Bilibili video subtitles and top comments. Output as study notes, summaries, threads, or blog posts. + +## 功能 / Features + +- **三策略字幕获取** — AI 自动生成 (覆盖率 ~90%) → 投稿者上传 → 本地 Whisper ASR 兜底 +- **热评提取** — 按赞数排序，包括 UP主置顶和观众高频讨论 +- **多 P 支持** — 自动识别 URL 中的 `p=` 参数 +- **多格式输出** — 教程笔记、章节摘要、博客、时间线 + +## 适用场景 / When to Use + +用户发 B站链接、要总结视频、做学习笔记、提取字幕全文时使用。支持 `bilibili.com/video/`、`b23.tv` 短链、BV 号、AV 号。 + +## 安装 / Setup + +```bash +# 核心依赖 +pip install bilibili-api-python httpx + +# 可选：本地 ASR 兜底 +pip install faster-whisper +# macOS: brew install ffmpeg +``` + +### B站 AI 字幕（需 Cookie） + +```bash +# 1. 浏览器登录 B站 → F12 → Application → Cookies → 复制 SESSDATA +# 2. 设置环境变量 +export BILIBILI_SESSDATA="你的值" +``` + +不设 Cookie 时只能抓投稿者字幕（极少视频有），加 `--asr-fallback` 可启用本地 Whisper 兜底。 + +## 使用方法 / Usage + +`SKILL_DIR` 为本 SKILL.md 所在目录。 + +```bash +# 完整提取（字幕 + 评论，JSON） +python3 SKILL_DIR/scripts/fetch_content.py "https://www.bilibili.com/video/BV1xx411c7m2" + +# 只取字幕纯文本 +python3 SKILL_DIR/scripts/fetch_content.py "BV1xx411c7m2" --subtitle-only + +# 带时间戳 +python3 SKILL_DIR/scripts/fetch_content.py "URL" --subtitle-only --timestamps + +# 跳过评论（最快） +python3 SKILL_DIR/scripts/fetch_content.py "URL" --no-comments + +# 指定分 P +python3 SKILL_DIR/scripts/fetch_content.py "URL" --page 17 # 0-based, P18 + +# ASR 兜底 +python3 SKILL_DIR/scripts/fetch_content.py "URL" --asr-fallback +python3 SKILL_DIR/scripts/fetch_content.py "URL" --asr-fallback --asr-model small +``` + +## 输出格式 / Output Formats + +根据视频类型和用户需求选择： + +- **学习笔记**（教程类）— 分层知识点、代码块、⚠️ 注意事项、💡 小技巧 +- **章节摘要** — 按内容变化分章 +- **全文摘要** — 5-10 句概括 +- **博客文章** — 标题、分段、核心要点 +- **时间线** — 推文风格，每条 ≤280 字 +- **评论精华** — UP主信息 + 高赞观点补充 + +教程/课程类默认输出 **学习笔记**，其他类型默认 **全文摘要**。 + +## 字幕策略 / Subtitle Strategy + +``` +1. player/wbi/v2 → AI 自动生成 (ai-zh) 覆盖率: ~90% 需: Cookie +2. get_subtitle() → 投稿者上传 (zh) 覆盖率: <5% 需: 无 +3. Whisper ASR → 本地转写 覆盖率: 100% 需: ffmpeg + faster-whisper +``` + +策略 3 仅在 `--asr-fallback` 时触发。 + +## JSON 结构 + +```json +{ + "video_id": "BV1xx411c7m2", + "title": "视频标题", + "duration": "12:34", + "current_page": 1, + "subtitle": { + "source": "ai", + "language": "ai-zh", + "segment_count": 519, + "full_text": "完整字幕...", + "timestamped_text": "0:00 文本\n0:05 ..." + }, + "comments": { + "total_ac": 1234, + "top_comments": [ + {"user": "UP主", "content": "...", "likes": 233} + ] + } +} +``` + +## 工作流 / Workflow + +1. **抓取** — 运行 `fetch_content.py` +2. **验证** — 确认字幕非空。无字幕时注明 +3. **分块** — 超 ~50K 字时分块处理 +4. **转换** — 按视频类型格式化输出 +5. **复查** — 检查时间戳和连贯性 + +## 错误处理 + +| 情况 | 处理 | +|------|------| +| 无字幕 | 告知用户，评论仍可用 | +| 无 Cookie | 建议设置 `BILIBILI_SESSDATA` 或加 `--asr-fallback` | +| 依赖缺失 | `pip install bilibili-api-python httpx` | +| 私有/已删除 | 转达 API 错误 | + +## 文件结构 + +``` +bilibili-content/ +├── SKILL.md +├── scripts/ +│ └── fetch_content.py +└── references/ + └── output-formats.md +``` diff --git a/bilibili-content/references/output-formats.md b/bilibili-content/references/output-formats.md new file mode 100644 index 00000000..91ce2b52 --- /dev/null +++ b/bilibili-content/references/output-formats.md @@ -0,0 +1,78 @@ +# Bilibili Content — Output Format Reference + +## 默认行为 + +**教程/课程类视频（默认）→ 学习笔记**，其他类型视频 → 摘要。 + +判断标准：标题含"教程/入门/课程/学习/讲解"等关键词，或内容有明显教学结构（知识点分层、案例演示、课后练习）。 + +## 0. 学习笔记 (Study Notes) — 教程视频默认格式 + +完整结构化笔记，保留所有知识点。层次： + +``` +# P{集数} {标题} — 学习笔记 + +> 课程信息行 + +## 一、知识定位 +上一讲学了什么 → 本讲在知识体系中的位置 → 与前后讲的关系 +用树形图展示章节结构 + +## 二、核心知识点详解 +按讲师讲解顺序展开，每个知识点： + - 含义 + 示例代码（```python 代码块） + - ⚠️ 关键注意（容易踩的坑） + - > 💡 小技巧（实用建议） + +## 三、完整演示推导 +讲师上机演示的逐步推演，展示变量值变化过程 + +## 四、记忆技巧 +表格、口诀、规律总结 + +## 五、课后任务 +讲师布置的练习题 +``` + +**原则**： +- 保留讲师的语言风格和讲解逻辑，不要过度精简 +- 代码示例必须可运行，变量名与原视频保持一致 +- 标注所有坑点和注意事项，这是用户最需要的 +- 不混入弹幕和评论分析，除非用户明确要求 +- 不要结尾问问题或催促动手 + +## 1. 章节 (Chapters) + +仅用于非教程类视频，或用户明确要求章节格式时。 + +## 2. 摘要 (Summary) + +仅用于非教程类视频，或用户要求简短概括时。5-10 句。 + +## 3. 博客 (Blog Post) + +完整文章，适合非教程类深度内容。含视频信息、正文、弹幕热度、高赞评论、总结。 + +## 4. 时间线 (Thread) + +每条 ≤280 字符，适合发社交平台。 + +## 5. 弹幕分析 (Danmaku Analysis) + +仅用户明确要求时输出。格式： + +``` +📊 弹幕总数: N +🔥 热点时间段: ... +💬 高频弹幕: ... +``` + +## 6. 评论精华 (Comment Highlights) + +仅用户明确要求时输出。格式： + +``` +👍 热评: + - @用户A (+N): "..." +``` diff --git a/bilibili-content/scripts/fetch_content.py b/bilibili-content/scripts/fetch_content.py new file mode 100644 index 00000000..ed59236e --- /dev/null +++ b/bilibili-content/scripts/fetch_content.py @@ -0,0 +1,454 @@ +#!/usr/bin/env python3 +""" +Fetch Bilibili video subtitles and top comments as structured JSON. + +字幕获取支持三策略： +1. AI 自动生成 (ai-zh) — player/wbi/v2，需 Cookie +2. 投稿者上传 (zh) — get_subtitle()，无需 Cookie +3. 本地 Whisper ASR — --asr-fallback，需 ffmpeg + faster-whisper + +Usage: + python fetch_content.py [--language zh-CN] [--no-comments] + python fetch_content.py --asr-fallback [--asr-model base] + +Output (JSON): + { + "video_id": "BV1xx411c7m2", + "title": "...", + "description": "...", + "duration": "12:34", + "subtitle": { + "source": "ai|uploader|whisper", + "language": "ai-zh", + "segments": [...], + "full_text": "...", + "timestamped_text": "00:00 ..." + }, + "comments": { + "total_ac": 1234, + "top_comments": [...] + } + } + +Install dependency: pip install bilibili-api-python httpx +""" + +import argparse +import asyncio +import json +import re +import sys +from typing import Optional + + +def extract_bvid(url_or_id: str) -> str: + """从 B站各种 URL 格式中提取 BV 号。""" + url_or_id = url_or_id.strip() + if re.match(r'^BV[a-zA-Z0-9]{10}$', url_or_id): + return url_or_id + match = re.search(r'(BV[a-zA-Z0-9]{10})', url_or_id) + if match: + return match.group(1) + match = re.search(r'av(\d+)', url_or_id, re.IGNORECASE) + if match: + return match.group(0) + return url_or_id + + +def format_timestamp(seconds: float) -> str: + """秒数 → HH:MM:SS 或 MM:SS""" + total = int(seconds) + h, remainder = divmod(total, 3600) + m, s = divmod(remainder, 60) + if h > 0: + return f"{h}:{m:02d}:{s:02d}" + return f"{m}:{s:02d}" + + +# ═══════════════════════════════════════════════ +# 字幕获取 +# ═══════════════════════════════════════════════ + +async def fetch_subtitle(v, cid: int, preferred_lang: Optional[str] = None) -> Optional[dict]: + """获取视频字幕（AI自动生成 + 投稿者上传）。 + + 策略 1: player/wbi/v2 → AI 字幕 (ai-zh)，覆盖率 ~90%，需 Cookie + 策略 2: get_subtitle() → 投稿者字幕 (zh)，覆盖率 <5%，无需 Cookie + """ + subtitle_url = None + language = "unknown" + language_display = "" + + # 策略 1: player/wbi/v2（含 AI 字幕，优先） + try: + player_info = await v.get_player_info(cid=cid) + sub_data = player_info.get("subtitle", {}) or {} + subtitles = sub_data.get("subtitles", []) + if subtitles: + tiers = [] + if preferred_lang: + tiers.append(preferred_lang) + tiers.extend(["ai-zh", "zh", "ai-en", "en"]) + chosen = None + for tier in tiers: + for sub in subtitles: + lan = sub.get("lan", "") + if lan == tier or lan.startswith(tier): + chosen = sub + break + if chosen: + break + if not chosen: + chosen = subtitles[0] + subtitle_url = chosen.get("subtitle_url", "") + language = chosen.get("lan", "unknown") + language_display = chosen.get("lan_doc", "") + except Exception: + pass + + # 策略 2: get_subtitle()（投稿者字幕 fallback） + if not subtitle_url: + try: + subtitle_info = await v.get_subtitle(cid) + subtitles = subtitle_info.get("subtitles", []) + if subtitles: + chosen = subtitles[0] + if preferred_lang: + for sub in subtitles: + if sub.get("lan", "").startswith(preferred_lang): + chosen = sub + break + subtitle_url = chosen.get("subtitle_url", "") + language = chosen.get("lan", "unknown") + language_display = chosen.get("lan_doc", "") + except Exception: + pass + + if not subtitle_url: + return None + + if subtitle_url.startswith("//"): + subtitle_url = "https:" + subtitle_url + + raw = None + try: + import httpx + async with httpx.AsyncClient(timeout=30) as client: + resp = await client.get(subtitle_url) + resp.raise_for_status() + raw = resp.json() + except ImportError: + import urllib.request + import ssl + ctx = ssl.create_default_context() + req = urllib.request.Request(subtitle_url, headers={"User-Agent": "Mozilla/5.0"}) + with urllib.request.urlopen(req, context=ctx, timeout=30) as resp: + raw = json.loads(resp.read().decode("utf-8")) + except Exception: + return None + + if not raw: + return None + + body = raw.get("body", []) + if not body: + return None + + segments = [] + for item in body: + segments.append({ + "text": item.get("content", ""), + "start": float(item.get("from", 0)), + "duration": float(item.get("to", 0)) - float(item.get("from", 0)), + }) + + full_text = " ".join(seg["text"] for seg in segments) + timestamped = "\n".join( + f"{format_timestamp(seg['start'])} {seg['text']}" for seg in segments + ) + + return { + "language": language, + "language_display": language_display, + "source": "ai" if language.startswith("ai-") else "uploader", + "segment_count": len(segments), + "segments": segments, + "full_text": full_text, + "timestamped_text": timestamped, + } + + +async def fetch_subtitle_asr(v, cid: int, page_index: int, credential, + model_name: str = "base", + preferred_lang: Optional[str] = None) -> Optional[dict]: + """策略 3: 下载音频流 → 本地 Whisper 转文字。 + + 当策略 1 和 2 都失败时使用。需要 ffmpeg + faster-whisper。 + """ + import os + import tempfile + import subprocess + import shutil + + # 1. 获取音频流 URL + audio_url = None + try: + url_info = await v.get_download_url(page_index=page_index, cid=cid) + dash = url_info.get("dash", {}) if isinstance(url_info, dict) else {} + audios = dash.get("audio", []) + if audios: + audio_url = audios[0].get("base_url") or audios[0].get("baseUrl", "") + except Exception: + pass + + if not audio_url: + return None + + # 2. 下载音频 + tmpdir = tempfile.mkdtemp(prefix="bilibili_asr_") + audio_path = os.path.join(tmpdir, "audio.m4a") + wav_path = os.path.join(tmpdir, "audio.wav") + + try: + import httpx + headers = {"User-Agent": "Mozilla/5.0", "Referer": "https://www.bilibili.com/"} + async with httpx.AsyncClient(timeout=300, follow_redirects=True) as client: + resp = await client.get(audio_url, headers=headers) + resp.raise_for_status() + with open(audio_path, "wb") as f: + f.write(resp.content) + except ImportError: + import urllib.request + req = urllib.request.Request(audio_url, + headers={"User-Agent": "Mozilla/5.0", "Referer": "https://www.bilibili.com/"}) + with urllib.request.urlopen(req, timeout=300) as resp: + with open(audio_path, "wb") as f: + f.write(resp.read()) + except Exception: + shutil.rmtree(tmpdir, ignore_errors=True) + return None + + # 3. ffmpeg 转 WAV (16kHz mono) + if shutil.which("ffmpeg"): + try: + subprocess.run( + ["ffmpeg", "-y", "-i", audio_path, "-ar", "16000", "-ac", "1", + "-f", "wav", wav_path], + capture_output=True, timeout=120 + ) + transcribe_path = wav_path if (os.path.exists(wav_path) and + os.path.getsize(wav_path) > 0) else audio_path + except Exception: + transcribe_path = audio_path + else: + transcribe_path = audio_path + + # 4. Whisper + try: + from faster_whisper import WhisperModel + + device = "cpu" + compute_type = "int8" + if hasattr(subprocess, "check_output"): + try: + if subprocess.check_output(["uname", "-m"], text=True).strip() == "arm64": + compute_type = "auto" + except Exception: + pass + + model = WhisperModel(model_name, device=device, compute_type=compute_type) + lang = preferred_lang if preferred_lang else None + segments_raw, info = model.transcribe(transcribe_path, language=lang, beam_size=5) + + segments = [] + for seg in segments_raw: + segments.append({ + "text": seg.text.strip(), + "start": seg.start, + "duration": seg.end - seg.start, + }) + + if not segments: + shutil.rmtree(tmpdir, ignore_errors=True) + return None + + full_text = " ".join(seg["text"] for seg in segments) + timestamped = "\n".join( + f"{format_timestamp(seg['start'])} {seg['text']}" for seg in segments + ) + + shutil.rmtree(tmpdir, ignore_errors=True) + return { + "language": info.language, + "language_display": f"ASR ({model_name})", + "source": "whisper", + "model": model_name, + "segment_count": len(segments), + "segments": segments, + "full_text": full_text, + "timestamped_text": timestamped, + } + + except ImportError: + shutil.rmtree(tmpdir, ignore_errors=True) + return None + except Exception: + shutil.rmtree(tmpdir, ignore_errors=True) + return None + + +# ═══════════════════════════════════════════════ +# 评论获取 +# ═══════════════════════════════════════════════ + +async def fetch_comments(oid: int) -> Optional[dict]: + """获取按赞数排序的评论（UP主置顶 + 热门讨论）。""" + try: + from bilibili_api.comment import get_comments_lazy, OrderType, CommentResourceType + + result = await get_comments_lazy( + oid=oid, + type_=CommentResourceType.VIDEO, + order=OrderType.LIKE, + ) + except Exception: + return None + + replies = result.get("replies", []) + top_comments = [] + for reply in replies[:20]: + top_comments.append({ + "user": reply.get("member", {}).get("uname", "未知"), + "content": reply.get("content", {}).get("message", ""), + "likes": reply.get("like", 0), + "timestamp": reply.get("ctime", 0), + }) + + return { + "top_comments": top_comments, + "total_ac": result.get("page", {}).get("ac_count", 0), + } + + +# ═══════════════════════════════════════════════ +# 主流程 +# ═══════════════════════════════════════════════ + +async def main_async(args): + import os + from bilibili_api import video, Credential + + bvid = extract_bvid(args.url) + + # Cookie（AI 字幕需要） + credential = None + sessdata = os.environ.get("BILIBILI_SESSDATA", "") + if sessdata: + credential = Credential(sessdata=sessdata) + + v = video.Video(bvid=bvid, credential=credential) + + # 基础信息 + info = await v.get_info() + pages = await v.get_pages() + + duration_sec = info.get("duration", 0) + if pages: + duration_sec = sum(p.get("duration", 0) for p in pages) + + result = { + "video_id": bvid, + "title": info.get("title", ""), + "description": info.get("desc", ""), + "duration": format_timestamp(duration_sec), + "duration_seconds": duration_sec, + "pages": len(pages) if pages else 1, + "url": f"https://www.bilibili.com/video/{bvid}", + } + + # 分 P + page_index = args.page if hasattr(args, 'page') and args.page is not None else 0 + cid = await v.get_cid(page_index=page_index) + result["current_page"] = page_index + 1 + + # 字幕（策略 1 + 2 + 3） + subtitle = await fetch_subtitle(v, cid, args.language) + if subtitle: + result["subtitle"] = subtitle + elif getattr(args, 'asr_fallback', False): + model_name = getattr(args, 'asr_model', 'base') + sys.stderr.write(f"[ASR] No subtitle found, falling back to whisper ({model_name})...\n") + sys.stderr.flush() + subtitle = await fetch_subtitle_asr(v, cid, page_index, credential, + model_name, args.language) + if subtitle: + sys.stderr.write(f"[ASR] Done — {subtitle['segment_count']} segments\n") + sys.stderr.flush() + result["subtitle"] = subtitle + else: + result["subtitle"] = None + else: + result["subtitle"] = None + + # 评论 + if not args.no_comments: + aid = info.get("aid", 0) + comments = await fetch_comments(aid) + result["comments"] = comments if comments else None + else: + result["comments"] = None + + return result + + +def main(): + parser = argparse.ArgumentParser( + description="Fetch Bilibili video subtitles and comments as JSON" + ) + parser.add_argument("url", help="Bilibili URL or BV/AV ID") + parser.add_argument("--language", "-l", default=None, + help="Preferred subtitle language (e.g. zh-CN, en)") + parser.add_argument("--no-comments", action="store_true", + help="Skip comment fetching") + parser.add_argument("--subtitle-only", action="store_true", + help="Output subtitle plain text only") + parser.add_argument("--timestamps", "-t", action="store_true", + help="Include timestamps in --subtitle-only output") + parser.add_argument("--page", "-p", type=int, default=None, + help="Page index for multi-P videos (0-based)") + parser.add_argument("--asr-fallback", action="store_true", + help="Use local Whisper ASR when no subtitle available") + parser.add_argument("--asr-model", default="base", + help="Whisper model: tiny, base, small, medium, large") + args = parser.parse_args() + + # URL 中 p= 参数自动提取 + if args.page is None: + match = re.search(r'[?&]p=(\d+)', args.url) + if match: + args.page = int(match.group(1)) - 1 + + try: + result = asyncio.run(main_async(args)) + except ImportError as e: + print(json.dumps({ + "error": f"Missing dependency: {e}. Run: pip install bilibili-api-python httpx" + }, ensure_ascii=False)) + sys.exit(1) + except Exception as e: + print(json.dumps({"error": str(e)}, ensure_ascii=False)) + sys.exit(1) + + if args.subtitle_only: + sub = result.get("subtitle") + if not sub: + print("(no subtitle available)") + sys.exit(1) + print(sub["timestamped_text"] if args.timestamps else sub["full_text"]) + return + + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/capture-screen/SKILL.md b/capture-screen/SKILL.md index ab88d264..8cf693ba 100644 --- a/capture-screen/SKILL.md +++ b/capture-screen/SKILL.md @@ -227,6 +227,65 @@ These methods were tested and confirmed to fail on macOS: | Python `import Quartz` (PyObjC) | `ModuleNotFoundError` | PyObjC not installed in system Python; don't attempt to install it — use Swift instead | | `osascript` window id | Wrong format | Returns AppleScript window index, not CGWindowID needed by `screencapture -l` | +## Permission Troubleshooting + +`swift scripts/get_window_id.swift` reads on-screen windows via CoreGraphics, so it needs Screen Recording permission on macOS. + +Use this order: + +1. Confirm trigger +2. Confirm target identity +3. Add/enable exact app in Settings + +If the command fails with `ERROR: Failed to enumerate windows`, do this: + +```bash +open "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture" +``` + +Or print the same checklist directly from the script: + +```bash +swift scripts/get_window_id.swift --permission-hint screen +swift scripts/get_window_id.swift --permission-hint microphone +``` + +Then: + +1. In Privacy & Security → Screen Recording, enable the target app. +2. If your app is missing from the list: + - Ensure you granted permission to the real app bundle (not `swift` / terminal helpers). + - For CLI tools, build/run as a packaged `.app` during permission verification. + - Click `+` and add the `.app` manually from `/Applications`. +3. Re-run the command after restarting the app. +4. If this is a CLI workflow, also check whether the launcher is a helper binary: + - In most cases the entry shown in TCC is the helper process (`swift`, `Terminal`, `iTerm`, etc.), not the business app. + - Permission still works after helper-level grant, but it is not ideal for final UX. + +For mic-access-related prompts, use the same pattern with the microphone pane: + +```bash +open "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone" +``` + +The same rule still applies: the system can only show permissions for a concrete `.app` bundle. If the request is made by a helper binary, the settings list can be misleading or empty for your product app. + +### Quick Check Template + +```text +1) Error: permission denied +2) Open target pane +3) Verify identity shown by OS = identity you granted +4) If not matched, use the script-reported candidate identities and grant the launcher process +5) Reopen/restart and verify +``` + +For production apps, avoid requesting permissions via `swift`/`python` entry points; always route permission checks in the packaged app process so users only see one target. + +If you maintain another macOS permission-related flow, reuse this standardized triage template: + +- [permission-triage-template.md](references/permission-triage-template.md) + ## Supported Applications | Application | Window ID | AppleScript Control | Notes | diff --git a/capture-screen/references/permission-triage-template.md b/capture-screen/references/permission-triage-template.md new file mode 100644 index 00000000..16c3919a --- /dev/null +++ b/capture-screen/references/permission-triage-template.md @@ -0,0 +1,42 @@ +# macOS 权限排障模板（Screen Recording / 麦克风） + +## 排障目标 +- 在系统设置里找不到目标应用 +- 权限拒绝但设置项看起来已打开 +- 通过终端/脚本入口触发时，用户不知道该给谁授权 + +## 标准排查顺序（必须按序执行） + +1. 确认触发点 + - 明确是哪个权限被拒绝（Screen Recording / 麦克风）。 +2. 确认 TCC 实体 + - 不是脚本文件名。 + - 先确认“当前触发进程”与“最终应用体”是否一致。 + - 关注脚本输出里的候选身份列表（invoker/runtime）并逐项核验。 +3. 确认设置面板 + - 直接跳转到对应隐私面板 + - 允许该进程/应用 + - 重启进程后复验 + +## 通用动作模板 + +```bash +# Screen Recording +open "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture" + +# Microphone +open "x-apple.systempreferences:com.apple.preference.security?Privacy_Microphone" +``` + +## 不在列表时处理 + +- 优先确认请求来自真实 .app Bundle（签名、打包） +- 如果当前为 CLI/脚本入口，先给宿主进程授权（Terminal/iTerm/swift/python） +- 在设置面板点击 `+` 手工添加目标 `.app` +- 变更后退出并重启应用，重新测试 + +## 验收标准（用户侧） + +- 用户能看到一条明确的“应授权对象” +- 错误提示中有“找不到对象时下一步该做什么” +- 无需反复猜测在设置里要点击什么 diff --git a/capture-screen/scripts/get_window_id.swift b/capture-screen/scripts/get_window_id.swift index c19ef7fd..8ccef145 100755 --- a/capture-screen/scripts/get_window_id.swift +++ b/capture-screen/scripts/get_window_id.swift @@ -4,9 +4,11 @@ // Enumerate on-screen windows and print their Window IDs. // // Usage: -// swift get_window_id.swift # List all windows -// swift get_window_id.swift Excel # Filter by keyword -// swift get_window_id.swift "Chrome" # Filter by app name +// swift scripts/get_window_id.swift # List all windows +// swift scripts/get_window_id.swift Excel # Filter by keyword +// swift scripts/get_window_id.swift "Chrome" # Filter by app name +// swift scripts/get_window_id.swift --permission-hint screen # Print Screen Recording triage +// swift scripts/get_window_id.swift --permission-hint microphone # Print Microphone triage // // Output format: // WID=12345 | App=Microsoft Excel | Title=workbook.xlsx @@ -15,10 +17,192 @@ // import CoreGraphics +import Foundation -let keyword = CommandLine.arguments.count > 1 - ? CommandLine.arguments[1] - : "" +enum PermissionKind: String { + case screen + case microphone +} + +let invocationPath = (CommandLine.arguments.first ?? "") +let invokerName = URL(fileURLWithPath: invocationPath).lastPathComponent +let runtimeProcessName = ProcessInfo.processInfo.processName +let invokerIsBundle = invocationPath.hasSuffix(".app") || invocationPath.contains(".app/") +let scriptPath: String? = invocationPath.hasSuffix(".swift") ? invocationPath : nil +let helperBinaries: Set = [ + "swift", + "swift-frontend", + "python", + "python3", + "node", + "uv", + "npm", + "bun", + "pnpm", + "yarn", + "bash", + "zsh", + "sh", + "osascript", + "Terminal", + "iTerm2", + "iTerm" +] + +let invokerCandidates: [String] = { + var candidates = [String]() + var seen = Set() + func append(_ value: String) { + guard !value.isEmpty, !seen.contains(value) else { return } + seen.insert(value) + candidates.append(value) + } + if let scriptPath = scriptPath { + append(scriptPath) + } + if !invokerName.isEmpty { + append(invokerName) + } + if !runtimeProcessName.isEmpty && runtimeProcessName != invokerName { + append(runtimeProcessName) + } + return candidates +}() + +let args = Array(CommandLine.arguments.dropFirst()) + +var permissionHintTarget: PermissionKind? +var keyword = "" +var expectPermissionTarget = false + +func printUsage() { + fputs("Usage:\n", stderr) + fputs(" swift scripts/get_window_id.swift [keyword]\n", stderr) + fputs(" swift scripts/get_window_id.swift --permission-hint [screen|microphone]\n", stderr) + fputs("\n", stderr) + fputs("Options:\n", stderr) + fputs(" --permission-hint [screen|microphone] Print permission triage instructions\n", stderr) + fputs(" -h, --help Show this help\n", stderr) + fputs("\n", stderr) + fputs("Examples:\n", stderr) + fputs(" swift scripts/get_window_id.swift Excel\n", stderr) + fputs(" swift scripts/get_window_id.swift --permission-hint screen\n", stderr) + fputs(" swift scripts/get_window_id.swift --permission-hint microphone\n", stderr) +} + +for arg in args { + if expectPermissionTarget { + if let kind = PermissionKind(rawValue: arg.lowercased()) { + permissionHintTarget = kind + expectPermissionTarget = false + continue + } + fputs("Unknown permission target: \(arg)\n", stderr) + printUsage() + exit(2) + } + + if arg == "-h" || arg == "--help" { + printUsage() + exit(0) + } else if arg == "--permission-hint" { + permissionHintTarget = .screen + expectPermissionTarget = true + } else if arg.hasPrefix("--permission-hint=") { + let target = String(arg.dropFirst("--permission-hint=".count)).lowercased() + guard let kind = PermissionKind(rawValue: target) else { + fputs("Unknown permission target: \(target)\n", stderr) + printUsage() + exit(2) + } + permissionHintTarget = kind + } else if arg == "--permission-hint-screen" { + permissionHintTarget = .screen + } else if arg == "--permission-hint-microphone" || arg == "--permission-hint-mic" { + permissionHintTarget = .microphone + } else if arg.hasPrefix("-") { + fputs("Unknown option: \(arg)\n", stderr) + printUsage() + exit(2) + } else if keyword.isEmpty { + keyword = arg + } +} + +if expectPermissionTarget { + fputs("Missing permission hint target after --permission-hint.\n", stderr) + printUsage() + exit(2) +} + +if let kind = permissionHintTarget { + switch kind { + case .screen: + fputs("Screen Recording permission required.\n", stderr) + printCommonPermissionHint( + pane: "Privacy_ScreenCapture", + label: "Screen Recording" + ) + printPermissionContextHint() + case .microphone: + fputs("Microphone permission required.\n", stderr) + printCommonPermissionHint( + pane: "Privacy_Microphone", + label: "Microphone" + ) + printPermissionContextHint() + } + exit(0) +} + +func printCommonPermissionHint(pane: String, label: String, missing: Bool = true) { + let openCommand = "x-apple.systempreferences:com.apple.preference.security?\(pane)" + fputs("Troubleshooting:\n", stderr) + fputs(" - Open Settings: `open \"\(openCommand)\"`\n", stderr) + fputs(" - In Privacy & Security → \(label), enable the target application.\n", stderr) + if missing { + fputs(" - If the target app is not in the list:\n", stderr) + fputs(" - Granting happens by real .app bundle, not by helper/terminal scripts.\n", stderr) + fputs(" - For CLI workflows, grant to the host app you launch from (Terminal, iTerm, iTerm2, Swift, etc.) if no dedicated .app exists yet.\n", stderr) + fputs(" - Click `+` and add the actual `.app` from `/Applications`.\n", stderr) + } + fputs(" - If permission status does not refresh, quit/reopen terminal/app and retry.\n", stderr) + if helperBinaries.contains(invokerName) || helperBinaries.contains(runtimeProcessName) { + fputs(" - Current launcher is a helper/runtime process (`\(runtimeProcessName)`) -> OS may show this entry instead of the tool name.\n", stderr) + } else if invokerIsBundle { + fputs(" - The launcher path looks like a bundled app, which is the preferred state for permissions.\n", stderr) + } + if let scriptPath = scriptPath { + fputs(" - Script entry: `\(scriptPath)`\n", stderr) + } +} + +func printPermissionContextHint() { + fputs(" - Invoker path: `\(invocationPath)`\n", stderr) + fputs(" - Runtime process: `\(runtimeProcessName)`\n", stderr) + if !invokerCandidates.isEmpty { + fputs(" - Candidate identities in System Settings:\n", stderr) + for identity in invokerCandidates { + fputs(" - \(identity)\n", stderr) + } + } + if invokerIsBundle { + fputs(" This looks like a bundled app path, so the setting should match the app identity.\n", stderr) + } else { + fputs(" If this is not your final app binary, permissions can be inconsistent.\n", stderr) + } + fputs(" - Recommended for production: keep permission requests inside your signed `.app` process.\n", stderr) + if let scriptPath = scriptPath { + fputs(" - Script entry currently used: `\(scriptPath)`.\n", stderr) + } +} + +func printScreenRecordingPermissionHint() { + fputs("Screen Recording permission required.\n", stderr) + fputs("Troubleshooting:\n", stderr) + printCommonPermissionHint(pane: "Privacy_ScreenCapture", label: "Screen Recording") + printPermissionContextHint() +} guard let windowList = CGWindowListCopyWindowInfo( .optionOnScreenOnly, kCGNullWindowID @@ -27,6 +211,7 @@ guard let windowList = CGWindowListCopyWindowInfo( fputs("Possible causes:\n", stderr) fputs(" - No applications with visible windows are running\n", stderr) fputs(" - Screen Recording permission not granted (System Settings → Privacy & Security → Screen Recording)\n", stderr) + printScreenRecordingPermissionHint() exit(1) } diff --git a/cli-demo-generator/SKILL.md b/cli-demo-generator/SKILL.md index 929085c5..21ba1aff 100644 --- a/cli-demo-generator/SKILL.md +++ b/cli-demo-generator/SKILL.md @@ -1,346 +1,233 @@ --- name: cli-demo-generator -description: This skill should be used when users want to create animated CLI demos, terminal recordings, or command-line demonstration GIFs. It supports both manual tape file creation and automated demo generation from command descriptions. Use when users mention creating demos, recording terminal sessions, or generating animated GIFs of CLI workflows. +description: Generates professional animated CLI demos as GIFs using VHS terminal recordings. Handles tape file creation, self-bootstrapping demos with hidden setup, output noise filtering, post-processing speed-up, and frame-level verification. Use when users want to create terminal demos, record CLI workflows as GIFs, generate animated documentation, build demo tapes for README files, or need to showcase any command-line tool visually. Also triggers on "record terminal", "VHS tape", "demo GIF", "animate my CLI", or any request to visually demonstrate shell commands. --- # CLI Demo Generator -Generate professional animated CLI demos with ease. This skill supports both automated generation from command descriptions and manual control for custom demos. +Create professional animated CLI demos. Four approaches, from fully automated to pixel-precise manual control. -## When to Use This Skill +## Quick Start -Trigger this skill when users request: -- "Create a demo showing how to install my package" -- "Generate a CLI demo of these commands" -- "Make an animated GIF of my terminal workflow" -- "Record a terminal session and convert to GIF" -- "Batch generate demos from this config" -- "Create an interactive typing demo" +**Simplest path** — give commands, get GIF: -## Core Capabilities - -### 1. Automated Demo Generation (Recommended) - -Use the `auto_generate_demo.py` script for quick, automated demo creation. This is the easiest and most common approach. - -**Basic Usage:** ```bash -scripts/auto_generate_demo.py \ +python3 ${CLAUDE_SKILL_DIR}/scripts/auto_generate_demo.py \ -c "npm install my-package" \ -c "npm run build" \ -o demo.gif ``` -**With Options:** +**Self-bootstrapping demo** — for repeatable recordings that clean their own state: + ```bash -scripts/auto_generate_demo.py \ - -c "command1" \ - -c "command2" \ - -o output.gif \ - --title "Installation Demo" \ - --theme "Dracula" \ - --width 1400 \ - --height 700 +python3 ${CLAUDE_SKILL_DIR}/scripts/auto_generate_demo.py \ + -c "npm install my-package" \ + -c "npm run build" \ + -o demo.gif \ + --bootstrap "npm uninstall my-package 2>/dev/null" \ + --speed 2 ``` -**Script Parameters:** -- `-c, --command`: Command to include (can be specified multiple times) -- `-o, --output`: Output GIF file path (required) -- `--title`: Demo title (optional, shown at start) -- `--theme`: VHS theme (default: Dracula) -- `--font-size`: Font size (default: 16) -- `--width`: Terminal width (default: 1400) -- `--height`: Terminal height (default: 700) -- `--no-execute`: Generate tape file only, don't execute VHS - -**Smart Features:** -- Automatic timing based on command complexity -- Optimized sleep durations (1-3s depending on operation) -- Proper spacing between commands -- Professional defaults - -### 2. Batch Demo Generation +## Critical: VHS Parser Limitations -Use `batch_generate.py` for creating multiple demos from a configuration file. +VHS `Type` strings cannot contain `$`, `\"`, or backticks. These cause parse errors: -**Configuration File (YAML):** -```yaml -demos: - - name: "Install Demo" - output: "install.gif" - title: "Installation" - theme: "Dracula" - commands: - - "npm install my-package" - - "npm run build" - - - name: "Usage Demo" - output: "usage.gif" - commands: - - "my-package --help" - - "my-package run" +```tape +# FAILS — VHS parser rejects special chars +Type "echo \"hello $USER\"" +Type "claude() { command claude \"$@\"; }" ``` -**Usage:** +**Workaround: base64 encode the command**, decode at runtime: + ```bash -scripts/batch_generate.py config.yaml --output-dir ./demos +# 1. Encode your complex command +echo 'claude() { command claude "$@" 2>&1 | grep -v "noise"; }' | base64 +# Output: Y2xhdWRlKCkgey4uLn0K + +# 2. Use in tape +Type "echo Y2xhdWRlKCkgey4uLn0K | base64 -d > /tmp/wrapper.sh && source /tmp/wrapper.sh" ``` -**When to Use Batch Generation:** -- Creating a suite of related demos -- Documenting multiple features -- Generating demos for tutorials or documentation -- Maintaining consistent demo series +This pattern is essential for output filtering, function definitions, and any command with shell special characters. -### 3. Interactive Recording +## Approaches -Use `record_interactive.sh` for recording live terminal sessions. +### 1. Automated Generation (Recommended) -**Usage:** ```bash -scripts/record_interactive.sh output.gif \ - --theme "Dracula" \ - --width 1400 +python3 ${CLAUDE_SKILL_DIR}/scripts/auto_generate_demo.py \ + -c "command1" -c "command2" \ + -o output.gif \ + --title "My Demo" \ + --theme "Catppuccin Latte" \ + --font-size 24 \ + --width 1400 --height 600 ``` -**Recording Process:** -1. Script starts asciinema recording -2. Type commands naturally in your terminal -3. Press Ctrl+D when finished -4. Script auto-converts to GIF via VHS - -**When to Use Interactive Recording:** -- Demonstrating complex workflows -- Showing real command output -- Capturing live interactions -- Recording debugging sessions - -### 4. Manual Tape File Creation - -For maximum control, create VHS tape files manually using templates. - -**Available Templates:** -- `assets/templates/basic.tape` - Simple command demo -- `assets/templates/interactive.tape` - Typing simulation +| Flag | Default | Description | +|------|---------|-------------| +| `-c` | required | Command to include (repeatable) | +| `-o` | required | Output GIF path | +| `--title` | none | Title shown at start | +| `--theme` | Dracula | VHS theme name | +| `--font-size` | 16 | Font size in pt | +| `--width` | 1400 | Terminal width px | +| `--height` | 700 | Terminal height px | +| `--bootstrap` | none | Hidden setup command (repeatable) | +| `--filter` | none | Regex pattern to filter from output | +| `--speed` | 1 | Playback speed multiplier (uses gifsicle) | +| `--no-execute` | false | Generate .tape only | -**Example Workflow:** -1. Copy template: `cp assets/templates/basic.tape my-demo.tape` -2. Edit commands and timing -3. Generate GIF: `vhs < my-demo.tape` +Smart timing: `install`/`build`/`test`/`deploy` → 3s, `ls`/`pwd`/`echo` → 1s, others → 2s. -Consult `references/vhs_syntax.md` for complete VHS syntax reference. +### 2. Batch Generation -## Workflow Guidance +Create multiple demos from one config: -### For Simple Demos (1-3 commands) - -Use automated generation for quick results: +```yaml +# demos.yaml +demos: + - name: "Install" + output: "install.gif" + commands: ["npm install my-package"] + - name: "Usage" + output: "usage.gif" + commands: ["my-package --help", "my-package run"] +``` ```bash -scripts/auto_generate_demo.py \ - -c "echo 'Hello World'" \ - -c "ls -la" \ - -o hello-demo.gif \ - --title "Hello Demo" +python3 ${CLAUDE_SKILL_DIR}/scripts/batch_generate.py demos.yaml --output-dir ./gifs ``` -### For Multiple Related Demos - -Create a batch configuration file and use batch generation: - -1. Create `demos-config.yaml` with all demo definitions -2. Run: `scripts/batch_generate.py demos-config.yaml --output-dir ./output` -3. All demos generate automatically with consistent settings - -### For Interactive/Complex Workflows +### 3. Interactive Recording -Use interactive recording to capture real behavior: +Record a live terminal session: ```bash -scripts/record_interactive.sh my-workflow.gif -# Type commands naturally -# Ctrl+D when done +bash ${CLAUDE_SKILL_DIR}/scripts/record_interactive.sh output.gif --theme "Catppuccin Latte" +# Type commands naturally, Ctrl+D when done ``` -### For Custom Timing/Layout +Requires asciinema (`brew install asciinema`). -Create manual tape file with precise control: +### 4. Manual Tape File -1. Start with template or generate base tape with `--no-execute` -2. Edit timing, add comments, customize layout -3. Generate: `vhs < custom-demo.tape` +For maximum control, write a tape directly. Templates in `assets/templates/`: -## Best Practices +- `basic.tape` — simple command sequence +- `interactive.tape` — typing simulation +- `self-bootstrap.tape` — **self-cleaning demo with hidden setup** (recommended for repeatable demos) -Refer to `references/best_practices.md` for comprehensive guidelines. Key recommendations: +## Advanced Patterns -**Timing:** -- Quick commands (ls, pwd): 1s sleep -- Standard commands (grep, cat): 2s sleep -- Heavy operations (install, build): 3s+ sleep +These patterns come from production use. See `references/advanced_patterns.md` for full details. -**Sizing:** -- Standard: 1400x700 (recommended) -- Compact: 1200x600 -- Presentations: 1800x900 +### Self-Bootstrapping Demos -**Themes:** -- Documentation: Nord, GitHub Dark -- Code demos: Dracula, Monokai -- Presentations: High-contrast themes +Demos that clean previous state, set up environment, and hide all of it from the viewer: -**Duration:** -- Target: 15-30 seconds -- Maximum: 60 seconds -- Create series for complex topics +```tape +Hide +Type "cleanup-previous-state 2>/dev/null" +Enter +Sleep 2s +Type "clear" +Enter +Sleep 500ms +Show + +Type "the-command-users-see" +Enter +Sleep 3s +``` -## Troubleshooting +The `Hide` → commands → `clear` → `Show` sequence is critical. `clear` wipes the terminal buffer so hidden commands don't leak into the GIF. -### VHS Not Installed +### Output Noise Filtering -```bash -# macOS -brew install vhs +Filter noisy progress lines from commands that produce verbose output: -# Linux (via Go) -go install github.com/charmbracelet/vhs@latest +```tape +# Hidden: create a wrapper function that filters noise +Hide +Type "echo | base64 -d > /tmp/w.sh && source /tmp/w.sh" +Enter +Sleep 500ms +Type "clear" +Enter +Sleep 500ms +Show + +# Visible: clean command, filtered output +Type "my-noisy-command" +Enter +Sleep 3s ``` -### Asciinema Not Installed +### Frame Verification + +After recording, verify GIF content by extracting key frames: ```bash -# macOS -brew install asciinema +# Extract frames at specific positions +ffmpeg -i demo.gif -vf "select=eq(n\,100)" -frames:v 1 /tmp/frame.png -y 2>/dev/null -# Linux -sudo apt install asciinema +# View the frame (Claude can read images) +# Use Read tool on /tmp/frame.png to verify content ``` -### Demo File Too Large - -**Solutions:** -1. Reduce duration (shorter sleep times) -2. Use smaller dimensions (1200x600) -3. Consider MP4 format: `Output demo.mp4` -4. Split into multiple shorter demos - -### Output Not Readable +### Post-Processing Speed-Up -**Solutions:** -1. Increase font size: `--font-size 18` -2. Use wider terminal: `--width 1600` -3. Choose high-contrast theme: `--theme "Dracula"` -4. Test on target display device - -## Examples - -### Example 1: Quick Install Demo - -User request: "Create a demo showing npm install" +Use gifsicle to speed up recordings without re-recording: ```bash -scripts/auto_generate_demo.py \ - -c "npm install my-package" \ - -o install-demo.gif \ - --title "Package Installation" -``` - -### Example 2: Multi-Step Tutorial - -User request: "Create a demo showing project setup with git clone, install, and run" +# 2x speed (halve frame delay) +gifsicle -d2 original.gif "#0-" > fast.gif -```bash -scripts/auto_generate_demo.py \ - -c "git clone https://github.com/user/repo.git" \ - -c "cd repo" \ - -c "npm install" \ - -c "npm start" \ - -o setup-demo.gif \ - --title "Project Setup" \ - --theme "Nord" +# 1.5x speed +gifsicle -d4 original.gif "#0-" > faster.gif ``` -### Example 3: Batch Generation +### Template Placeholder Pattern -User request: "Generate demos for all my CLI tool features" +Keep tape files generic with placeholders, replace at build time: -1. Create `features-demos.yaml`: -```yaml -demos: - - name: "Help Command" - output: "help.gif" - commands: ["my-tool --help"] - - - name: "Init Command" - output: "init.gif" - commands: ["my-tool init", "ls -la"] - - - name: "Run Command" - output: "run.gif" - commands: ["my-tool run --verbose"] -``` +```tape +# In tape file +Type "claude plugin marketplace add MARKETPLACE_REPO" -2. Generate all: -```bash -scripts/batch_generate.py features-demos.yaml --output-dir ./demos +# In build script +sed "s|MARKETPLACE_REPO|$DETECTED_REPO|g" template.tape > rendered.tape +vhs rendered.tape ``` -### Example 4: Interactive Session - -User request: "Record me using my CLI tool interactively" +## Timing & Sizing Reference -```bash -scripts/record_interactive.sh my-session.gif --theme "Tokyo Night" -# User types commands naturally -# Ctrl+D to finish -``` +| Context | Width | Height | Font | Duration | +|---------|-------|--------|------|----------| +| README/docs | 1400 | 600 | 16-20 | 10-20s | +| Presentation | 1800 | 900 | 24 | 15-30s | +| Compact embed | 1200 | 600 | 14-16 | 10-15s | +| Wide output | 1600 | 800 | 16 | 15-30s | -## Bundled Resources +See `references/best_practices.md` for detailed guidelines. -### scripts/ -- **`auto_generate_demo.py`** - Automated demo generation from command lists -- **`batch_generate.py`** - Generate multiple demos from YAML/JSON config -- **`record_interactive.sh`** - Record and convert interactive terminal sessions - -### references/ -- **`vhs_syntax.md`** - Complete VHS tape file syntax reference -- **`best_practices.md`** - Demo creation guidelines and best practices +## Troubleshooting -### assets/ -- **`templates/basic.tape`** - Basic command demo template -- **`templates/interactive.tape`** - Interactive typing demo template -- **`examples/batch-config.yaml`** - Example batch configuration file +| Problem | Solution | +|---------|----------| +| VHS not installed | `brew install charmbracelet/tap/vhs` | +| gifsicle not installed | `brew install gifsicle` | +| GIF too large | Reduce dimensions, sleep times, or use `--speed 2` | +| Text wraps/breaks | Increase `--width` or decrease `--font-size` | +| VHS parse error on `$` or `\"` | Use base64 encoding (see Critical section above) | +| Hidden commands leak into GIF | Add `clear` + `Sleep 500ms` before `Show` | +| Commands execute before previous finishes | Increase `Sleep` duration | ## Dependencies -**Required:** -- VHS (https://github.com/charmbracelet/vhs) - -**Optional:** -- asciinema (for interactive recording) -- PyYAML (for batch YAML configs): `pip install pyyaml` - -## Output Formats - -VHS supports multiple output formats: - -```tape -Output demo.gif # GIF (default, best for documentation) -Output demo.mp4 # MP4 (better compression for long demos) -Output demo.webm # WebM (smaller file size) -``` - -Choose based on use case: -- **GIF**: Documentation, README files, easy embedding -- **MP4**: Longer demos, better quality, smaller size -- **WebM**: Web-optimized, smallest file size - -## Summary - -This skill provides three main approaches: - -1. **Automated** (`auto_generate_demo.py`) - Quick, easy, smart defaults -2. **Batch** (`batch_generate.py`) - Multiple demos, consistent settings -3. **Interactive** (`record_interactive.sh`) - Live recording, real output +**Required:** VHS (`brew install charmbracelet/tap/vhs`) -Choose the approach that best fits the user's needs. For most cases, automated generation is the fastest and most convenient option. +**Optional:** gifsicle (speed-up), asciinema (interactive recording), ffmpeg (frame verification), PyYAML (batch YAML configs) diff --git a/cli-demo-generator/assets/templates/self-bootstrap.tape b/cli-demo-generator/assets/templates/self-bootstrap.tape new file mode 100644 index 00000000..993f3240 --- /dev/null +++ b/cli-demo-generator/assets/templates/self-bootstrap.tape @@ -0,0 +1,51 @@ +# Self-bootstrapping demo template +# Cleans previous state, sets up environment, records clean demo +# +# MARKETPLACE_REPO — replaced by recording script via sed +# BASE64_WRAPPER — base64-encoded output filter function +# To create: echo 'my_func() { command my_func "$@" 2>&1 | grep -v "noise"; }' | base64 + +Output demo.gif +Set Theme "Catppuccin Latte" +Set FontSize 24 +Set Width 1400 +Set Height 600 +Set Padding 20 +Set TypingSpeed 10ms +Set Shell zsh + +# Hidden bootstrap: cleanup + output filter + clear screen +Hide +Type "cleanup-command-here 2>/dev/null" +Enter +Sleep 3s +# Base64-encoded wrapper filters noisy output lines. +# VHS cannot parse shell special chars ($, \") in Type strings, so base64 is the workaround. +Type "echo BASE64_WRAPPER | base64 -d > /tmp/cw.sh && source /tmp/cw.sh" +Enter +Sleep 500ms +Type "clear" +Enter +Sleep 500ms +Show + +# Stage 1: Setup +Type "setup-command MARKETPLACE_REPO" +Enter +Sleep 8s +Enter +Sleep 300ms + +# Stage 2: Main action +Type "main-command" +Enter +Sleep 3s +Enter +Sleep 300ms + +# Stage 3: Verify +Type "verify-command" +Enter +Sleep 2s + +Sleep 1s diff --git a/cli-demo-generator/references/advanced_patterns.md b/cli-demo-generator/references/advanced_patterns.md new file mode 100644 index 00000000..c4d7383f --- /dev/null +++ b/cli-demo-generator/references/advanced_patterns.md @@ -0,0 +1,240 @@ +# Advanced VHS Demo Patterns + +Battle-tested patterns from production demo recording workflows. + +## Contents +- Self-bootstrapping tapes +- Output noise filtering with base64 wrapper +- Frame-level verification +- Post-processing with gifsicle +- Auto-detection and template rendering +- Recording script structure + +## Self-Bootstrapping Tapes + +A self-bootstrapping tape cleans its own state before recording, so running it twice produces identical output. Three phases: + +```tape +# Phase 1: HIDDEN CLEANUP — remove previous state +Hide +Type "my-tool uninstall 2>/dev/null; my-tool reset 2>/dev/null" +Enter +Sleep 3s + +# Phase 2: HIDDEN SETUP — create helpers (base64 for special chars) +Type "echo | base64 -d > /tmp/helper.sh && source /tmp/helper.sh" +Enter +Sleep 500ms + +# Phase 3: CLEAR + SHOW — wipe buffer before revealing +Type "clear" +Enter +Sleep 500ms +Show + +# Phase 4: VISIBLE DEMO — what the viewer sees +Type "my-tool install" +Enter +Sleep 3s +``` + +**Why `clear` before `Show`:** VHS's `Hide` stops recording frames, but the terminal buffer still accumulates text. Without `clear`, the hidden commands' text appears in the first visible frame. + +## Output Noise Filtering with Base64 + +Many CLI tools produce verbose progress output that clutters demos. The solution: a hidden shell wrapper that filters noise lines. + +### Step 1: Create the wrapper function + +```bash +# The function you want (can't type directly in VHS due to $/" chars) +my_tool() { command my_tool "$@" 2>&1 | grep -v -E "cache|progress|downloading|timeout"; } +``` + +### Step 2: Base64 encode it + +```bash +echo 'my_tool() { command my_tool "$@" 2>&1 | grep -v -E "cache|progress|downloading|timeout"; }' | base64 +# Output: bXlfdG9vbCgpIHsgY29tbWFuZC4uLn0K +``` + +### Step 3: Use in tape + +```tape +Hide +Type "echo bXlfdG9vbCgpIHsgY29tbWFuZC4uLn0K | base64 -d > /tmp/w.sh && source /tmp/w.sh" +Enter +Sleep 500ms +Type "clear" +Enter +Sleep 500ms +Show + +# Now `my_tool` calls the wrapper — clean output +Type "my_tool deploy" +Enter +Sleep 5s +``` + +### When to filter + +- Git operations: filter "Cloning", "Refreshing", cache messages +- Package managers: filter download progress, cache hits +- Build tools: filter intermediate compilation steps +- Any command with `SSH not configured`, `timeout: 120s`, etc. + +## Frame-Level Verification + +After recording, extract and inspect key frames to verify the GIF shows what you expect. + +### Extract specific frames + +```bash +# Frame at position N (0-indexed) +ffmpeg -i demo.gif -vf "select=eq(n\,100)" -frames:v 1 /tmp/frame_100.png -y 2>/dev/null + +# Multiple frames at once +for n in 50 200 400; do + ffmpeg -i demo.gif -vf "select=eq(n\,$n)" -frames:v 1 "/tmp/frame_$n.png" -y 2>/dev/null +done +``` + +### Check total frame count and duration + +```bash +ffmpeg -i demo.gif 2>&1 | grep -E "Duration|fps" +# Duration: 00:00:10.50, ... 25 fps → 262 frames total +``` + +### What to verify + +| Frame | Check | +|-------|-------| +| First (~frame 5) | No leaked hidden commands | +| Mid (~frame N/2) | Key output visible, no noise | +| Final (~frame N-10) | All commands completed, result shown | + +### Claude can read frames + +Use the Read tool on extracted PNG files — Claude's vision can verify text content in terminal screenshots. + +## Post-Processing with gifsicle + +Speed up or optimize GIFs after recording, avoiding re-recording. + +### Speed control + +```bash +# 2x speed — halve frame delay (most common) +gifsicle -d2 input.gif "#0-" > output.gif + +# 1.5x speed +gifsicle -d4 input.gif "#0-" > output.gif + +# 3x speed +gifsicle -d1 input.gif "#0-" > output.gif +``` + +### Optimize file size + +```bash +# Lossless optimization +gifsicle -O3 input.gif > optimized.gif + +# Reduce colors (lossy but smaller) +gifsicle --colors 128 input.gif > smaller.gif +``` + +### Typical recording script pattern + +```bash +# Record at normal speed +vhs demo.tape + +# Speed up 2x for final output +cp demo.gif /tmp/demo_raw.gif +gifsicle -d2 /tmp/demo_raw.gif "#0-" > demo.gif +rm /tmp/demo_raw.gif +``` + +## Auto-Detection and Template Rendering + +For demos that need to adapt to the environment (e.g., different repo URLs, detected tools). + +### Template placeholders + +Use `sed` to replace placeholders before recording: + +```tape +# demo.tape (template) +Type "tool marketplace add REPO_PLACEHOLDER" +Enter +``` + +```bash +# Build script detects the correct repo +REPO=$(detect_repo) +sed "s|REPO_PLACEHOLDER|$REPO|g" demo.tape > /tmp/rendered.tape +vhs /tmp/rendered.tape +``` + +### Auto-detect pattern (shell function) + +```bash +detect_repo() { + local upstream origin + upstream=$(git remote get-url upstream 2>/dev/null | sed 's|.*github.com[:/]||; s|\.git$||') || true + origin=$(git remote get-url origin 2>/dev/null | sed 's|.*github.com[:/]||; s|\.git$||') || true + + # Check upstream first (canonical), then origin (fork) + if [[ -n "$upstream" ]] && gh api "repos/$upstream/contents/.target-file" &>/dev/null; then + echo "$upstream" + elif [[ -n "$origin" ]]; then + echo "$origin" + else + echo "fallback/default" + fi +} +``` + +## Recording Script Structure + +A complete recording script follows this pattern: + +```bash +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" + +# 1. Check prerequisites +for cmd in vhs gifsicle; do + command -v "$cmd" &>/dev/null || { echo "Missing: $cmd"; exit 1; } +done + +# 2. Auto-detect dynamic values +REPO=$(detect_repo) +echo "Using repo: $REPO" + +# 3. Render tape template +sed "s|PLACEHOLDER|$REPO|g" "$SCRIPT_DIR/demo.tape" > /tmp/rendered.tape + +# 4. Clean previous state +cleanup_state || true + +# 5. Record +(cd "$REPO_DIR" && vhs /tmp/rendered.tape) + +# 6. Speed up +cp "$REPO_DIR/demo.gif" /tmp/raw.gif +gifsicle -d2 /tmp/raw.gif "#0-" > "$REPO_DIR/demo.gif" + +# 7. Clean up +cleanup_state || true +rm -f /tmp/raw.gif /tmp/rendered.tape + +# 8. Report +SIZE=$(ls -lh "$REPO_DIR/demo.gif" | awk '{print $5}') +echo "Done: demo.gif ($SIZE)" +``` diff --git a/cli-demo-generator/scripts/auto_generate_demo.py b/cli-demo-generator/scripts/auto_generate_demo.py index 455fff97..7e0da3df 100755 --- a/cli-demo-generator/scripts/auto_generate_demo.py +++ b/cli-demo-generator/scripts/auto_generate_demo.py @@ -2,10 +2,14 @@ """ Auto-generate CLI demos from command descriptions. -This script creates VHS tape files and generates GIF demos automatically. +Creates VHS tape files and generates GIF demos with support for: +- Hidden bootstrap commands (self-cleaning state) +- Output noise filtering via base64-encoded wrapper +- Post-processing speed-up via gifsicle """ import argparse +import base64 import subprocess import sys from pathlib import Path @@ -21,21 +25,54 @@ def create_tape_file( width: int = 1400, height: int = 700, padding: int = 20, + bootstrap: Optional[List[str]] = None, + filter_pattern: Optional[str] = None, ) -> str: """Generate a VHS tape file from commands.""" tape_lines = [ f'Output {output_gif}', - '', + f'Set Theme "{theme}"', f'Set FontSize {font_size}', f'Set Width {width}', f'Set Height {height}', - f'Set Theme "{theme}"', f'Set Padding {padding}', + 'Set TypingSpeed 10ms', + 'Set Shell zsh', '', ] - # Add title if provided + # Hidden bootstrap: cleanup + optional output filter + has_hidden = bootstrap or filter_pattern + if has_hidden: + tape_lines.append('Hide') + + if bootstrap: + # Combine all bootstrap commands with semicolons + combined = "; ".join( + cmd if "2>/dev/null" in cmd else f"{cmd} 2>/dev/null" + for cmd in bootstrap + ) + tape_lines.append(f'Type "{combined}"') + tape_lines.append('Enter') + tape_lines.append('Sleep 3s') + + if filter_pattern: + # Create a wrapper function that filters noisy output + wrapper = f'_wrap() {{ "$@" 2>&1 | grep -v -E "{filter_pattern}"; }}' + encoded = base64.b64encode(wrapper.encode()).decode() + tape_lines.append(f'Type "echo {encoded} | base64 -d > /tmp/cw.sh && source /tmp/cw.sh"') + tape_lines.append('Enter') + tape_lines.append('Sleep 500ms') + + # Clear screen before Show to prevent hidden text from leaking + tape_lines.append('Type "clear"') + tape_lines.append('Enter') + tape_lines.append('Sleep 500ms') + tape_lines.append('Show') + tape_lines.append('') + + # Title if title: tape_lines.extend([ f'Type "# {title}" Sleep 500ms Enter', @@ -43,68 +80,94 @@ def create_tape_file( '', ]) - # Add commands with smart timing + # Commands with smart timing for i, cmd in enumerate(commands, 1): - # Type the command - tape_lines.append(f'Type "{cmd}" Sleep 500ms') + # If filter is active, prefix with _wrap + if filter_pattern: + tape_lines.append(f'Type "_wrap {cmd}"') + else: + tape_lines.append(f'Type "{cmd}"') tape_lines.append('Enter') # Smart sleep based on command complexity - if any(keyword in cmd.lower() for keyword in ['install', 'build', 'test', 'deploy']): + if any(kw in cmd.lower() for kw in ['install', 'build', 'test', 'deploy', 'marketplace']): sleep_time = '3s' - elif any(keyword in cmd.lower() for keyword in ['ls', 'pwd', 'echo', 'cat']): + elif any(kw in cmd.lower() for kw in ['ls', 'pwd', 'echo', 'cat', 'grep']): sleep_time = '1s' else: sleep_time = '2s' tape_lines.append(f'Sleep {sleep_time}') - # Add spacing between commands + # Empty line between stages for readability if i < len(commands): + tape_lines.append('Enter') + tape_lines.append('Sleep 300ms') tape_lines.append('') + tape_lines.append('') + tape_lines.append('Sleep 1s') return '\n'.join(tape_lines) +def speed_up_gif(gif_path: str, speed: int) -> bool: + """Speed up GIF using gifsicle. Returns True on success.""" + try: + subprocess.run(['gifsicle', '--version'], capture_output=True, check=True) + except (subprocess.CalledProcessError, FileNotFoundError): + print("⚠ gifsicle not found, skipping speed-up. Install: brew install gifsicle", file=sys.stderr) + return False + + # delay = 10 / speed (10 = normal, 5 = 2x, 3 = ~3x) + delay = max(1, 10 // speed) + tmp = f"/tmp/demo_raw_{Path(gif_path).stem}.gif" + + subprocess.run(['cp', gif_path, tmp], check=True) + with open(gif_path, 'wb') as out: + subprocess.run(['gifsicle', f'-d{delay}', tmp, '#0-'], stdout=out, check=True) + Path(tmp).unlink(missing_ok=True) + return True + + def main(): parser = argparse.ArgumentParser( description='Auto-generate CLI demos from commands', formatter_class=argparse.RawDescriptionHelpFormatter, epilog=''' Examples: - # Generate demo from single command + # Simple demo %(prog)s -c "npm install" -o demo.gif - # Generate demo with multiple commands - %(prog)s -c "git clone repo" -c "cd repo" -c "npm install" -o setup.gif + # With hidden bootstrap (self-cleaning) + %(prog)s -c "my-tool run" -o demo.gif \\ + --bootstrap "my-tool reset" --speed 2 - # Custom theme and size - %(prog)s -c "ls -la" -o demo.gif --theme Monokai --width 1200 - - # With title - %(prog)s -c "echo Hello" -o demo.gif --title "My Demo" + # With output noise filtering + %(prog)s -c "deploy-tool push" -o demo.gif \\ + --filter "cache|progress|downloading" ''' ) parser.add_argument('-c', '--command', action='append', required=True, - help='Command to include in demo (can be specified multiple times)') + help='Command to include (repeatable)') parser.add_argument('-o', '--output', required=True, help='Output GIF file path') - parser.add_argument('--title', help='Demo title (optional)') - parser.add_argument('--theme', default='Dracula', - help='VHS theme (default: Dracula)') - parser.add_argument('--font-size', type=int, default=16, - help='Font size (default: 16)') - parser.add_argument('--width', type=int, default=1400, - help='Terminal width (default: 1400)') - parser.add_argument('--height', type=int, default=700, - help='Terminal height (default: 700)') + parser.add_argument('--title', help='Demo title') + parser.add_argument('--theme', default='Dracula', help='VHS theme (default: Dracula)') + parser.add_argument('--font-size', type=int, default=16, help='Font size (default: 16)') + parser.add_argument('--width', type=int, default=1400, help='Terminal width (default: 1400)') + parser.add_argument('--height', type=int, default=700, help='Terminal height (default: 700)') + parser.add_argument('--bootstrap', action='append', + help='Hidden setup command run before demo (repeatable)') + parser.add_argument('--filter', + help='Regex pattern to filter from command output') + parser.add_argument('--speed', type=int, default=1, + help='Playback speed multiplier (default: 1, uses gifsicle)') parser.add_argument('--no-execute', action='store_true', - help='Generate tape file only, do not execute VHS') + help='Generate tape file only') args = parser.parse_args() - # Generate tape file content tape_content = create_tape_file( commands=args.command, output_gif=args.output, @@ -113,37 +176,38 @@ def main(): font_size=args.font_size, width=args.width, height=args.height, + bootstrap=args.bootstrap, + filter_pattern=args.filter, ) - # Write tape file output_path = Path(args.output) tape_file = output_path.with_suffix('.tape') - - with open(tape_file, 'w') as f: - f.write(tape_content) - + tape_file.write_text(tape_content) print(f"✓ Generated tape file: {tape_file}") if not args.no_execute: - # Check if VHS is installed try: subprocess.run(['vhs', '--version'], capture_output=True, check=True) except (subprocess.CalledProcessError, FileNotFoundError): - print("✗ VHS is not installed!", file=sys.stderr) - print("Install it with: brew install vhs", file=sys.stderr) - print(f"✓ You can manually run: vhs < {tape_file}", file=sys.stderr) + print("✗ VHS not installed. Install: brew install charmbracelet/tap/vhs", file=sys.stderr) + print(f"✓ Run manually: vhs {tape_file}", file=sys.stderr) return 1 - # Execute VHS - print(f"Generating GIF: {args.output}") + print(f"Recording: {args.output}") try: subprocess.run(['vhs', str(tape_file)], check=True) - print(f"✓ Demo generated: {args.output}") - print(f" Size: {output_path.stat().st_size / 1024:.1f} KB") except subprocess.CalledProcessError as e: - print(f"✗ VHS execution failed: {e}", file=sys.stderr) + print(f"✗ VHS failed: {e}", file=sys.stderr) return 1 + # Post-processing speed-up + if args.speed > 1: + print(f"Speeding up {args.speed}x...") + speed_up_gif(args.output, args.speed) + + size_kb = output_path.stat().st_size / 1024 + print(f"✓ Done: {args.output} ({size_kb:.0f} KB)") + return 0 diff --git a/cloud/lessons-learned.md b/cloud/lessons-learned.md new file mode 100644 index 00000000..629e5198 --- /dev/null +++ b/cloud/lessons-learned.md @@ -0,0 +1,68 @@ +# WeChat Article Scraper - 经验教训 + +## 核心原则：用户旅程优先于功能堆砌 + +### 问题 +在 Ralph Loop 迭代中，我犯了**功能堆砌**的错误： +- 添加了 AI摘要、Daily Review、高亮批注、TTS朗读、沉浸式阅读、键盘快捷键、阅读进度追踪... +- 但没有验证核心用户旅程是否通畅 + +### 核心发现 +**微信文章抓取的核心流程是断的！** + +问题链： +1. 扩展调用 `/api/scrape` 让服务器抓取微信文章 +2. 服务器没有微信登录态 → 抓取失败 +3. `/api/articles/import` 端点不存在 → 扩展无法直接上传 +4. 用户无法真正保存微信文章 + +### 修复方案 +1. 创建 `POST /api/articles/import` 端点，接收扩展直接上传的内容 +2. 修改扩展 `saveWeChatArticle` 函数： + - 旧：调用 `/api/scrape` 让服务器抓取（失败） + - 新：提取页面内容 → 调用 `/api/articles/import` 上传（成功） + +### 关键教训 + +#### 1. 先验证核心旅程，再加功能 +**核心用户旅程**（必须100%可用）： +``` +发现微信文章 → 点击扩展 → 提取内容 → 上传保存 → Web阅读 +``` + +**锦上添花功能**（只有核心旅程可用后才有意义）： +- AI摘要、TTS朗读、阅读进度、键盘快捷键... + +#### 2. 架构设计要理解业务场景 +微信文章抓取的特殊性： +- 需要登录态（cookies） +- 反爬严格 +- 必须在已登录的浏览器环境中提取 + +**错误设计**：服务器端抓取 +**正确设计**：扩展提取 + 直接上传 + +#### 3. 测试验证比代码更重要 +写100个功能不如验证1个核心流程。 + +### 决策原则 + +1. **功能添加前**：这是否让核心用户旅程更顺畅？ +2. **代码提交前**：我测试过这个功能真的能用吗？ +3. **迭代方向**：解决问题 > 添加功能 + +### 当前状态 + +**已修复**： +- ✅ 创建 `/api/articles/import` 端点 +- ✅ 修复扩展 `saveWeChatArticle` 函数 +- ✅ 核心抓取流程已打通 + +**待验证**： +- 扩展提取微信文章内容的准确性 +- 图片、视频等特殊内容的处理 +- 批量导入的稳定性 + +--- +记录时间：2026-04-13 +记录原因：Ralph Loop 反思 - 功能堆砌 vs 用户旅程 diff --git a/competitors-analysis/SKILL.md b/competitors-analysis/SKILL.md index bde9f417..ff4bb74b 100644 --- a/competitors-analysis/SKILL.md +++ b/competitors-analysis/SKILL.md @@ -3,7 +3,7 @@ name: competitors-analysis description: Analyze competitor repositories with evidence-based approach. Use when tracking competitors, creating competitor profiles, or generating competitive analysis. CRITICAL - all analysis must be based on actual cloned code, never assumptions. Triggers include "analyze competitor", "add competitor", "competitive analysis", or "竞品分析". context: fork agent: general-purpose -argument-hint: [product-name] [competitor-url] +argument-hint: "[product-name] [competitor-url]" --- # Competitors Analysis diff --git a/claude-code-history-files-finder/.INTEGRATION_SUMMARY.md b/daymade-claude-code/claude-code-history-files-finder/.INTEGRATION_SUMMARY.md similarity index 100% rename from claude-code-history-files-finder/.INTEGRATION_SUMMARY.md rename to daymade-claude-code/claude-code-history-files-finder/.INTEGRATION_SUMMARY.md diff --git a/claude-code-history-files-finder/.security-scan-passed b/daymade-claude-code/claude-code-history-files-finder/.security-scan-passed similarity index 100% rename from claude-code-history-files-finder/.security-scan-passed rename to daymade-claude-code/claude-code-history-files-finder/.security-scan-passed diff --git a/claude-code-history-files-finder/SKILL.md b/daymade-claude-code/claude-code-history-files-finder/SKILL.md similarity index 88% rename from claude-code-history-files-finder/SKILL.md rename to daymade-claude-code/claude-code-history-files-finder/SKILL.md index 3a0f69c1..5cca5b6d 100644 --- a/claude-code-history-files-finder/SKILL.md +++ b/daymade-claude-code/claude-code-history-files-finder/SKILL.md @@ -58,7 +58,7 @@ Extract files from session history: python3 scripts/recover_content.py /path/to/session.jsonl ``` -Extracts all Write tool calls and saves files to `./recovered_content/`. +Extracts all Write tool calls and saves files to `./recovered_content/`, preserving the original directory structure. **Filtering by keywords**: @@ -125,14 +125,14 @@ python3 scripts/recover_content.py session.jsonl -o ./feature_xy_history/ After recovery, always verify content: ```bash -# Check file list -ls -lh ./recovered_content/ +# Check directory structure (files preserved in subdirectories) +find ./recovered_content/ -type f -# Read recovery report +# Read recovery report (shows full output paths) cat ./recovered_content/recovery_report.txt -# Spot-check content -head -20 ./recovered_content/ImportantFile.jsx +# Spot-check content (use actual path from report) +head -20 ./recovered_content/src/components/ImportantFile.jsx ``` ## Limitations @@ -200,7 +200,7 @@ Always sanitize before sharing: ```bash # Remove absolute paths -sed -i '' 's|/Users/[^/]*/|/Users/username/|g' file.js +sed -i '' 's|~/|/|g' file.js # Verify no credentials grep -i "api_key\|password\|token" recovered_content/* @@ -209,3 +209,15 @@ grep -i "api_key\|password\|token" recovered_content/* ### Safe Storage Recovered content inherits sensitivity from original sessions. Store securely and follow organizational policies for handling session data. + +## Next Step: Resume Interrupted Work + +After finding relevant session history, suggest continuing the work: + +``` +Found [N] relevant sessions with recoverable context. + +Options: +A) Resume work — run /continue-claude-work to pick up where you left off (Recommended) +B) Just show me the content — I'll decide what to do with it +``` diff --git a/claude-code-history-files-finder/references/session_file_format.md b/daymade-claude-code/claude-code-history-files-finder/references/session_file_format.md similarity index 97% rename from claude-code-history-files-finder/references/session_file_format.md rename to daymade-claude-code/claude-code-history-files-finder/references/session_file_format.md index f7763098..41684ed2 100644 --- a/claude-code-history-files-finder/references/session_file_format.md +++ b/daymade-claude-code/claude-code-history-files-finder/references/session_file_format.md @@ -15,8 +15,8 @@ Claude Code stores conversation history in JSONL (JSON Lines) format, where each **Path normalization**: Project paths are converted by replacing `/` with `-` Example: -- Project: `/Users/username/Workspace/js/myproject` -- Directory: `~/.claude/projects/-Users-username-Workspace-js-myproject/` +- Project: `~/Workspace/js/myproject` +- Directory: `~/.claude/projects/-Users--Workspace-js-myproject/` ### File Types diff --git a/claude-code-history-files-finder/references/workflow_examples.md b/daymade-claude-code/claude-code-history-files-finder/references/workflow_examples.md similarity index 92% rename from claude-code-history-files-finder/references/workflow_examples.md rename to daymade-claude-code/claude-code-history-files-finder/references/workflow_examples.md index 62c7ab58..20bb0bed 100644 --- a/claude-code-history-files-finder/references/workflow_examples.md +++ b/daymade-claude-code/claude-code-history-files-finder/references/workflow_examples.md @@ -40,8 +40,9 @@ python3 scripts/recover_content.py session1.jsonl -k componentName -o ./v1/ python3 scripts/recover_content.py session2.jsonl -k componentName -o ./v2/ python3 scripts/recover_content.py session3.jsonl -k componentName -o ./v3/ -# 4. Compare versions -diff ./v1/componentName.jsx ./v2/componentName.jsx +# 4. Compare versions (files retain original directory structure) +# Use find to locate the file in subdirectories, or reference the recovery_report.txt +find ./v1/ -name "componentName.jsx" -exec diff {} ./v2/{} \; ``` ## Find Session with Specific Implementation diff --git a/claude-code-history-files-finder/scripts/analyze_sessions.py b/daymade-claude-code/claude-code-history-files-finder/scripts/analyze_sessions.py similarity index 98% rename from claude-code-history-files-finder/scripts/analyze_sessions.py rename to daymade-claude-code/claude-code-history-files-finder/scripts/analyze_sessions.py index 09708748..25939b1c 100755 --- a/claude-code-history-files-finder/scripts/analyze_sessions.py +++ b/daymade-claude-code/claude-code-history-files-finder/scripts/analyze_sessions.py @@ -36,13 +36,13 @@ def find_project_sessions(self, project_path: str) -> List[Path]: Find all session files for a specific project. Args: - project_path: Project path (e.g., /Users/user/Workspace/js/myproject) + project_path: Project path (e.g., ~/Workspace/js/myproject) Returns: List of session file paths """ # Convert project path to Claude's directory naming - # Example: /Users/user/Workspace/js/myproject -> -Users-user-Workspace-js-myproject + # Example: ~/Workspace/js/myproject -> -Users--Workspace-js-myproject normalized = project_path.replace("/", "-") project_dir = self.projects_dir / normalized diff --git a/claude-code-history-files-finder/scripts/recover_content.py b/daymade-claude-code/claude-code-history-files-finder/scripts/recover_content.py similarity index 85% rename from claude-code-history-files-finder/scripts/recover_content.py rename to daymade-claude-code/claude-code-history-files-finder/scripts/recover_content.py index e7e69458..3b169584 100755 --- a/claude-code-history-files-finder/scripts/recover_content.py +++ b/daymade-claude-code/claude-code-history-files-finder/scripts/recover_content.py @@ -120,7 +120,7 @@ def save_recovered_files( self, write_calls: List[Dict[str, Any]], keywords: Optional[List[str]] = None ) -> List[Dict[str, Any]]: """ - Save recovered files to disk. + Save recovered files to disk, preserving original directory structure. Args: write_calls: List of Write tool calls @@ -152,18 +152,43 @@ def save_recovered_files( # Save files for file_path, call in files_by_path.items(): try: - filename = Path(file_path).name - if not filename: + if not file_path: continue - output_file = self.output_dir / filename + # Preserve original directory structure + # Convert absolute path to relative path within output directory + original_path = Path(file_path) + + # Handle absolute paths: extract meaningful relative path + # e.g., /Users/username/project/src/file.py -> src/file.py + # e.g., /home/user/workspace/project/lib/module.py -> lib/module.py + path_parts = original_path.parts + if len(path_parts) > 1 and path_parts[0] == "/": + # For absolute paths, try to find a project-like directory + # Skip leading /, Users/username, home/username patterns + start_idx = 1 # Skip leading "/" + if len(path_parts) > 2 and path_parts[1].lower() in ("users", "home", "user"): + start_idx = 3 # Skip /Users/username or /home/user + relative_parts = path_parts[start_idx:] + else: + relative_parts = path_parts + + # Construct output path preserving structure + if relative_parts: + output_file = self.output_dir.joinpath(*relative_parts) + else: + # Fallback to filename only if path is too shallow + output_file = self.output_dir / original_path.name + + # Create parent directories + output_file.parent.mkdir(parents=True, exist_ok=True) with open(output_file, "w") as f: f.write(call["content"]) saved.append( { - "file": filename, + "file": output_file.name, "original_path": file_path, "size": len(call["content"]), "lines": call["content"].count("\n") + 1, diff --git a/daymade-claude-code/claude-export-txt-better/.security-scan-passed b/daymade-claude-code/claude-export-txt-better/.security-scan-passed new file mode 100644 index 00000000..33e556ca --- /dev/null +++ b/daymade-claude-code/claude-export-txt-better/.security-scan-passed @@ -0,0 +1,4 @@ +Security scan passed +Scanned at: 2026-04-11T23:12:01.572016 +Tool: gitleaks + pattern-based validation +Content hash: c295177ee2b1aa1844a2758bcf6e5395f8819c445a050b80a2d34b8983515459 diff --git a/daymade-claude-code/claude-export-txt-better/SKILL.md b/daymade-claude-code/claude-export-txt-better/SKILL.md new file mode 100644 index 00000000..3caa42d2 --- /dev/null +++ b/daymade-claude-code/claude-export-txt-better/SKILL.md @@ -0,0 +1,101 @@ +--- +name: fixing-claude-export-conversations +description: > + Fixes broken line wrapping in Claude Code exported conversation files (.txt), + reconstructing tables, paragraphs, paths, and tool calls that were hard-wrapped + at fixed column widths. Includes an automated validation suite (generic, file-agnostic checks). + Triggers when the user has a Claude Code export file with broken formatting, + mentions "fix export", "fix conversation", "exported conversation", "make export + readable", references a file matching YYYY-MM-DD-HHMMSS-*.txt, or has a .txt + file with broken tables, split paths, or mangled tool output from Claude Code. +--- + +# Fixing Claude Code Export Conversations + +Reconstruct broken line wrapping in Claude Code exported `.txt` files. + +## Quick Start + +```bash +# Fix and show stats +uv run /scripts/fix-claude-export.py --stats + +# Custom output +uv run /scripts/fix-claude-export.py -o fixed.txt + +# Validate the result (53 automated checks) +uv run /scripts/validate-claude-export-fix.py fixed.txt +``` + +Replace `` with the resolved path to this skill's directory. Find it with: +```bash +find ~/.claude -path "*/fixing-claude-export-conversations/scripts" -type d 2>/dev/null +``` + +## Workflow + +Copy this checklist and track progress: + +``` +- [ ] Step 1: Locate the exported .txt file +- [ ] Step 2: Run fix script with --stats +- [ ] Step 3: Run validation suite +- [ ] Step 4: Spot-check output (tables, CJK paragraphs, tool results) +- [ ] Step 5: Deliver fixed file to user +``` + +**Step 1: Locate the file.** Claude Code exports use the naming pattern `YYYY-MM-DD-HHMMSS-.txt`. + +**Step 2: Run the fix script.** +```bash +uv run /scripts/fix-claude-export.py -o --stats +``` +Review the stats output — typical results: 20-25% line reduction, 80+ table borders fixed, 160+ table cells fixed. + +**Step 3: Run the validation suite.** +```bash +uv run /scripts/validate-claude-export-fix.py +``` +All checks must pass. If any fail, investigate before delivering. Use `--verbose` for full details on passing checks too. + +**Step 4: Spot-check.** Open the output and verify: +- Tables have intact borders (box-drawing characters on single lines) +- CJK/English mixed text has pangu spacing (`Portal 都需要`, not `Portal都需要`) +- Tool result blocks (`⎿`) have complete content on joined lines +- Diff output within tool results has each line number on its own line + +**Step 5: Deliver** the fixed file to the user. + +## What Gets Fixed + +The script handles 10 content types using a state-machine with next-line look-ahead: + +- **User prompts** (`❯` prefix, dw=76 padding) — paragraph joins with pangu spacing +- **Claude responses** (`●` prefix) — narrative, bullet, and numbered list joins +- **Claude paragraphs** (2-space indent) — next-line look-ahead via `_is_continuation_fragment` +- **Tables** — border reconstruction, cell re-padding with pipe-count tracking +- **Tool calls** (`● Bash(` etc.) — path and argument reconstruction +- **Tool results** (`⎿` prefix) — continuation joins including deeper-indented fragments +- **Plan text** (5-space indent) — next-line look-ahead via `_is_plan_continuation_fragment` +- **Agent tree** (`├─`/`└─`) — preserved structure +- **Separators** (`────`, `---`) — never joined +- **Tree connectors** (standalone `│`) — preserved + +## Key Design Decisions + +**Next-line look-ahead** (not dw thresholds): Instead of asking "was this line wrapped?" (fragile threshold), the script asks "does the next line look like a continuation?" by examining its content patterns — lowercase start, CJK ideograph start, opening bracket, hyphen/slash/underscore continuation. + +**Pangu spacing**: Inserts spaces between ASCII alphanumeric characters and CJK ideographs at join boundaries. Also triggers for `%`, `#`, `+`, `:` adjacent to CJK. + +**Mid-token detection**: Joins without space when boundaries indicate identifiers (`BASE_` + `URL`), paths (`documents` + `/05-team`), or hyphenated names (`ready` + `-together`). Exception: `--` prefix gets a space (`run` + `--headed`). + +## Safety + +- Never modifies the original file +- Marker counts verified: `❯`, `●`, `✻`, `⎿`, `…` must match input/output +- Runaway join detection: warns if any line exceeds 500 display-width +- Strict UTF-8 encoding — no silent fallbacks + +## Dependencies + +Python 3.10+ via `uv run` — zero external packages (stdlib only: `unicodedata`, `argparse`, `re`, `pathlib`, `dataclasses`). \ No newline at end of file diff --git a/daymade-claude-code/claude-export-txt-better/evals/evals.json b/daymade-claude-code/claude-export-txt-better/evals/evals.json new file mode 100644 index 00000000..b0d7dc4e --- /dev/null +++ b/daymade-claude-code/claude-export-txt-better/evals/evals.json @@ -0,0 +1,32 @@ +{ + "skill_name": "fixing-claude-export-conversations", + "evals": [ + { + "id": 1, + "prompt": "Fix this exported conversation file: — it was exported from Claude Code and has broken line wrapping everywhere. Output the fixed version next to the original.", + "expected_output": "A *-fixed.txt file with correct line wrapping, intact table borders, matching marker counts, and fewer total lines than the input.", + "files": [], + "assertions": [ + {"name": "output_file_exists", "description": "The fixed output file was created successfully"}, + {"name": "line_count_reduced", "description": "Output has fewer lines than input. Reasonable reduction is 10-25%."}, + {"name": "marker_counts_match", "description": "The count of ❯, ●, ✻, ⎿, and … characters in output matches input exactly"}, + {"name": "no_broken_table_borders", "description": "Every line starting with ┌/├/└ also contains the matching right border on the same line"}, + {"name": "no_runaway_joins", "description": "No output line exceeds 500 display-width characters"} + ] + }, + { + "id": 2, + "prompt": "我从 Claude Code 导出了一个对话记录 ，表格和段落的换行全部乱了。帮我修复一下，顺便看看统计数据——修了多少行？", + "expected_output": "Fixed file created with --stats output showing join counts per content type. Tables should be properly reconstructed with correct column padding.", + "files": [], + "assertions": [ + {"name": "output_file_exists", "description": "The fixed output file was created successfully"}, + {"name": "line_count_reduced", "description": "Output has fewer lines than input. Reasonable reduction is 10-25%."}, + {"name": "marker_counts_match", "description": "The count of ❯, ●, ✻, ⎿, and … characters in output matches input exactly"}, + {"name": "no_broken_table_borders", "description": "Every line starting with ┌/├/└ also contains the matching right border on the same line"}, + {"name": "no_runaway_joins", "description": "No output line exceeds 500 display-width characters"}, + {"name": "stats_output_present", "description": "The run log contains statistics output with join counts (e.g., 'Table rows merged', 'Tool results fixed')"} + ] + } + ] +} diff --git a/daymade-claude-code/claude-export-txt-better/scripts/fix-claude-export.py b/daymade-claude-code/claude-export-txt-better/scripts/fix-claude-export.py new file mode 100644 index 00000000..27c50d6e --- /dev/null +++ b/daymade-claude-code/claude-export-txt-better/scripts/fix-claude-export.py @@ -0,0 +1,2065 @@ +#!/usr/bin/env python3 +"""Fix broken line wrapping in Claude Code exported conversation files. + +Claude Code exports hard-wrap lines at fixed column widths, breaking tables, +paragraphs, and paths. This script reconstructs the original logical lines +using a state-machine + lookahead merge approach. + +Usage: + uv run scripts/fix-claude-export.py + uv run scripts/fix-claude-export.py -o + uv run scripts/fix-claude-export.py --stats --dry-run +""" + +from __future__ import annotations + +import argparse +import re +import sys +import unicodedata +from dataclasses import dataclass +from pathlib import Path + +# --------------------------------------------------------------------------- +# Display-width helpers +# --------------------------------------------------------------------------- + +def display_width(s: str) -> int: + """Calculate display width accounting for CJK double-width characters.""" + w = 0 + for ch in s: + eaw = unicodedata.east_asian_width(ch) + w += 2 if eaw in ("W", "F") else 1 + return w + + +def is_wide_char(ch: str) -> bool: + """Return True if *ch* occupies two display columns (CJK full-width).""" + return unicodedata.east_asian_width(ch) in ("W", "F") + + +def _is_cjk_ideograph(ch: str) -> bool: + """Return True if *ch* is a CJK ideograph (not punctuation/symbol). + + CJK ideographs have Unicode category ``Lo`` (Letter, other) — e.g. + ``你``, ``好``, ``接``. CJK punctuation (``。，！？；：「」（）``) has + categories ``Ps``, ``Pe``, ``Po``, etc. and should NOT match. + + This distinction matters for pangu spacing: a space is inserted between + ASCII alphanumeric characters and CJK ideographs, but NOT between + CJK punctuation and anything. + """ + return is_wide_char(ch) and unicodedata.category(ch) == "Lo" + + +# --------------------------------------------------------------------------- +# Join helpers +# --------------------------------------------------------------------------- + +def smart_join(left: str, right_content: str) -> str: + """Join text with CJK-aware spacing (pangu style). + + Spacing rules at the join boundary: + + - **CJK ↔ CJK**: no space (both characters wide). + - **ASCII alnum ↔ CJK ideograph**: insert one space. This is "pangu + spacing" — the standard practice of separating Han characters from + Latin letters / digits in mixed CJK/English text. The original + content almost always has these spaces; they are lost when Claude + hard-wraps at the column boundary. + - **CJK punctuation ↔ anything**: no space. Punctuation like ``，`` + ``。`` ``）`` ``（`` clings to its neighbor. + - **ASCII ↔ ASCII**: insert one space (English word boundary). + """ + left_s = left.rstrip() + right_s = right_content.lstrip() + if not left_s or not right_s: + return left_s + right_s + last_ch = left_s[-1] + first_ch = right_s[0] + + last_wide = is_wide_char(last_ch) + first_wide = is_wide_char(first_ch) + + if last_wide and first_wide: + # Both CJK — no space. + return left_s + right_s + + if last_wide or first_wide: + # Mixed CJK/ASCII boundary — apply pangu spacing. + # In addition to alnum, certain symbols that attach to numbers + # or abbreviations (%, #, +, :) also trigger pangu spacing because + # they're part of the same "token" as the adjacent number/word. + _PANGU_SYMBOLS = "%#+:" + if last_wide: + # CJK → ASCII: space if CJK ideograph + ASCII alnum/symbol. + if _is_cjk_ideograph(last_ch) and (first_ch.isalnum() or first_ch in _PANGU_SYMBOLS): + return left_s + " " + right_s + return left_s + right_s + else: + # ASCII → CJK: space if ASCII alnum/symbol + CJK ideograph. + if (last_ch.isalnum() or last_ch in _PANGU_SYMBOLS) and _is_cjk_ideograph(first_ch): + return left_s + " " + right_s + return left_s + right_s + + # Both ASCII — mid-token detection before adding space. + # Path continuation: alnum + / (e.g., "documents" + "/05-team") + # Hyphen continuation: alnum + - (e.g., "ready" + "-together") + # Underscore continuation: _ + alnum (e.g., "BASE_" + "URL") + # Exception: -- prefix is a CLI flag (e.g., "run" + "--headed"). + if last_ch.isalnum() and first_ch in ("-", "/"): + if first_ch == "-" and right_s.startswith("--"): + pass # fall through to default (add space) + else: + return left_s + right_s + if last_ch in ("-", "/") and first_ch.isalnum(): + return left_s + right_s + # Underscore at identifier boundary (e.g., "E2E_PO_BASE_" + "URL") + if last_ch == "_" and first_ch.isalnum(): + return left_s + right_s + if last_ch.isalnum() and first_ch == "_": + return left_s + right_s + + # Default: word boundary → add space. + return left_s + " " + right_s + + +def raw_join(left: str, right: str) -> str: + """Strip trailing whitespace from *left*, leading spaces from *right*, concat.""" + return left.rstrip() + right.lstrip() + + +def _table_cell_content_join( + left: str, right: str, *, left_filled: bool = False +) -> str: + """Join two cell-content fragments from a multi-row table cell. + + Claude wraps long cell content across multiple physical rows at fixed + column widths. This function reassembles the original content by + detecting mid-word breaks (e.g. ``Contr`` + ``oller``) vs word + boundaries (e.g. ``Backend`` + ``Risk``). + + When *left_filled* is True, the left fragment filled its entire column + (≤1 trailing space before the ``│`` delimiter), meaning the split was + forced at the column boundary — almost certainly mid-word. This is the + strongest signal and overrides other heuristics. + + Heuristic priority: + 1. left_filled → raw join (column-boundary split) + 2. Continuation punctuation at boundary (``- _ . /``) → raw join + 3. Left ends with letter, right starts with lowercase → mid-word + 4. Left ends with digit, right starts with digit → mid-number + 5. Otherwise → smart_join (CJK-aware spacing) + """ + left_s = left.rstrip() + right_s = right.lstrip() + if not left_s or not right_s: + return left_s + right_s + + last_ch = left_s[-1] + first_ch = right_s[0] + + # Continuation punctuation at boundary — usually concatenate directly. + # Exception: double-hyphen at right boundary (``--flag``) is a CLI + # argument prefix, not a continuation of the left content. + if last_ch in ("-", "_", ".", "/") or first_ch in ("-", "_", ".", "/"): + if first_ch == "-" and right_s.startswith("--"): + pass # fall through to other checks + else: + return left_s + right_s + + if left_filled: + # Column-boundary split: content filled the cell width. + # The break is at a fixed position and likely mid-token, but we + # verify with character-class checks to avoid false positives + # like "Behavioral" + "Spec" (complete word at column edge). + if last_ch.isalpha() and first_ch.islower(): + return left_s + right_s + if last_ch.isupper() and first_ch.isupper(): + return left_s + right_s + if last_ch.isdigit() and first_ch.isdigit(): + return left_s + right_s + if last_ch.isalnum() and first_ch.isdigit(): + return left_s + right_s + if last_ch.isdigit() and first_ch.islower(): + # Hex hash continuation: c3df79 + b → c3df79b + return left_s + right_s + # Filled but no mid-word evidence → fall through to smart_join. + + # Everything else: word-boundary or ambiguous → CJK-aware spacing. + return smart_join(left_s, right_s) + + +def table_cell_join(left: str, right: str) -> str: + """Join table cell continuation, preserving column spacing. + + Unlike raw_join, this strips only the table indent (matching the left + side's indent) from the right side, keeping internal cell padding. + It also ensures a space between ``│`` and adjacent cell content. + """ + ls = left.rstrip() + if not ls: + return right.rstrip() + # Determine indent from left side (typically 5 spaces for plan tables) + indent = len(ls) - len(ls.lstrip()) + rs = right.rstrip() + # Strip exactly the indent, keep remaining whitespace (cell padding) + if len(rs) >= indent and rs[:indent].strip() == "": + rs = rs[indent:] + else: + rs = rs.lstrip() + if not rs: + return ls + # Ensure space between │ and non-│/non-space content + if ls[-1] == "│" and rs[0] not in " │": + return ls + " " + rs + if ls[-1] not in " │" and rs[0] == "│": + return ls + " " + rs + return ls + rs + + +def boundary_aware_join(left: str, right: str) -> str: + """Join with heuristic for mid-word vs word-boundary splits. + + If *left* had a trailing space before stripping, the hard wrap was at a + word boundary -- preserve one space. Otherwise the wrap split mid-word + -- concatenate directly (no space). + """ + # Check if left had trailing whitespace (word-boundary wrap). + left_had_trailing_space = left != left.rstrip() + left_s = left.rstrip() + right_s = right.lstrip() + if not left_s or not right_s: + return left_s + right_s + if left_had_trailing_space: + # Word-boundary wrap: preserve spacing via smart_join which + # handles pangu spacing (ASCII alnum ↔ CJK ideograph). + return smart_join(left_s, right_s) + # Mid-word wrap: no space. + return left_s + right_s + + +def _dw_aware_join(left: str, right: str, left_dw: int) -> str: + """Join with display-width-aware mid-word detection. + + Claude drops the trailing space at wrap points, making it impossible to + distinguish word-boundary from mid-word breaks by whitespace alone. + This function uses the display width of the *left* physical line to + resolve the ambiguity: + + - **dw < 74**: the line ended well below the wrap column (~76) — the + break was at a natural word boundary → smart_join. + - **dw >= 75**: the line was forced to break near the column limit. + Use character-class heuristics: alpha→lower = likely mid-word (raw + join); all other transitions = word boundary (smart_join). + """ + if left_dw < 75: + return smart_join(left, right) + + # Near wrap limit — check trailing space first. + left_had_trailing = left != left.rstrip() + left_s = left.rstrip() + right_s = right.lstrip() + if not left_s or not right_s: + return left_s + right_s + + if left_had_trailing: + return smart_join(left_s, right_s) + + # No trailing space, near wrap limit. + # Character-class heuristics for mid-word detection. + last_ch = left_s[-1] + first_ch = right_s[0] + # alpha→lower: mid-word (e.g., "Backgrou" + "nd") + if last_ch.isalpha() and first_ch.islower(): + return left_s + right_s + # Path/hyphenated-name continuations: slash or hyphen at boundary + # (e.g., ".claude/skills" + "/generating-..." or "ready" + "-together") + if last_ch.isalnum() and first_ch in ("-", "/"): + return left_s + right_s + if last_ch in ("-", "/") and first_ch.isalnum(): + return left_s + right_s + # digit→alpha (e.g., "md-e" + "2e-section" — hex/version fragments) + if last_ch.isalpha() and first_ch.isdigit(): + return left_s + right_s + + # All other cases: treat as word boundary. + return smart_join(left_s, right_s) + + +def _bullet_join(left: str, right: str) -> str: + """Join a bullet line with its wrapped continuation. + + Like smart_join, but with mid-word detection: when left ends with an + ASCII letter and right (after stripping) starts with a lowercase ASCII + letter, concatenate directly (the hard-wrap split a word mid-token, + e.g. ``RiskModelAss`` + ``ignment``). Otherwise delegate to smart_join + for CJK-aware spacing. + """ + left_s = left.rstrip() + right_s = right.lstrip() + if not left_s or not right_s: + return left_s + right_s + last_ch = left_s[-1] + first_ch = right_s[0] + # Mid-word split: left ends with a letter, right starts with lowercase. + if last_ch.isalpha() and first_ch.islower(): + return left_s + right_s + # Hyphenated names / paths split at boundary. + # e.g. "ready" + "-together-project" or "skills" + "/generating" + if last_ch.isalnum() and first_ch in ("-", "/"): + return left_s + right_s + if last_ch in ("-", "/") and first_ch.isalnum(): + return left_s + right_s + return smart_join(left_s, right_s) + + +# --------------------------------------------------------------------------- +# Line classification helpers +# --------------------------------------------------------------------------- + +# Markers that ALWAYS start a new logical line (never join TO these). +_USER_PROMPT_RE = re.compile(r"^❯ ") +_CLAUDE_ACTION_RE = re.compile(r"^● ") +_THINKING_RE = re.compile(r"^✻ ") +_HR_RE = re.compile(r"^ ---") +_BOX_HR_RE = re.compile(r"^ ────") +_AGENT_TREE_RE = re.compile(r"^ [├└]─") +_TOOL_RESULT_RE = re.compile(r"^ ⎿") +_BULLET_RE = re.compile(r"^ - ") +_NUMBERED_RE = re.compile(r"^ \d+\. ") + +# Indented bullets / numbered items inside plan blocks (5-space indent). +_PLAN_BULLET_RE = re.compile(r"^ - ") +_PLAN_NUMBERED_RE = re.compile(r"^ \d+\. ") + +# Tool call openers (● ToolName( ...). +_TOOL_CALL_RE = re.compile( + r"^● (?:Bash|Read|Write|Glob|Grep|Edit|Update|Searched|NotebookEdit)\(" +) + +# Table box-drawing characters. +_TABLE_CORNERS = set("┐┤┘") + + +def _is_truly_empty(line: str) -> bool: + """A truly empty line (zero length after stripping the newline).""" + return len(line) == 0 + + +def _is_structural_break(line: str) -> bool: + """Return True if *line* is a structural marker that must never be joined TO.""" + if _is_truly_empty(line): + return True + if _USER_PROMPT_RE.match(line): + return True + if _CLAUDE_ACTION_RE.match(line): + return True + if _THINKING_RE.match(line): + return True + if _HR_RE.match(line): + return True + if _BOX_HR_RE.match(line): + return True + if _AGENT_TREE_RE.match(line): + return True + if _TOOL_RESULT_RE.match(line): + return True + if _BULLET_RE.match(line): + return True + if _NUMBERED_RE.match(line): + return True + return False + + +# Regex for CJK labels like 模块:, 输出文件:, 状态:, 覆盖范围: +_CJK_LABEL_RE = re.compile(r"[\u4e00-\u9fff]{1,6}[:：]") +# Regex for English labelled list items: "Phase 1:", "Step 2:", "Layer 3:" +_LABELLED_ITEM_RE = re.compile(r"[A-Z]\w+ \d+[:.] ") + + +def _is_continuation_fragment(nl: str, acc: str) -> bool: + """Return True if *nl* looks like a wrapped continuation of *acc*. + + This is the core predicate for joining 2-space-indented paragraph text. + Instead of asking "was the current line wrapped?" (fragile dw threshold), + it asks "does the NEXT line look like a continuation fragment?" + + A continuation fragment has NO structural identity — it is not a new + bullet, numbered item, labelled field, or structural marker. It + typically starts with a lowercase letter, CJK ideograph, or is a short + uppercase fragment of a sentence that wrapped mid-phrase. + """ + # Must be 2-space indent (not deeper, not tool result). + if not nl.startswith(" "): + return False + if nl.startswith(" "): # 5+ space = plan/tool block + return False + if nl.startswith(" ⎿"): + return False + if _is_structural_break(nl): + return False + + stripped = nl.lstrip() + if not stripped: + return False + + # --- New-item patterns (NOT a continuation) --- + if stripped.startswith("- "): + return False + if re.match(r"\d+[.)] ", stripped): + return False + if _LABELLED_ITEM_RE.match(stripped): + return False + # CJK labels: 模块:, 输出文件:, 状态:, 覆盖范围: etc. + if _CJK_LABEL_RE.match(stripped): + return False + # Column layout (side-by-side comparison with internal spacing). + if " " in stripped: # 8+ internal spaces + return False + + nl_dw = display_width(nl.rstrip()) + # A "continuation" line that is itself full-width is likely independent. + if nl_dw >= 76: + return False + + first_ch = stripped[0] + + # --- Strong continuation signals --- + # Lowercase → mid-sentence continuation. + if first_ch.islower(): + return True + # CJK ideograph → continuing Chinese text. + if _is_cjk_ideograph(first_ch): + return True + # CJK/fullwidth punctuation → continues previous CJK content. + if is_wide_char(first_ch) and not _is_cjk_ideograph(first_ch): + return True + # Opening bracket → e.g. "(c67e5ded-..." UUID, list in parens. + if first_ch in ("(", "[", "{", "（", "「", "【"): + return True + # Hyphen/slash continuation: acc ends with alnum, next starts with -//. + # (e.g. "ready" + "-together-project", "skills" + "/generating") + if first_ch in ("-", "/"): + _acc_s = acc.rstrip() + if _acc_s and _acc_s[-1].isalnum(): + return True + + # --- Check if accumulated text signals continuation --- + acc_stripped = acc.rstrip() + if acc_stripped: + last_acc = acc_stripped[-1] + # Continuation operators at end of acc → next line must continue. + if last_acc in (",", "+", "→", "=", "&", "|", "、"): + return True + # Acc ends with sentence-terminal → next line is a new sentence. + if last_acc in "。.！!？?；;": + return False + + # --- Uppercase start: ambiguous --- + # Short fragment (< 55 dw) is likely a sentence fragment. + if nl_dw < 55: + return True + return False + + +def _is_plan_continuation_fragment(nl: str, acc: str) -> bool: + """Return True if *nl* looks like a wrapped continuation in 5-space plan text. + + Same design philosophy as _is_continuation_fragment: examine the NEXT + line's content to decide if it is a new item or a continuation fragment. + """ + if not nl.startswith(" "): + return False + if nl.startswith(" "): # 7+ space = deeper indent, separate item + return False + if _is_truly_empty(nl): + return False + if _is_structural_break(nl): + return False + if _is_plan_structural(nl): + return False + + stripped = nl.lstrip() + if not stripped: + return False + + # CJK labels (模块:, 输出文件:, 状态:) + if _CJK_LABEL_RE.match(stripped): + return False + # English labelled items with number (Phase 1:, Step 2:) + if _LABELLED_ITEM_RE.match(stripped): + return False + # English labels without number (Plan:, Context:, Summary:) + if re.match(r"[A-Z][a-zA-Z]+: ", stripped): + return False + # ASCII terminal labels (error:, hint:, remote:) + if re.match(r"[a-z]+: ", stripped): + return False + # Diff output line numbers (e.g., "600 - **Test Data**:") + # Pattern: 1-5 digits followed by 2+ spaces (diff line number format). + if re.match(r"\d{1,5}\s{2,}", stripped): + return False + # Column layout + if " " in stripped: + return False + + nl_dw = display_width(nl.rstrip()) + if nl_dw >= 76: + return False + + first_ch = stripped[0] + + # --- New-item patterns that start with lowercase --- + # ASCII labels like "error:", "hint:", "remote:" are terminal output + # lines, not continuations. They start a new message. + if re.match(r"[a-z]+: ", stripped): + return False + + # --- Strong continuation signals --- + if first_ch.islower(): + return True + if _is_cjk_ideograph(first_ch): + return True + if is_wide_char(first_ch) and not _is_cjk_ideograph(first_ch): + return True + if first_ch in ("(", "[", "{", "（", "「", "【"): + return True + # Hyphen/slash continuation (compound names, paths). + if first_ch in ("-", "/"): + _acc_s = acc.rstrip() + if _acc_s and _acc_s[-1].isalnum(): + return True + + # Non-alnum, non-CJK, non-bracket starts (!, #, >, *, etc.) + # are structural markers in terminal output, not continuations. + if not first_ch.isalnum(): + return False + + # --- Check accumulated text ending --- + acc_stripped = acc.rstrip() + if acc_stripped: + last_acc = acc_stripped[-1] + if last_acc in (",", "+", "→", "=", "&", "|", "、"): + return True + if last_acc in "。.！!？?；;": + return False + + # Uppercase start, ambiguous: short fragment = likely continuation. + if nl_dw < 55: + return True + return False + + +def _is_plan_structural(line: str) -> bool: + """Structural markers within 5-space-indented plan blocks.""" + if _PLAN_BULLET_RE.match(line): + return True + if _PLAN_NUMBERED_RE.match(line): + return True + stripped = line.lstrip() + if stripped.startswith("##"): + return True + # Box-drawing separators within plan blocks (5-space indent). + # The 2-space variant is caught by _BOX_HR_RE in _is_structural_break, + # but 5-space-indented ones slip through. + if stripped.startswith("────"): + return True + # Markdown HR within plan blocks (5-space indent). + if stripped == "---": + return True + # Tree connector (standalone │ used in ASCII dependency diagrams). + # Must not be confused with table data rows (which have 2+ │ chars). + if stripped == "│": + return True + # File-tree lines (├── or └── patterns) within plan blocks. + if stripped.startswith("├──") or stripped.startswith("└──"): + return True + # Expansion indicators ("… +N lines (ctrl+o to expand)"). + if stripped.startswith("…"): + return True + return False + + +def _has_continuation_signal(line: str) -> bool: + """Detect if *line* was almost certainly hard-wrapped mid-content. + + Lines ending with a trailing comma, a CJK character, or an unclosed + bracket are continuation signals — they indicate the content continues + on the next line regardless of how narrow the display width is. + """ + stripped = line.rstrip() + if not stripped: + return False + last_ch = stripped[-1] + if last_ch == ",": + return True + if is_wide_char(last_ch): + return True + if last_ch in ("(", "[", "{"): + return True + return False + + +def _has_unclosed_bracket(line: str) -> bool: + """Detect if *line* contains an opening bracket with no matching close. + + An unclosed bracket means the parenthetical/list continues on the next + physical line — a strong continuation signal regardless of display width. + This catches cases like ``Requirements（P0/P1 分层，每个 Feature`` where + Claude wraps at a word boundary well below the column limit because the + remaining CJK text would push past it. + """ + _PAIRS = (("(", ")"), ("[", "]"), ("{", "}"), + ("（", "）"), ("「", "」"), ("【", "】")) + for open_ch, close_ch in _PAIRS: + if open_ch in line and close_ch not in line: + return True + return False + + +def _looks_like_mid_word(left: str, right: str) -> bool: + """Heuristic: detect if a tool-call wrap split a token mid-character. + + Returns True when both sides appear to be fragments of one continuous + filesystem path or identifier. This is deliberately conservative -- + when in doubt, return False so a space gets inserted. + """ + if not left or not right: + return False + lc = left[-1] + rc = right[0] + # Mid-path: one side has '/' at the boundary. + if lc == "/" or rc == "/": + return True + # Mid-word with hyphen/underscore continuation + # (e.g., "/skills/gener" + "ating-e2e-test-suite"). + # Must distinguish from command-argument boundaries + # (e.g., "git add" + "test-cases/...") by checking the character + # preceding the left side's last token. + if lc.isalpha() and rc.isalpha() and lc.islower() and rc.islower(): + first_token = "" + for ch in right: + if ch.isalnum() or ch in "-_.": + first_token += ch + else: + break + if "-" in first_token or "_" in first_token: + # Find start of last token on the left side + last_token_start = len(left) + while last_token_start > 0 and ( + left[last_token_start - 1].isalnum() + or left[last_token_start - 1] in "-_." + ): + last_token_start -= 1 + # If preceded by '/', it might be mid-path -- but check if the + # token is a complete filename (has file extension like .md/.py). + if last_token_start > 0 and left[last_token_start - 1] == "/": + last_token = left[last_token_start:] + if re.search(r"\.\w{1,5}$", last_token): + return False # Complete filename, not a fragment + return True # Mid-path fragment (e.g., /skills/gener) + # If preceded by space or start-of-string, it's a word + # boundary (e.g., "git add" + "test-cases") -> insert space + return False + return False + + +# --------------------------------------------------------------------------- +# Table detection helpers +# --------------------------------------------------------------------------- + +def is_table_border_split(current: str, next_line: str) -> bool: + """Detect a table border that was split across two lines.""" + cs = current.rstrip() + if not cs or cs[-1] != "─": + return False + ns = next_line.lstrip() + if not ns: + return False + return ns[0] == "─" and ns.rstrip()[-1] in _TABLE_CORNERS + + +def _is_table_border_line(line: str) -> bool: + """Check if line is a table border (├─, └─, ┌─ patterns).""" + stripped = line.lstrip() + if not stripped: + return False + return stripped[0] in "├└┌" and "─" in stripped + + +def _count_expected_pipes(border_line: str) -> int: + """Count expected │ per data row from a ┌ or ├ border line. + + A border like ``┌──┬──┬──┐`` has 2 ``┬`` → 3 columns → 4 ``│`` per row. + """ + return border_line.count("┬") + 2 + + +def _parse_column_widths(border_line: str) -> list[int]: + """Extract column display widths from a ┌ or ├ border line. + + Splits by column separators (┬ or ┼) and counts ``─`` chars per segment. + Returns a list of column widths (inner content width, not including │). + """ + stripped = border_line.strip() + if len(stripped) < 2: + return [] + inner = stripped[1:-1] # Remove corner chars (┌/┐ or ├/┤) + segments = re.split("[┬┼]", inner) + return [len(seg) for seg in segments] + + +def _repad_table_row(row: str, col_widths: list[int]) -> str: + """Re-pad each cell in a table data row to match the column widths. + + Preserves existing left padding and content; only adds right-padding + so each cell reaches the correct display width from the border. + """ + parts = row.split("│") + # parts[0] = indent before first │; parts[-1] = after last │ (empty) + if len(parts) < 3: + return row + indent = parts[0] + cells = parts[1:-1] + if len(cells) != len(col_widths): + return row # Column count mismatch — leave unchanged + new_cells = [] + for cell, width in zip(cells, col_widths): + cell_dw = display_width(cell) + if cell_dw < width: + new_cells.append(cell + " " * (width - cell_dw)) + else: + new_cells.append(cell) + return indent + "│" + "│".join(new_cells) + "│" + + +# --------------------------------------------------------------------------- +# Statistics +# --------------------------------------------------------------------------- + +@dataclass +class Stats: + input_lines: int = 0 + output_lines: int = 0 + user_lines_joined: int = 0 + claude_lines_joined: int = 0 + table_borders_fixed: int = 0 + table_cells_fixed: int = 0 + tool_calls_fixed: int = 0 + tool_results_fixed: int = 0 + agent_tree_fixed: int = 0 + bullet_text_joined: int = 0 + plan_text_joined: int = 0 + table_multirow_merged: int = 0 + table_borders_realigned: int = 0 + box_rows_merged: int = 0 + + def summary(self) -> str: + lines = [ + "--- Statistics ---", + f" Input lines: {self.input_lines}", + f" Output lines: {self.output_lines}", + f" User lines joined: {self.user_lines_joined}", + f" Claude lines joined: {self.claude_lines_joined}", + f" Table borders fixed: {self.table_borders_fixed}", + f" Table cells fixed: {self.table_cells_fixed}", + f" Table rows merged: {self.table_multirow_merged}", + f" Borders realigned: {self.table_borders_realigned}", + f" Box rows merged: {self.box_rows_merged}", + f" Tool calls fixed: {self.tool_calls_fixed}", + f" Tool results fixed: {self.tool_results_fixed}", + f" Agent tree fixed: {self.agent_tree_fixed}", + f" Bullet text joined: {self.bullet_text_joined}", + f" Plan text joined: {self.plan_text_joined}", + ] + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Main processing logic +# --------------------------------------------------------------------------- + + +def process(lines: list[str], stats: Stats) -> list[str]: + """Process all *lines* and return the list of fixed output lines.""" + stats.input_lines = len(lines) + output: list[str] = [] + i = 0 + n = len(lines) + + def peek(offset: int = 1) -> str | None: + idx = i + offset + return lines[idx] if idx < n else None + + while i < n: + line = lines[i] + + # --------------------------------------------------------------- + # 1) User prompt blocks (❯ at column 0, continuations at dw=76) + # --------------------------------------------------------------- + if _USER_PROMPT_RE.match(line): + i = _process_user_block(lines, i, n, output, stats) + continue + + # --------------------------------------------------------------- + # 2) Table border split → join, then enter table region + # --------------------------------------------------------------- + next_line = peek() + if next_line is not None and is_table_border_split(line, next_line): + acc = raw_join(line, next_line) + stats.table_borders_fixed += 1 + i += 2 + while i < n and is_table_border_split(acc, lines[i]): + acc = raw_join(acc, lines[i]) + stats.table_borders_fixed += 1 + i += 1 + output.append(acc) + # If this was a ┌ border, process the full table body + if "┌" in acc: + expected_pipes = _count_expected_pipes(acc) + col_widths = _parse_column_widths(acc) + i = _process_table_body( + lines, i, n, output, stats, expected_pipes, col_widths, + ) + continue + + # --------------------------------------------------------------- + # 2b) Non-split ┌ border → enter table body processor + # --------------------------------------------------------------- + stripped_for_border = line.lstrip() + if ( + stripped_for_border.startswith("┌") + and "─" in stripped_for_border + and "┐" in stripped_for_border + ): + expected_pipes = _count_expected_pipes(line) + col_widths = _parse_column_widths(line) + output.append(line) + i += 1 + i = _process_table_body( + lines, i, n, output, stats, expected_pipes, col_widths, + ) + continue + + # --------------------------------------------------------------- + # 3) Table cell row (│ in line, outside tracked table region) + # Fallback for tables whose ┌ border was not split and was + # already emitted before we entered this logic. + # --------------------------------------------------------------- + if "│" in line: + stripped = line.lstrip() + if stripped.startswith("│") or stripped.endswith("│"): + # Likely a table row. Check if PREVIOUS output line was a + # ┌ or ├ border to determine expected_pipes. + expected_pipes = 0 + fallback_col_widths: list[int] = [] + for prev in reversed(output): + ps = prev.strip() + if ps and (ps[0] in "┌├"): + expected_pipes = _count_expected_pipes(prev) + fallback_col_widths = _parse_column_widths(prev) + break + if ps and ps[0] not in "│": + break + if expected_pipes > 0: + acc = line.rstrip() + pipe_count = acc.count("│") + i += 1 + while pipe_count < expected_pipes and i < n: + nl = lines[i] + if _is_truly_empty(nl): + break + if _is_table_border_line(nl): + break + # Check for border split on this line + if i + 1 < n and is_table_border_split(nl, lines[i + 1]): + break + acc = table_cell_join(acc, nl) + pipe_count = acc.count("│") + stats.table_cells_fixed += 1 + i += 1 + if fallback_col_widths: + acc = _repad_table_row(acc, fallback_col_widths) + output.append(acc) + continue + + # --------------------------------------------------------------- + # 4) Tool call continuation (● Bash(, ● Read(, etc.) + # Continuations are 6-space indented. + # --------------------------------------------------------------- + if _TOOL_CALL_RE.match(line): + acc = line + i += 1 + while i < n: + nl = lines[i] + # 6-space continuation (tool call argument wrapping) + if nl.startswith(" ") and not _is_structural_break(nl): + # Strip the 6-char continuation indent, preserving + # any additional whitespace from the original. + right_part = nl[6:] + left_s = acc.rstrip() + if right_part and right_part[0] == " ": + # Extra space means the original had whitespace + # at this position -- preserve it. + acc = left_s + right_part + elif left_s and _looks_like_mid_word(left_s, right_part): + # Mid-word/mid-path split: concatenate directly. + acc = left_s + right_part + else: + # Argument boundary where the space was consumed + # by wrapping. Restore it. + acc = left_s + " " + right_part + stats.tool_calls_fixed += 1 + i += 1 + else: + break + output.append(acc) + continue + + # --------------------------------------------------------------- + # 5) Tool result continuation ( ⎿ ... at dw>=74) + # --------------------------------------------------------------- + if _TOOL_RESULT_RE.match(line): + last_raw_dw = display_width(line.rstrip()) + acc = line + i += 1 + # Phase 1: join 5-space continuations of the ⎿ line itself. + # Only join when the PREVIOUS raw line was near the wrap limit + # (dw >= 74). Lines well below the limit ended naturally — + # subsequent 5-space lines are separate output lines (e.g. + # git log entries), not wrapped continuations. + # Character-class check: alpha→lower = mid-word (raw join); + # all other transitions = word boundary (smart_join). + while last_raw_dw >= 74 and i < n: + nl = lines[i] + if nl.startswith(" ") and not _is_structural_break(nl) and not _is_plan_structural(nl): + prev_dw = last_raw_dw + last_raw_dw = display_width(nl.rstrip()) + acc = _dw_aware_join(acc, nl, prev_dw) + stats.tool_results_fixed += 1 + i += 1 + else: + break + # Phase 1 fallback: if ⎿ line ends with trailing space + # (word-boundary wrap just below the 74 threshold), join + # short continuation fragments. This catches lines like + # "Plan saved to: ... · /plan to " + "edit" (dw=72). + while i < n and acc != acc.rstrip(): + nl = lines[i] + if nl.startswith(" ") and not nl.startswith(" "): + if _is_plan_continuation_fragment(nl, acc): + # Trailing space = word-boundary wrap → smart_join + # (not _bullet_join, which would raw-join alpha→lower + # like "to" + "edit" → "toedit"). + acc = smart_join(acc, nl) + stats.tool_results_fixed += 1 + i += 1 + continue + break + output.append(acc) + + # Phase 2: handle remaining tool output lines at 6-space indent. + # After the ⎿ line, tool output continues at 6-space indent. + # Each such line may itself be wrapped, with 5-space continuations. + # We emit each output line individually, joining only its + # wrapped fragments. + while i < n: + nl = lines[i] + if _is_truly_empty(nl): + break + if _is_structural_break(nl): + break + # Tool output lines use 6-space indent (5-space lines that + # aren't continuations would be plan text, etc.) + if not nl.startswith(" ") or nl.startswith(" "): + break + acc2 = nl + last_raw_dw2 = display_width(nl.rstrip()) + i += 1 + # Join continuations of this output line. + # Phase 2a: high-dw continuations at 5-6 space indent. + while last_raw_dw2 >= 74 and i < n: + nl2 = lines[i] + if nl2.startswith(" ") and not _is_structural_break(nl2) and not _is_plan_structural(nl2): + prev_dw2 = last_raw_dw2 + last_raw_dw2 = display_width(nl2.rstrip()) + acc2 = _dw_aware_join(acc2, nl2, prev_dw2) + stats.tool_results_fixed += 1 + i += 1 + else: + break + # Phase 2b: deeper-indented continuations when the + # output line ends with a continuation signal: + # - comma (list continuation) + # - trailing space (word-boundary wrap) + # - underscore (identifier split, e.g. E2E_PO_BASE_ + URL) + while i < n: + _acc2_s = acc2.rstrip() + if not _acc2_s: + break + _last2 = _acc2_s[-1] + _has_trailing = acc2 != _acc2_s + if _last2 not in (",", "_") and not _has_trailing: + break # no continuation signal + nl2 = lines[i] + # Only accept DEEPER-indented continuations (7+ spaces). + # Same-indent lines (6 spaces) are sibling entries in the + # tool output — e.g. separate diff lines that happen to + # have trailing padding spaces. + if not nl2.startswith(" ") or _is_structural_break(nl2): + break + nl2_dw = display_width(nl2.rstrip()) + if nl2_dw >= 76: + break # full-width = independent line + acc2 = smart_join(acc2, nl2) + last_raw_dw2 = nl2_dw + stats.tool_results_fixed += 1 + i += 1 + output.append(acc2) + continue + + # --------------------------------------------------------------- + # 6) Agent tree continuation (├─ or └─ at 3-space indent) + # --------------------------------------------------------------- + if _AGENT_TREE_RE.match(line): + dw = display_width(line.rstrip()) + acc = line + i += 1 + if dw >= 70: + while i < n: + nl = lines[i] + if _is_structural_break(nl): + break + nl_stripped = nl.lstrip() + # Never join lines that are sub-results (contain ⎿) + # or new tree nodes (├─, └─, │) + if "⎿" in nl_stripped: + break + if nl_stripped.startswith("├─") or nl_stripped.startswith("└─"): + break + if nl_stripped.startswith("│"): + break + # Same-indent non-structural continuation + if nl.startswith(" "): + acc = raw_join(acc, nl) + stats.agent_tree_fixed += 1 + i += 1 + else: + break + output.append(acc) + continue + + # --------------------------------------------------------------- + # 7) Claude narrative (● text with dw>=55+, NOT tool call/short marker) + # Threshold lowered from 77 to 55 because CJK-heavy lines end + # content well before the 77-80 column wrap limit (CJK chars take + # 2 columns each, so word boundaries fall earlier). + # --------------------------------------------------------------- + if _CLAUDE_ACTION_RE.match(line) and not _TOOL_CALL_RE.match(line): + dw = display_width(line.rstrip()) + acc = line + i += 1 + if dw >= 55 or _has_continuation_signal(line): + while i < n: + nl = lines[i] + if _is_structural_break(nl): + break + nl_dw = display_width(nl.rstrip()) + # 2-space continuation, short, not structural + if nl.startswith(" ") and not nl.startswith(" ⎿") and nl_dw < 82: + nl_stripped = nl.lstrip() + if nl_stripped.startswith("- "): + break + if re.match(r"\d+\. ", nl_stripped): + break + acc = smart_join(acc, nl) + stats.claude_lines_joined += 1 + i += 1 + else: + break + output.append(acc) + continue + + # --------------------------------------------------------------- + # 7b) Bullet item in Claude response ( - text) + # When a bullet line has high dw or ends with a continuation + # signal (CJK char, comma, etc.), its wrapped continuation + # on the next line (2-space indent, NOT a bullet/numbered) + # must be joined. Uses _bullet_join to handle mid-word + # breaks (e.g. "RiskModelAss" + "ignment") correctly. + # --------------------------------------------------------------- + if _BULLET_RE.match(line): + dw = display_width(line.rstrip()) + acc = line + i += 1 + if dw >= 55 or _has_continuation_signal(line) or _has_unclosed_bracket(line): + while i < n: + nl = lines[i] + if not _is_continuation_fragment(nl, acc): + break + acc = _bullet_join(acc, nl) + stats.bullet_text_joined += 1 + i += 1 + # Peek-ahead for plan-context bullets (5+ space indent). + # Short plan bullets (dw < 55) whose text wraps to the base + # 5-space indent won't enter the join loop above. Check if + # the next line is a non-structural 5-space continuation. + elif line.startswith(" ") and i < n: + nl = lines[i] + if ( + nl.startswith(" ") + and not _is_structural_break(nl) + and not _is_plan_structural(nl) + ): + nl_dw = display_width(nl.rstrip()) + if nl_dw < 70: + acc = smart_join(acc, nl) + stats.bullet_text_joined += 1 + i += 1 + output.append(acc) + continue + + # --------------------------------------------------------------- + # 8) Numbered list item in Claude response ( N. text) + # --------------------------------------------------------------- + if _NUMBERED_RE.match(line): + dw = display_width(line.rstrip()) + acc = line + i += 1 + if dw >= 55 or _has_continuation_signal(line): + while i < n: + nl = lines[i] + if _is_structural_break(nl): + break + # Numbered list continuation is at 2-space indent + if nl.startswith(" ") and not nl.startswith(" ⎿"): + nl_stripped = nl.lstrip() + if nl_stripped.startswith("- "): + break + if re.match(r"\d+\. ", nl_stripped): + break + dw_nl = display_width(nl.rstrip()) + if dw_nl < 82: + acc = _bullet_join(acc, nl) + stats.claude_lines_joined += 1 + i += 1 + else: + break + else: + break + # Peek-ahead for plan-context numbered items (5+ space indent). + elif line.startswith(" ") and i < n: + nl = lines[i] + if ( + nl.startswith(" ") + and not _is_structural_break(nl) + and not _is_plan_structural(nl) + ): + nl_dw = display_width(nl.rstrip()) + if nl_dw < 70: + acc = _bullet_join(acc, nl) + stats.claude_lines_joined += 1 + i += 1 + output.append(acc) + continue + + # --------------------------------------------------------------- + # 8c) Claude paragraph text (2-space indent, standalone) + # Text paragraphs within Claude response blocks that aren't + # preceded by a ● marker — they appear after tables, bullet + # lists, or between structural elements. + # + # DESIGN: Instead of measuring the current line's dw to guess + # whether it was wrapped (fragile threshold), we examine the + # NEXT line via _is_continuation_fragment() to decide whether + # it is a new item or a continuation of the current paragraph. + # --------------------------------------------------------------- + if line.startswith(" ") and not line.startswith(" "): + acc = line + i += 1 + # Skip column-layout lines (side-by-side comparison format). + _content = line.strip() + if " " not in _content: # 8+ internal spaces = layout + while i < n: + nl = lines[i] + if _is_continuation_fragment(nl, acc): + acc = _bullet_join(acc, nl) + stats.claude_lines_joined += 1 + i += 1 + else: + break + output.append(acc) + continue + + # --------------------------------------------------------------- + # 9) Plan / indented text (5+ space indent) + # + # DESIGN: Like step 8c, uses next-line look-ahead via + # _is_plan_continuation_fragment() instead of dw thresholds. + # The join function (_dw_aware_join) still uses the last + # segment's dw to decide mid-word vs word-boundary joins. + # --------------------------------------------------------------- + if line.startswith(" "): + acc = line + last_seg_dw = display_width(line.rstrip()) + i += 1 + while i < n: + nl = lines[i] + if _is_plan_continuation_fragment(nl, acc): + prev_dw = last_seg_dw + last_seg_dw = display_width(nl.rstrip()) + acc = _dw_aware_join(acc, nl, prev_dw) + stats.plan_text_joined += 1 + i += 1 + else: + break + output.append(acc) + continue + + # --------------------------------------------------------------- + # 10) Default: emit as-is + # --------------------------------------------------------------- + output.append(line) + i += 1 + + # Post-processing: merge multi-row table cells. + # After the main loop, each physical data row is complete (correct pipe + # count, re-padded). But a single logical row may still span multiple + # physical rows when Claude wrapped cell content at column width. + # This pass collapses those into one row per logical cell. + output = _merge_multirow_table_cells(output, stats) + + # Post-processing: realign table borders. + # After merging, merged data rows may exceed original column widths. + # This pass recalculates border widths to match the widest data cells. + output = _realign_table_borders(output, stats) + + # Post-processing: merge wrapped text within single-column box items. + # Single-column boxes (exactly 2 │ per row) contain numbered/bulleted + # items that may wrap across multiple rows. This pass merges continuation + # lines back into their parent item. + output = _merge_singlecol_box_rows(output, stats) + + stats.output_lines = len(output) + return output + + +# --------------------------------------------------------------------------- +# Post-processing: multi-row table cell merge +# --------------------------------------------------------------------------- + +def _is_table_data_row(stripped: str) -> bool: + """Check if *stripped* (leading-whitespace-removed) is a multi-column table data row. + + A data row starts and ends with ``│`` and contains no ``─`` (which + would make it a border row). Requires at least 3 ``│`` (i.e. 2+ + columns) so that single-column boxes (which have only 2 ``│``) + are excluded — their rows are independent items, not wrapped cells. + """ + return ( + len(stripped) >= 5 + and stripped[0] == "│" + and stripped[-1] == "│" + and "─" not in stripped + and stripped.count("│") >= 3 + ) + + +def _merge_multirow_table_cells(lines: list[str], stats: Stats) -> list[str]: + """Collapse consecutive table data rows into single logical rows. + + Between border/separator rows (``┌├└``), Claude may emit multiple + physical data rows for one logical row when cell content exceeds the + column width. This function detects such groups and merges them, + joining cell content with ``_table_cell_content_join`` which handles + mid-word and CJK boundaries. + """ + result: list[str] = [] + i = 0 + n = len(lines) + + while i < n: + line = lines[i] + stripped = line.lstrip() + + if not _is_table_data_row(stripped): + result.append(line) + i += 1 + continue + + # Collect consecutive data rows (same table region). + group = [line] + j = i + 1 + while j < n: + next_stripped = lines[j].lstrip() + if _is_table_data_row(next_stripped): + group.append(lines[j]) + j += 1 + else: + break + + if len(group) == 1: + result.append(line) + i = j + continue + + # Multiple physical rows → merge into one logical row. + merged = _merge_row_group(group, stats) + result.append(merged) + i = j + + return result + + +def _merge_row_group(rows: list[str], stats: Stats) -> str: + """Merge a group of physical table data rows into one logical row.""" + first = rows[0] + indent = first[: len(first) - len(first.lstrip())] + + # Split each row into cell contents — keep both raw and stripped forms. + # The raw form preserves trailing spaces, which we use to detect whether + # the content filled the entire column (≤1 trailing space → mid-word split). + all_cells: list[list[str]] = [] + all_raw: list[list[str]] = [] + for row in rows: + parts = row.strip().split("│") + # parts[0] is empty (before first │), parts[-1] is empty (after last │) + raw_cells = parts[1:-1] + cells = [c.strip() for c in raw_cells] + all_cells.append(cells) + all_raw.append(raw_cells) + + num_cols = max(len(cells) for cells in all_cells) + + # Merge each column's fragments. + merged_cells: list[str] = [] + for col_idx in range(num_cols): + fragments: list[str] = [] + raw_fragments: list[str] = [] + for row_idx, row_cells in enumerate(all_cells): + if col_idx < len(row_cells) and row_cells[col_idx]: + fragments.append(row_cells[col_idx]) + raw_fragments.append(all_raw[row_idx][col_idx]) + + if not fragments: + merged_cells.append("") + elif len(fragments) == 1: + merged_cells.append(fragments[0]) + else: + acc = fragments[0] + for k in range(1, len(fragments)): + # Determine if the previous fragment filled its column. + prev_raw = raw_fragments[k - 1] + trailing_spaces = len(prev_raw) - len(prev_raw.rstrip()) + left_filled = trailing_spaces <= 1 + acc = _table_cell_content_join( + acc, fragments[k], left_filled=left_filled + ) + merged_cells.append(acc) + + # Reconstruct the row with 1-space padding per cell. + cell_parts = [f" {cell} " if cell else " " for cell in merged_cells] + merged = indent + "│" + "│".join(cell_parts) + "│" + + stats.table_multirow_merged += len(rows) - 1 + return merged + + +# --------------------------------------------------------------------------- +# Post-processing: merge wrapped text within single-column box items +# --------------------------------------------------------------------------- + +def _is_singlecol_data_row(line: str) -> bool: + """Check if *line* is a single-column box data row. + + A single-column data row starts and ends with ``│``, has exactly 2 + ``│`` characters, and contains no ``─`` (which would indicate a border). + """ + stripped = line.lstrip() + return ( + len(stripped) >= 3 + and stripped[0] == "│" + and stripped[-1] == "│" + and "─" not in stripped + and stripped.count("│") == 2 + ) + + +def _is_singlecol_border(line: str) -> bool: + """Check if *line* is a single-column box border (┌─┐, ├─┤, or └─┘).""" + stripped = line.lstrip() + if not stripped: + return False + return ( + stripped[0] in "┌├└" + and "─" in stripped + and stripped[-1] in "┐┤┘" + and "┬" not in stripped + and "┼" not in stripped + and "┴" not in stripped + ) + + +_ITEM_START_RE = re.compile(r"^\s*\d+\.\s") +_BULLET_START_RE = re.compile(r"^\s*[-*]\s") + + +def _merge_singlecol_box_rows(lines: list[str], stats: Stats) -> list[str]: + """Merge wrapped text within single-column box items. + + Single-column boxes (2 ``│`` per row) contain numbered/bulleted items + that may wrap across multiple rows. This function merges continuation + lines back into their parent item while keeping separate items on + separate rows. + + Title boxes (a single data row between borders) are left untouched. + """ + result: list[str] = [] + i = 0 + n = len(lines) + + while i < n: + line = lines[i] + + # Look for the start of a single-column box (┌ border with no ┬). + if not _is_singlecol_border(line) or not line.lstrip().startswith("┌"): + result.append(line) + i += 1 + continue + + # Found a ┌ border. Collect the entire box (border + data + └). + box_lines: list[str] = [line] + j = i + 1 + found_close = False + while j < n: + if _is_singlecol_data_row(lines[j]): + box_lines.append(lines[j]) + j += 1 + elif _is_singlecol_border(lines[j]): + box_lines.append(lines[j]) + if lines[j].lstrip().startswith("└"): + found_close = True + j += 1 + break + elif lines[j].lstrip().startswith("├"): + j += 1 + else: + j += 1 + break + else: + break # Non-box line — box ended unexpectedly + + if not found_close: + # Incomplete box — emit as-is. + for bl in box_lines: + result.append(bl) + i = j + continue + + # Extract data rows (skip borders). + data_indices = [ + idx for idx, bl in enumerate(box_lines) if _is_singlecol_data_row(bl) + ] + + # Title boxes: single data row between borders → skip merging. + if len(data_indices) <= 1: + for bl in box_lines: + result.append(bl) + i = j + continue + + # Group data rows into logical items and merge continuations. + data_rows = [box_lines[idx] for idx in data_indices] + merged_contents = _merge_box_items(data_rows, stats) + + # Determine whether box borders need to grow. + indent = line[: len(line) - len(line.lstrip())] + + # Get current box inner width from the ┌ border. + top_border = line.lstrip() + border_inner_dw = display_width(top_border) - 2 + + # Compute max content width after merge. + max_content_dw = 0 + for content in merged_contents: + # Content needs 2 spaces padding (1 left + 1 right minimum). + content_dw = display_width(content) + 2 + max_content_dw = max(max_content_dw, content_dw) + + new_inner_width = max(border_inner_dw, max_content_dw) + + # Rebuild the box: borders + merged data rows. + # Emit ┌ border (potentially wider). + result.append( + _rebuild_singlecol_border(top_border, new_inner_width, indent) + ) + + # Emit merged data rows. + for content in merged_contents: + padded = _pad_singlecol_content( + " " + content + " ", new_inner_width, + ) + result.append(indent + "│" + padded + "│") + + # Emit any ├ and └ borders (potentially wider). + for idx, bl in enumerate(box_lines): + if idx == 0: + continue # Already emitted ┌ + stripped_bl = bl.lstrip() + if stripped_bl and stripped_bl[0] in "├└" and "─" in stripped_bl: + result.append( + _rebuild_singlecol_border(stripped_bl, new_inner_width, indent) + ) + + i = j + + return result + + +def _merge_box_items( + data_rows: list[str], stats: Stats, +) -> list[str]: + """Merge continuation rows within each logical item of a single-column box. + + Returns a list of merged content strings (one per logical item). + """ + # Parse content from each row (strip │ and whitespace). + contents: list[str] = [] + for row in data_rows: + stripped = row.lstrip() + inner = stripped[1:-1] # Remove │ ... │ + contents.append(inner.strip()) + + # Group into logical items. + items: list[list[str]] = [] + for content in contents: + if _ITEM_START_RE.match(content) or _BULLET_START_RE.match(content): + items.append([content]) + elif not items: + items.append([content]) + else: + items[-1].append(content) + + # Merge fragments within each item. + merged: list[str] = [] + for fragments in items: + if len(fragments) == 1: + merged.append(fragments[0]) + else: + acc = fragments[0] + for frag in fragments[1:]: + acc = _table_cell_content_join(acc, frag) + stats.box_rows_merged += 1 + merged.append(acc) + + return merged + + +def _pad_singlecol_content(content: str, inner_width: int) -> str: + """Pad content to fill *inner_width* display columns inside a box.""" + content_dw = display_width(content) + if content_dw >= inner_width: + return content + return content + " " * (inner_width - content_dw) + + +def _rebuild_singlecol_border( + stripped: str, inner_width: int, indent: str, +) -> str: + """Rebuild a single-column box border to the given inner width. + + Preserves styled headers like ``┌─── Title ───┐`` by detecting + embedded text and re-centering it. + """ + left_corner = stripped[0] + right_corner = stripped[-1] + + # Check for embedded title text (e.g., ┌─── Pre-Suite ───┐). + inner = stripped[1:-1] + title_match = re.search(r"([^─]+)", inner) + if title_match: + title_text = title_match.group(1).strip() + if title_text: + # Styled header border: ─── Title ─── + title_with_spaces = f" {title_text} " + title_dw = display_width(title_with_spaces) + remaining = inner_width - title_dw + left_dashes = max(remaining // 2, 3) + right_dashes = max(remaining - left_dashes, 3) + return ( + indent + + left_corner + + "─" * left_dashes + + title_with_spaces + + "─" * right_dashes + + right_corner + ) + + # Plain border (all ─). + return indent + left_corner + "─" * inner_width + right_corner + + +# --------------------------------------------------------------------------- +# Post-processing: realign table borders after multi-row merge +# --------------------------------------------------------------------------- + +def _realign_table_borders(lines: list[str], stats: Stats) -> list[str]: + """Recalculate border widths to match (potentially wider) merged data rows. + + After ``_merge_multirow_table_cells`` collapses multi-row cells, the + merged content may exceed the original column widths encoded in the + border rows. This pass: + + 1. Identifies contiguous table *regions* (runs of border + data rows). + 2. For each region, computes the maximum display-width per column + across all data rows. + 3. Regenerates every border row with the correct ``─`` widths. + 4. Re-pads every data row so cells align with the new borders. + """ + result: list[str] = [] + i = 0 + n = len(lines) + + while i < n: + line = lines[i] + stripped = line.lstrip() + + # Detect start of a table region (┌ border). + if stripped.startswith("┌") and "─" in stripped and "┐" in stripped: + # Collect every line in this table region. + region_start = i + region: list[str] = [line] + i += 1 + while i < n: + s = lines[i].lstrip() + if _is_table_data_row(s): + region.append(lines[i]) + i += 1 + elif s and s[0] in "├└" and "─" in s: + region.append(lines[i]) + i += 1 + if s[0] == "└": + break # End of table + else: + break # Non-table line — region ended unexpectedly + + realigned = _realign_table_region(region, stats) + result.extend(realigned) + continue + + result.append(line) + i += 1 + + return result + + +def _realign_table_region(region: list[str], stats: Stats) -> list[str]: + """Realign borders and data rows within a single table region.""" + # Separate border and data rows; determine indent from first line. + first = region[0] + indent = first[: len(first) - len(first.lstrip())] + + # Collect data rows and their per-cell display widths. + data_rows: list[tuple[int, list[str]]] = [] # (index, cells) + border_indices: list[int] = [] + + for idx, row in enumerate(region): + stripped = row.lstrip() + if _is_table_data_row(stripped): + parts = stripped.split("│") + # parts[0] = '' (before first │), parts[-1] = '' (after last │) + cells = parts[1:-1] + data_rows.append((idx, cells)) + elif stripped and stripped[0] in "┌├└": + border_indices.append(idx) + + if not data_rows: + return region # No data rows — nothing to realign + + # Determine column count from data rows. + num_cols = max(len(cells) for _, cells in data_rows) + if num_cols == 0: + return region + + # Compute max display-width per column across all data rows. + # Each cell has 1-space padding on each side, so content width is what + # we see between the padding. But we measure the full cell (including + # padding) to get the column width that the border must span. + max_widths: list[int] = [0] * num_cols + for _, cells in data_rows: + for col_idx, cell in enumerate(cells): + if col_idx < num_cols: + cw = display_width(cell) + if cw > max_widths[col_idx]: + max_widths[col_idx] = cw + + # Ensure minimum width of 3 (1 space + 1 char + 1 space). + max_widths = [max(w, 3) for w in max_widths] + + # Check if any realignment is needed by comparing with current border. + current_widths = _parse_column_widths(region[border_indices[0]]) + if current_widths == max_widths: + return region # Already aligned + + # Rebuild the region. + rebuilt: list[str] = [] + for idx, row in enumerate(region): + stripped = row.lstrip() + if idx in border_indices: + rebuilt.append(_rebuild_border(stripped[0], stripped[-1], max_widths, indent)) + stats.table_borders_realigned += 1 + elif _is_table_data_row(stripped): + rebuilt.append(_repad_table_row_to_widths(row, max_widths, indent)) + else: + rebuilt.append(row) + + return rebuilt + + +def _rebuild_border( + left_corner: str, right_corner: str, col_widths: list[int], indent: str, +) -> str: + """Build a border row from corner chars and column widths. + + Maps corner pairs: + ┌ ┐ → separator ┬ + ├ ┤ → separator ┼ + └ ┘ → separator ┴ + """ + sep_map = {"┌": "┬", "├": "┼", "└": "┴"} + separator = sep_map.get(left_corner, "┼") + segments = ["─" * w for w in col_widths] + return indent + left_corner + separator.join(segments) + right_corner + + +def _repad_table_row_to_widths( + row: str, col_widths: list[int], indent: str, +) -> str: + """Re-pad a data row so each cell matches the given column widths.""" + stripped = row.lstrip() + parts = stripped.split("│") + if len(parts) < 3: + return row + cells = parts[1:-1] + if len(cells) != len(col_widths): + return row # Column count mismatch — leave unchanged + + new_cells: list[str] = [] + for cell, width in zip(cells, col_widths): + # Strip existing padding, then re-pad. + content = cell.strip() + content_dw = display_width(content) + # Target: 1 space left + content + right padding to fill width. + # Total cell display-width must equal `width`. + # Cell = " " + content + " " * (width - 1 - content_dw) + # But if content_dw + 2 > width, just use " content " (overflow). + if content: + right_pad = max(width - 1 - content_dw, 1) + new_cells.append(" " + content + " " * right_pad) + else: + new_cells.append(" " * width) + + return indent + "│" + "│".join(new_cells) + "│" + + +def _process_table_body( + lines: list[str], + start: int, + n: int, + output: list[str], + stats: Stats, + expected_pipes: int, + col_widths: list[int] | None = None, +) -> int: + """Process lines inside a table body (after ┌ border, until └ border). + + Uses pipe-count accumulation: each data row is accumulated until the + ``│`` count reaches *expected_pipes*, then emitted. Border lines + (├─, └─) are emitted directly (with split joining if needed). + + When *col_widths* is provided, each completed data row is re-padded + so every cell matches the column width from the border. + + Returns the next line index to process after the table ends. + """ + i = start + while i < n: + line = lines[i] + + # --- Empty line: table ended unexpectedly --- + if _is_truly_empty(line): + break + + # --- Border line (├ or └): join splits, emit, maybe exit --- + stripped = line.lstrip() + if stripped and stripped[0] in "├└" and "─" in stripped: + acc = line + i += 1 + # Join border split if needed + if acc.rstrip()[-1] == "─" and i < n: + nl = lines[i] + ns = nl.lstrip() + if ns and ns[0] == "─" and ns.rstrip()[-1] in _TABLE_CORNERS: + acc = raw_join(acc, nl) + stats.table_borders_fixed += 1 + i += 1 + while i < n and is_table_border_split(acc, lines[i]): + acc = raw_join(acc, lines[i]) + stats.table_borders_fixed += 1 + i += 1 + output.append(acc) + # └ border: table ends + if "└" in acc: + return i + continue + + # --- Data row: accumulate until pipe count matches --- + if "│" in line or (stripped and stripped[-1] == "│"): + acc = line.rstrip() + pipe_count = acc.count("│") + i += 1 + while pipe_count < expected_pipes and i < n: + nl = lines[i] + if _is_truly_empty(nl): + break + # Stop at border lines + nl_s = nl.lstrip() + if nl_s and nl_s[0] in "├└┌" and "─" in nl_s: + break + acc = table_cell_join(acc, nl) + pipe_count = acc.count("│") + stats.table_cells_fixed += 1 + i += 1 + # Re-pad cells to match column widths from border + if col_widths: + acc = _repad_table_row(acc, col_widths) + output.append(acc) + continue + + # --- Non-table line (shouldn't happen but be safe) --- + output.append(line) + i += 1 + + return i + + +def _process_user_block( + lines: list[str], + start: int, + n: int, + output: list[str], + stats: Stats, +) -> int: + """Process a user prompt block starting at *start*. + + User blocks begin with ``❯ `` and continue with lines that have + display_width == 76 (right-padded with trailing spaces). All content + lines in the block share this fixed width. + + Within the block: + - Blank lines (76 spaces) are paragraph separators. + - Lines whose stripped content starts with ``- `` are bullet items. + - Lines whose stripped content starts with ``\\d+. `` are numbered items. + - ````` ``` ````` toggles code-fence mode (content preserved as-is). + - Everything else is a continuation of the current paragraph/item. + + Returns the index of the first line AFTER the block. + """ + acc = lines[start] + i = start + 1 + + # Check if the first line itself is at dw=76 (padded). + first_dw = display_width(lines[start]) + if first_dw != 76: + # Short user prompt, no wrapping. Emit as-is. + output.append(acc) + return i + + in_code_fence = False + + while i < n: + line = lines[i] + dw = display_width(line) + + # The user block boundary: ALL lines inside have dw==76. + if dw != 76: + break + + stripped = line.rstrip() + + # --- Code fence toggle --- + # Detect ``` anywhere in the stripped content. + if stripped.lstrip().startswith("```"): + # Flush current accumulator before the fence marker. + if acc: + output.append(acc) + acc = "" + output.append(line) + in_code_fence = not in_code_fence + i += 1 + continue + + # Inside code fences: emit lines as-is (no joining). + if in_code_fence: + output.append(line) + i += 1 + continue + + # --- Blank line (paragraph separator) --- + if not stripped: + if acc: + output.append(acc) + acc = "" + output.append("") + i += 1 + continue + + # --- Structural item detection --- + content_no_indent = stripped.lstrip() + + # Bullet: stripped content starts with "- " + is_bullet = content_no_indent.startswith("- ") + + # Numbered list: stripped content starts with digit+". " + is_numbered = bool(re.match(r"\d+\. ", content_no_indent)) + + if is_bullet or is_numbered: + # Start a new logical line for this item. + if acc: + output.append(acc) + acc = line + i += 1 + continue + + # --- Normal continuation --- + if acc: + # Extract the content portion (skip the 2-space continuation + # indent that the export prepends). + join_content = stripped[2:] if stripped.startswith(" ") else stripped + acc = smart_join(acc, join_content) + stats.user_lines_joined += 1 + else: + # After a paragraph break (acc was reset to ""), this is the + # first line of a new paragraph. + acc = line + i += 1 + + # Flush remaining accumulated text. + if acc: + output.append(acc) + + return i + + +# --------------------------------------------------------------------------- +# Marker-count verification +# --------------------------------------------------------------------------- + +def _count_markers(lines: list[str]) -> dict[str, int]: + """Count occurrences of structural markers.""" + counts: dict[str, int] = {"●": 0, "❯": 0, "✻": 0} + for line in lines: + for marker in counts: + if line.startswith(marker): + counts[marker] += 1 + return counts + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + parser = argparse.ArgumentParser( + description="Fix broken line wrapping in Claude Code export files.", + ) + parser.add_argument("input", type=Path, help="Input .txt file") + parser.add_argument( + "-o", + "--output", + type=Path, + default=None, + help="Output path (default: -fixed.txt)", + ) + parser.add_argument( + "--stats", action="store_true", help="Print statistics to stderr" + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Process without writing output file", + ) + args = parser.parse_args() + + input_path: Path = args.input.resolve() + if not input_path.is_file(): + print(f"ERROR: Input file not found: {input_path}", file=sys.stderr) + sys.exit(1) + + if args.output is not None: + output_path: Path = args.output.resolve() + else: + output_path = input_path.with_stem(input_path.stem + "-fixed") + + if output_path == input_path: + print( + "ERROR: Output path must differ from input path. " + "Use -o to specify a different output file.", + file=sys.stderr, + ) + sys.exit(1) + + # Read input (strict UTF-8, no fallback). + text = input_path.read_text(encoding="utf-8", errors="strict") + raw_lines = text.split("\n") + # Remove trailing empty element from final newline (if present). + if raw_lines and raw_lines[-1] == "": + raw_lines.pop() + + stats = Stats() + result = process(raw_lines, stats) + + # --------------------------------------------------------------- + # Safety: marker-count verification + # --------------------------------------------------------------- + input_markers = _count_markers(raw_lines) + output_markers = _count_markers(result) + for marker, in_count in input_markers.items(): + out_count = output_markers.get(marker, 0) + if in_count != out_count: + print( + f"WARNING: Marker '{marker}' count mismatch: " + f"input={in_count}, output={out_count}", + file=sys.stderr, + ) + + # Safety: runaway join detection + for idx, line in enumerate(result): + if display_width(line) > 500: + print( + f"WARNING: Line {idx + 1} has display width " + f"{display_width(line)} (>500) — possible runaway join", + file=sys.stderr, + ) + + # --------------------------------------------------------------- + # Output + # --------------------------------------------------------------- + if args.dry_run: + print( + f"Dry run complete. Would write {len(result)} lines to {output_path}", + file=sys.stderr, + ) + else: + output_path.write_text( + "\n".join(result) + "\n", encoding="utf-8", errors="strict" + ) + print(f"Written: {output_path}", file=sys.stderr) + + if args.stats: + print(stats.summary(), file=sys.stderr) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/daymade-claude-code/claude-export-txt-better/scripts/validate-claude-export-fix.py b/daymade-claude-code/claude-export-txt-better/scripts/validate-claude-export-fix.py new file mode 100644 index 00000000..553ee70c --- /dev/null +++ b/daymade-claude-code/claude-export-txt-better/scripts/validate-claude-export-fix.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 +"""Automated validation for fix-claude-export.py output. + +Runs a comprehensive suite of checks against a fixed file and its original, +reporting PASS/FAIL for each with evidence. Designed to be run after every +fix iteration as a quality gate. + +Usage: + uv run scripts/validate-claude-export-fix.py + uv run scripts/validate-claude-export-fix.py --verbose +""" + +from __future__ import annotations + +import argparse +import re +import sys +import unicodedata +from dataclasses import dataclass, field +from pathlib import Path + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +def display_width(s: str) -> int: + return sum(2 if unicodedata.east_asian_width(ch) in ("W", "F") else 1 for ch in s) + + +def is_cjk_ideograph(ch: str) -> bool: + return unicodedata.east_asian_width(ch) in ("W", "F") and unicodedata.category(ch) == "Lo" + + +# --------------------------------------------------------------------------- +# Check infrastructure +# --------------------------------------------------------------------------- + +@dataclass +class CheckResult: + name: str + passed: bool + detail: str + category: str = "" + + +@dataclass +class ValidationReport: + results: list[CheckResult] = field(default_factory=list) + + def add(self, name: str, passed: bool, detail: str, category: str = ""): + self.results.append(CheckResult(name, passed, detail, category)) + + @property + def passed(self) -> int: + return sum(1 for r in self.results if r.passed) + + @property + def failed(self) -> int: + return sum(1 for r in self.results if not r.passed) + + def print_report(self, verbose: bool = False): + cats = {} + for r in self.results: + cats.setdefault(r.category or "General", []).append(r) + + for cat, checks in cats.items(): + print(f"\n{'=' * 60}") + print(f" {cat}") + print(f"{'=' * 60}") + for r in checks: + icon = "✓" if r.passed else "✗" + print(f" {icon} {r.name}") + if verbose or not r.passed: + for line in r.detail.split("\n"): + print(f" {line}") + + total = len(self.results) + print(f"\n{'=' * 60}") + print(f" TOTAL: {self.passed}/{total} passed, {self.failed} failed") + print(f"{'=' * 60}") + + +# --------------------------------------------------------------------------- +# Check implementations +# --------------------------------------------------------------------------- + +def check_marker_counts(orig: str, fixed: str, report: ValidationReport): + """Verify structural markers are preserved exactly.""" + markers = [ + ("❯", "User prompts"), + ("●", "Claude actions"), + ("✻", "Stars/crunched"), + ("⎿", "Tool results"), + ("…", "Expansion indicators"), + ] + for marker, name in markers: + orig_count = orig.count(marker) + fixed_count = fixed.count(marker) + report.add( + f"Marker {marker} ({name}): {orig_count}", + orig_count == fixed_count, + f"orig={orig_count} fixed={fixed_count}", + "Structural Integrity", + ) + + +def check_table_borders(fixed: str, report: ValidationReport): + """Verify table border corners are balanced.""" + for ch, name in [("┌", "top-left"), ("┐", "top-right"), ("┘", "bottom-right")]: + count = fixed.count(ch) + report.add( + f"Table corner {ch} ({name}): {count}", + True, # Just record the count + f"count={count}", + "Structural Integrity", + ) + tl = fixed.count("┌") + tr = fixed.count("┐") + br = fixed.count("┘") + report.add( + "Table corners balanced (┌ = ┐ = ┘)", + tl == tr == br, + f"┌={tl} ┐={tr} ┘={br}", + "Structural Integrity", + ) + + +def check_line_reduction(orig: str, fixed: str, report: ValidationReport): + """Output should have fewer lines than input (joins happened).""" + orig_lines = orig.count("\n") + fixed_lines = fixed.count("\n") + report.add( + f"Line reduction: {orig_lines} → {fixed_lines}", + fixed_lines < orig_lines, + f"delta={orig_lines - fixed_lines} ({(orig_lines - fixed_lines) / orig_lines * 100:.1f}% reduction)", + "Structural Integrity", + ) + + +def check_table_border_completeness(fixed: str, report: ValidationReport): + """Verify table border lines have matching left and right ends.""" + lines = fixed.split("\n") + broken = [] + for i, line in enumerate(lines): + stripped = line.strip() + if not stripped: + continue + # Lines starting with a left border char should have a right border char + if stripped[0] == "┌" and "┐" not in stripped: + broken.append((i + 1, "┌ without ┐", stripped[:80])) + elif stripped[0] == "├" and "┤" not in stripped: + broken.append((i + 1, "├ without ┤", stripped[:80])) + elif stripped[0] == "└" and "┘" not in stripped: + broken.append((i + 1, "└ without ┘", stripped[:80])) + + report.add( + f"Table borders complete: {len(broken)} broken", + len(broken) == 0, + "\n".join(f" L{ln}: {desc}" for ln, desc, _ in broken[:5]) + if broken else "all borders have matching ends", + "Structural Integrity", + ) + + +def check_phase_separation(fixed: str, report: ValidationReport): + """Verify Phase N: items are on separate lines.""" + lines = fixed.split("\n") + multi_phase_lines = [] + for i, line in enumerate(lines): + # Count "Phase N:" occurrences on this line + matches = re.findall(r"Phase \d+:", line) + if len(matches) >= 2: + # Allow pipeline diagrams with arrows (legitimate multi-phase) + if "→" in line: + continue + # Allow status updates like "Phase 3 进度: 3/5" + if line.strip().startswith("●"): + continue + multi_phase_lines.append((i + 1, matches, line[:80])) + + report.add( + "Phase items on separate lines", + len(multi_phase_lines) == 0, + f"{len(multi_phase_lines)} violations" + ( + "\n" + "\n".join(f" L{ln}: {m}" for ln, m, _ in multi_phase_lines[:5]) + if multi_phase_lines else "" + ), + "Over-Join Prevention", + ) + + +def check_runaway_joins(fixed: str, report: ValidationReport): + """Flag lines with very high display width that might be runaway joins.""" + lines = fixed.split("\n") + runaways = [] + for i, line in enumerate(lines): + dw = display_width(line) + if dw > 500: + # Check if it's a legitimate long line (user prompt) + if line.startswith("❯ "): + continue + runaways.append((i + 1, dw, line[:60])) + + report.add( + f"No runaway joins (dw > 500): {len(runaways)} found", + len(runaways) == 0, + "\n".join(f" L{ln}: dw={dw} [{preview}...]" for ln, dw, preview in runaways[:5]) + if runaways else "none", + "Over-Join Prevention", + ) + + +def check_en_cjk_no_space(fixed: str, report: ValidationReport): + """Count remaining EN-CJK adjacency without space (potential pangu misses).""" + # Only check at join boundaries (lines that were modified), not all text + pattern_alnum_cjk = re.compile(r"[a-zA-Z0-9][一-龥]") + pattern_cjk_alnum = re.compile(r"[一-龥][a-zA-Z0-9]") + + violations_ac = len(pattern_alnum_cjk.findall(fixed)) + violations_ca = len(pattern_cjk_alnum.findall(fixed)) + total = violations_ac + violations_ca + + # This is informational — some violations are in original content, code, etc. + report.add( + f"EN-CJK adjacency count: {total}", + True, # Informational + f"ASCII→CJK: {violations_ac}, CJK→ASCII: {violations_ca} (includes original content)", + "Pangu Spacing", + ) + + +def check_diff_lines_separate(fixed: str, report: ValidationReport): + """Verify diff output lines aren't merged (line numbers should be separate).""" + lines = fixed.split("\n") + merged_diffs = [] + for i, line in enumerate(lines): + # Look for two diff line numbers on the same line + matches = re.findall(r"\b(\d{3})\s{2,}[-+]?\s", line) + if len(matches) >= 2: + merged_diffs.append((i + 1, matches, line[:80])) + + report.add( + "Diff lines separate", + len(merged_diffs) == 0, + f"{len(merged_diffs)} violations" + ( + "\n" + "\n".join(f" L{ln}: numbers={m}" for ln, m, _ in merged_diffs[:5]) + if merged_diffs else "" + ), + "Over-Join Prevention", + ) + + +def check_cjk_label_separation(fixed: str, report: ValidationReport): + """Verify CJK label fields (模块:, 输出文件:, 状态:) are on separate lines.""" + lines = fixed.split("\n") + merged_labels = [] + # Pattern: field-label style "Label1: value1 Label2: value2" where + # each label starts at a position that looks like a separate field. + # Only flag when labels are at field boundaries (preceded by whitespace + # or start of line), not mid-sentence. + cjk_field_re = re.compile(r"(?:^|\s)([\u4e00-\u9fff]{1,4}[:：])\s") + for i, line in enumerate(lines): + stripped = line.strip() + matches = cjk_field_re.findall(stripped) + if len(matches) >= 3: # 3+ field labels = likely over-joined fields + merged_labels.append((i + 1, matches, stripped[:80])) + + report.add( + "CJK labels on separate lines", + len(merged_labels) == 0, + f"{len(merged_labels)} violations" + ( + "\n" + "\n".join(f" L{ln}: labels={m}" for ln, m, _ in merged_labels[:5]) + if merged_labels else "" + ), + "Over-Join Prevention", + ) + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("original", type=Path, help="Original exported file") + parser.add_argument("fixed", type=Path, help="Fixed output file") + parser.add_argument("--verbose", "-v", action="store_true", help="Show all details") + args = parser.parse_args() + + orig = args.original.read_text(encoding="utf-8") + fixed = args.fixed.read_text(encoding="utf-8") + + report = ValidationReport() + + # Run all checks + check_marker_counts(orig, fixed, report) + check_table_borders(fixed, report) + check_table_border_completeness(fixed, report) + check_line_reduction(orig, fixed, report) + check_phase_separation(fixed, report) + check_runaway_joins(fixed, report) + check_diff_lines_separate(fixed, report) + check_cjk_label_separation(fixed, report) + check_en_cjk_no_space(fixed, report) + + report.print_report(verbose=args.verbose) + + sys.exit(0 if report.failed == 0 else 1) + + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/claude-md-progressive-disclosurer/.security-scan-passed b/daymade-claude-code/claude-md-progressive-disclosurer/.security-scan-passed similarity index 100% rename from claude-md-progressive-disclosurer/.security-scan-passed rename to daymade-claude-code/claude-md-progressive-disclosurer/.security-scan-passed diff --git a/claude-md-progressive-disclosurer/SKILL.md b/daymade-claude-code/claude-md-progressive-disclosurer/SKILL.md similarity index 100% rename from claude-md-progressive-disclosurer/SKILL.md rename to daymade-claude-code/claude-md-progressive-disclosurer/SKILL.md diff --git a/claude-md-progressive-disclosurer/references/progressive_disclosure_principles.md b/daymade-claude-code/claude-md-progressive-disclosurer/references/progressive_disclosure_principles.md similarity index 100% rename from claude-md-progressive-disclosurer/references/progressive_disclosure_principles.md rename to daymade-claude-code/claude-md-progressive-disclosurer/references/progressive_disclosure_principles.md diff --git a/claude-skills-troubleshooting/SKILL.md b/daymade-claude-code/claude-skills-troubleshooting/SKILL.md similarity index 100% rename from claude-skills-troubleshooting/SKILL.md rename to daymade-claude-code/claude-skills-troubleshooting/SKILL.md diff --git a/claude-skills-troubleshooting/references/architecture.md b/daymade-claude-code/claude-skills-troubleshooting/references/architecture.md similarity index 100% rename from claude-skills-troubleshooting/references/architecture.md rename to daymade-claude-code/claude-skills-troubleshooting/references/architecture.md diff --git a/claude-skills-troubleshooting/references/known_issues.md b/daymade-claude-code/claude-skills-troubleshooting/references/known_issues.md similarity index 100% rename from claude-skills-troubleshooting/references/known_issues.md rename to daymade-claude-code/claude-skills-troubleshooting/references/known_issues.md diff --git a/claude-skills-troubleshooting/scripts/diagnose_plugins.py b/daymade-claude-code/claude-skills-troubleshooting/scripts/diagnose_plugins.py similarity index 100% rename from claude-skills-troubleshooting/scripts/diagnose_plugins.py rename to daymade-claude-code/claude-skills-troubleshooting/scripts/diagnose_plugins.py diff --git a/claude-skills-troubleshooting/scripts/enable_all_plugins.py b/daymade-claude-code/claude-skills-troubleshooting/scripts/enable_all_plugins.py similarity index 100% rename from claude-skills-troubleshooting/scripts/enable_all_plugins.py rename to daymade-claude-code/claude-skills-troubleshooting/scripts/enable_all_plugins.py diff --git a/daymade-claude-code/continue-claude-work/.security-scan-passed b/daymade-claude-code/continue-claude-work/.security-scan-passed new file mode 100644 index 00000000..4e1d205b --- /dev/null +++ b/daymade-claude-code/continue-claude-work/.security-scan-passed @@ -0,0 +1,4 @@ +Security scan passed +Scanned at: 2026-03-18T23:02:18.627209 +Tool: gitleaks + pattern-based validation +Content hash: 62e456422cabfe74e5757a802044dac45d1662341b2e90dc943685db81d8f659 diff --git a/daymade-claude-code/continue-claude-work/SKILL.md b/daymade-claude-code/continue-claude-work/SKILL.md new file mode 100644 index 00000000..0e3edbe5 --- /dev/null +++ b/daymade-claude-code/continue-claude-work/SKILL.md @@ -0,0 +1,141 @@ +--- +name: continue-claude-work +description: Recover actionable context from local `.claude` session artifacts and continue interrupted work without running `claude --resume`. This skill should be used when the user provides a Claude session ID, asks to continue prior work from local history, or wants to inspect `.claude` files before resuming implementation. +argument-hint: [session-id] +--- + +# Continue Claude Work + +## Overview + +Recover actionable context from a prior Claude Code session and continue execution in the current conversation. Use local session files as the source of truth, then continue with concrete edits and checks — not just summarizing. + +**Why this exists instead of `claude --resume`**: `claude --resume` replays the full session transcript into the context window. For long sessions this wastes tokens on resolved issues and stale state. This skill **selectively reconstructs** only actionable context — the latest compact summary, pending work, known errors, and current workspace state — giving a fresh start with prior knowledge. + +## File Structure Reference + +For directory layout, JSONL schemas, and compaction block format, see `references/file_structure.md`. + +## Workflow + +### Step 1: Extract Context (single script call) + +Run the bundled extraction script. It handles session discovery, compact-boundary parsing, noise filtering, and workspace state in one call: + +```bash +# Latest session for current project +python3 scripts/extract_resume_context.py + +# Specific session by ID +python3 scripts/extract_resume_context.py --session + +# Search by topic +python3 scripts/extract_resume_context.py --query "auth feature" + +# List recent sessions +python3 scripts/extract_resume_context.py --list +``` + +The script outputs a structured Markdown **briefing** containing: +- **Session metadata** from `sessions-index.json` +- **Compact summary** — Claude's own distilled summary from the last compaction boundary (highest-signal context) +- **Last user requests** — the most recent explicit asks +- **Last assistant responses** — what was claimed done +- **Errors encountered** — tool failures and error outputs +- **Unresolved tool calls** — indicates interrupted session +- **Subagent workflow state** — which subagents completed, which were interrupted, their last outputs +- **Session end reason** — clean exit, interrupted (ctrl-c), error cascade, or abandoned +- **Files touched** — files created/edited/read during the session +- **MEMORY.md** — persistent cross-session notes +- **Git state** — current status, branch, recent log + +The script automatically skips the currently active session (modified < 60s ago) to avoid self-extraction. + +### Step 2: Branch by Session End Reason + +The briefing includes a **Session end reason**. Use it to choose the right continuation strategy: + +| End Reason | Strategy | +|-----------|----------| +| **Clean exit** | Session completed normally. Read the last user request that was addressed. Continue from pending work if any. | +| **Interrupted** | Tool calls were dispatched but never got results (likely ctrl-c or timeout). Retry the interrupted tool calls or assess whether they are still needed. | +| **Error cascade** | Multiple API errors caused the session to fail. Do not retry blindly — diagnose the root cause first. | +| **Abandoned** | User sent a message but got no response. Treat the last user message as the current request. | + +If the briefing has a **Subagent Workflow** section with interrupted subagents, check what each was doing and whether to retry or skip. + +### Step 3: Reconcile and Continue + +Before making changes: +1. Confirm the current directory matches the session's project. +2. If the git branch has changed from the session's branch, note this and decide whether to switch. +3. Inspect files related to pending work — verify old claims still hold. +4. Do not assume old claims are valid without checking. + +Then: +- Implement the next concrete step aligned with the latest user request. +- Run deterministic verification (tests, type-checks, build). +- If blocked, state the exact blocker and propose one next action. + +### Step 4: Report + +Respond concisely: +- **Context recovered**: which session, key findings from the briefing +- **Work executed**: files changed, commands run, test results +- **Remaining**: pending tasks, if any + +## How the Script Works + +### Compact-Boundary-Aware Extraction + +The script finds the **last** compact boundary in the session JSONL and extracts its summary. This is the single highest-signal piece of context in any long session -- Claude's own distilled understanding of the entire conversation up to that point. For details on compaction format and JSONL schemas, see `references/file_structure.md`. + +### Size-Adaptive Strategy + +| Session size | Strategy | +|-------------|----------| +| Has compactions | Read last compact summary + all post-compact messages | +| < 500 KB, no compactions | Read last 60% of messages | +| 500 KB - 5 MB | Read last 30% of messages | +| > 5 MB | Read last 15% of messages | + +### Subagent Context Extraction + +When a session has subagent directories (`/subagents/`), the script parses each subagent's JSONL to extract agent type, completion status, and last text output. This enables recovery of multi-agent workflows -- e.g., if a 32-subagent evaluation pipeline was interrupted, the briefing shows which agents completed and which need retry. + +### Session End Reason Detection + +The script classifies how the session ended: +- **completed** -- assistant had the last word (clean exit) +- **interrupted** -- unresolved tool calls (ctrl-c or timeout) +- **error_cascade** -- 3+ API errors +- **abandoned** -- user sent a message with no response + +### Noise Filtering + +These message types are skipped (37-53% of lines in real sessions): +- `progress`, `queue-operation`, `file-history-snapshot` -- operational noise +- `api_error`, `turn_duration`, `stop_hook_summary` -- system subtypes +- ``, `` -- filtered from user text extraction + +## Guardrails + +- Do not run `claude --resume` or `claude --continue` — this skill provides context recovery within the current session. +- Do not treat compact summaries as complete truth — they are lossy. Always verify claims against current workspace. +- Do not overwrite unrelated working-tree changes. +- Do not load the full session file into context — always use the script. + +## Limitations + +- Cannot recover sessions whose `.jsonl` files have been deleted from `~/.claude/projects/`. +- Cannot access sessions from other machines (files are local only). +- Edit tool operations show deltas, not full file content — use `claude-code-history-files-finder` for full file recovery. +- Compact summaries are lossy — early conversation details may be missing. +- `sessions-index.json` can be stale (entries pointing to deleted files). The script falls back to filesystem-based discovery. + +## Example Trigger Phrases + +- "continue work from session `abc123-...`" +- "don't resume, just read the .claude files and continue" +- "check what I was working on in the last session and keep going" +- "search my sessions for the PR review work" diff --git a/daymade-claude-code/continue-claude-work/references/file_structure.md b/daymade-claude-code/continue-claude-work/references/file_structure.md new file mode 100644 index 00000000..c4772407 --- /dev/null +++ b/daymade-claude-code/continue-claude-work/references/file_structure.md @@ -0,0 +1,249 @@ +# Claude Code Local File Structure + +Ground-truth reference for `~/.claude/` directory layout and JSONL session format. + +## Directory Layout + +``` +~/.claude/ + projects/ # Per-project session storage (primary) + / + sessions-index.json # Master index of all sessions + .jsonl # Session transcript + / # Session subdirectory (optional) + subagents/ + agent-.meta.json # Agent metadata + agent-.jsonl # Agent transcript + tool-results/ + toolu_.txt # Large tool outputs + memory/ # Persistent memory files (MEMORY.md, etc.) + history.jsonl # Global prompt history (no session IDs) + tasks/ # Task tracking (per-session lock/highwatermark) + plans/ # Plan documents (random-name.md) + debug/ # Per-session debug logs (.txt) + transcripts/ # Global tool operation logs (ses_.jsonl) + file-history/ # File modification backups + todos/ # Todo items +``` + +## Path Normalization + +Project paths are encoded by replacing `/` with `-`: + +| Original | Normalized | +|----------|-----------| +| `/path/to/project` | `-path-to-project` | +| `/another/workspace/app` | `-another-workspace-app` | + +## sessions-index.json Schema + +```json +{ + "version": 1, + "entries": [ + { + "sessionId": "20089b2a-e3dd-48b8-809c-0647128bf3b8", + "fullPath": "~/.claude/projects/-path-to-project/20089b2a-....jsonl", + "fileMtime": 1741327503477, + "firstPrompt": "fix the login bug", + "summary": "Fixed authentication redirect...", + "messageCount": 42, + "created": "2026-03-07T03:25:03.477Z", + "modified": "2026-03-07T12:21:43.806Z", + "gitBranch": "main", + "projectPath": "/path/to/project", + "isSidechain": false + } + ], + "originalPath": "/path/to/project" +} +``` + +Key fields for session identification: +- `sessionId` — UUID v4 format +- `firstPrompt` — first user message (best for topic matching) +- `summary` — auto-generated summary of the session +- `modified` — last activity timestamp (ISO 8601) +- `gitBranch` — git branch at session time +- `isSidechain` — `false` for main conversations + +## Compaction in Session Files + +Claude Code uses [server-side compaction](https://platform.claude.com/docs/en/build-with-claude/compaction). When context fills up, two consecutive lines appear: + +### Line 1: compact_boundary marker + +```json +{ + "type": "system", + "subtype": "compact_boundary", + "parentUuid": null, + "logicalParentUuid": "prev-uuid", + "compactMetadata": { + "trigger": "input_tokens", + "preTokens": 180000 + } +} +``` + +### Line 2: Compact summary (special user message) + +```json +{ + "type": "user", + "isCompactSummary": true, + "isVisibleInTranscriptOnly": true, + "message": { + "role": "user", + "content": "This session is being continued from a previous conversation that ran out of context. The summary below covers the earlier portion of the conversation.\n\nAnalysis:\n1. **Initial Request**: User asked to...\n2. **Progress**: Completed X, Y, Z...\n3. **Current state**: Working on..." + } +} +``` + +Key properties: +- **`isCompactSummary: true`** — most reliable way to identify compact summaries +- **`isVisibleInTranscriptOnly: true`** — not sent to the API, only stored in the transcript +- Summary is always a plain string in `.message.content` (not an array) +- Typically 12K-31K characters (high-density information) +- A long session may have multiple compact boundaries (4+ is common for 10MB+ sessions) +- The **last** compact boundary's summary reflects the most recent state +- Messages after the last boundary are the "hot zone" — they were in Claude's live context + +### Default compaction prompt (from API docs) + +> "You have written a partial transcript for the initial task above. Please write a summary of the transcript. The purpose of this summary is to provide continuity so you can continue to make progress towards solving the task in a future context, where the raw history above may not be accessible and will be replaced with this summary." + +## Session JSONL Message Types + +Each `.jsonl` file has one JSON object per line. Common types: + +### file-history-snapshot (always first line) + +```json +{ + "type": "file-history-snapshot", + "messageId": "uuid", + "snapshot": { "trackedFileBackups": {}, "timestamp": "..." }, + "isSnapshotUpdate": false +} +``` + +### User message + +```json +{ + "parentUuid": "prev-uuid or null", + "isSidechain": false, + "cwd": "/path/to/project", + "sessionId": "session-uuid", + "version": "2.1.71", + "gitBranch": "main", + "type": "user", + "message": { + "role": "user", + "content": "fix the login bug" + }, + "uuid": "msg-uuid", + "timestamp": "2026-03-07T03:25:03.477Z" +} +``` + +**Important**: `.message.content` can be: +- A **string** for plain text user messages +- An **array** of content blocks for tool results and multi-part messages: + ```json + "content": [ + { "type": "tool_result", "tool_use_id": "toolu_...", "content": "..." }, + { "type": "text", "text": "now do X" } + ] + ``` + +### Assistant message + +```json +{ + "type": "assistant", + "message": { + "role": "assistant", + "model": "claude-opus-4-6", + "content": [ + { "type": "thinking", "thinking": "internal reasoning..." }, + { "type": "text", "text": "visible response text" }, + { "type": "tool_use", "id": "toolu_...", "name": "Bash", "input": { "command": "..." } } + ] + } +} +``` + +Content block types in assistant messages: +- `thinking` — internal reasoning (skip when extracting actionable context) +- `text` — visible response to user (extract this) +- `tool_use` — tool invocations (useful for understanding what was done) + +### Tool result (user message with tool output) + +```json +{ + "type": "user", + "message": { + "role": "user", + "content": [ + { "type": "tool_result", "tool_use_id": "toolu_...", "content": "command output..." } + ] + } +} +``` + +## history.jsonl Schema + +Global prompt log. Does NOT contain session IDs — only useful for finding when a prompt was issued and in which project: + +```json +{ + "display": "/init ", + "pastedContents": {}, + "timestamp": 1758996122446, + "project": "/path/to/project" +} +``` + +## Extraction Patterns + +### List recent sessions for current project + +```bash +cat ~/.claude/projects//sessions-index.json \ + | jq '.entries | sort_by(.modified) | reverse | .[:5] | .[] | {sessionId, firstPrompt, summary, messageCount, modified, gitBranch}' +``` + +### Extract user text from session tail + +Handles both string and array content formats: + +```bash +tail -n 200 .jsonl \ + | jq -r 'select(.type == "user" and .message.role == "user") + | .message.content + | if type == "string" then . + elif type == "array" then map(select(.type == "text") | .text) | join("\n") + else empty end' \ + | tail -n 80 +``` + +### Extract assistant text (excluding thinking/tool_use) + +```bash +tail -n 200 .jsonl \ + | jq -r 'select(.message.role == "assistant") + | .message.content + | if type == "array" then map(select(.type == "text") | .text) | join("\n") + else empty end' \ + | tail -n 120 +``` + +### Find sessions by keyword in firstPrompt + +```bash +cat ~/.claude/projects//sessions-index.json \ + | jq -r '.entries[] | select(.firstPrompt | test("keyword"; "i")) | "\(.modified) \(.sessionId) \(.firstPrompt)"' +``` diff --git a/daymade-claude-code/continue-claude-work/scripts/extract_resume_context.py b/daymade-claude-code/continue-claude-work/scripts/extract_resume_context.py new file mode 100755 index 00000000..fad677bc --- /dev/null +++ b/daymade-claude-code/continue-claude-work/scripts/extract_resume_context.py @@ -0,0 +1,841 @@ +#!/usr/bin/env python3 +""" +Extract actionable resume context from Claude Code session files. + +Produces a structured Markdown briefing by fusing: +- Session index metadata (sessions-index.json) +- Compact boundary summaries (highest-signal context) +- Post-compact user/assistant messages (the "hot zone") +- Subagent workflow state (multi-agent recovery) +- Session end reason detection +- Git workspace state +- MEMORY.md persistent context +- Interrupted tool-call detection + +Usage: + # Extract context from latest session for current project + python3 extract_resume_context.py + + # Extract context from a specific session + python3 extract_resume_context.py --session + + # Search sessions by topic + python3 extract_resume_context.py --query "auth feature" + + # List recent sessions + python3 extract_resume_context.py --list + + # Specify project path explicitly + python3 extract_resume_context.py --project /path/to/project +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import time +from pathlib import Path +from typing import Dict, List, Optional + +CLAUDE_DIR = Path.home() / ".claude" +PROJECTS_DIR = CLAUDE_DIR / "projects" + +# Message types that are noise — skip when extracting context +NOISE_TYPES = {"progress", "queue-operation", "file-history-snapshot", "last-prompt"} +# System message subtypes that are noise +NOISE_SUBTYPES = {"api_error", "turn_duration", "stop_hook_summary"} + +# Patterns that indicate system/internal content, not real user requests +NOISE_USER_PATTERNS = [ + "This session is being continued", + "", + "", +] + + +def normalize_path(project_path: str) -> str: + """Convert absolute path to Claude's normalized directory name.""" + return project_path.replace("/", "-") + + +def find_project_dir(project_path: str) -> Optional[Path]: + """Find the Claude projects directory for a given project path.""" + abs_path = os.path.abspath(project_path) + + # If the path is already inside ~/.claude/projects/, use it directly + projects_str = str(PROJECTS_DIR) + "/" + if abs_path.startswith(projects_str): + candidate = Path(abs_path) + if candidate.is_dir(): + return candidate + rel = abs_path[len(projects_str):] + top_dir = PROJECTS_DIR / rel.split("/")[0] + if top_dir.is_dir(): + return top_dir + + normalized = normalize_path(abs_path) + candidate = PROJECTS_DIR / normalized + if candidate.is_dir(): + return candidate + # Fallback: search for partial match + for d in PROJECTS_DIR.iterdir(): + if d.is_dir() and normalized in d.name: + return d + return None + + +def load_sessions_index(project_dir: Path) -> List[Dict]: + """Load and parse sessions-index.json, sorted by modified desc.""" + index_file = project_dir / "sessions-index.json" + if not index_file.exists(): + return [] + with open(index_file, encoding="utf-8") as f: + data = json.load(f) + entries = data.get("entries", []) + entries.sort(key=lambda e: e.get("modified", ""), reverse=True) + return entries + + +def search_sessions(entries: List[Dict], query: str) -> List[Dict]: + """Search sessions by keyword in firstPrompt and summary.""" + query_lower = query.lower() + results = [] + for entry in entries: + first_prompt = (entry.get("firstPrompt") or "").lower() + summary = (entry.get("summary") or "").lower() + if query_lower in first_prompt or query_lower in summary: + results.append(entry) + return results + + +def format_session_entry(entry: Dict, file_exists: bool = True) -> str: + """Format a session index entry for display.""" + sid = entry.get("sessionId", "?") + modified = entry.get("modified", "?") + msgs = entry.get("messageCount", "?") + branch = entry.get("gitBranch", "?") + prompt = (entry.get("firstPrompt") or "")[:80] + ghost = "" if file_exists else " [file missing]" + return f" {sid} [{branch}] {msgs} msgs {modified}{ghost}\n {prompt}" + + +# ── Session file parsing ──────────────────────────────────────────── + + +def parse_session_structure(session_file: Path) -> Dict: + """Parse a session JSONL file and return structured data.""" + file_size = session_file.stat().st_size + total_lines = 0 + + # First pass: find compact boundaries and count lines + compact_boundaries = [] # (line_num, summary_text) + with open(session_file, encoding="utf-8") as f: + prev_boundary_line = None + for i, raw_line in enumerate(f): + total_lines += 1 + + # Detect compact summary via isCompactSummary flag (most reliable) + if '"isCompactSummary"' in raw_line: + try: + obj = json.loads(raw_line) + if obj.get("isCompactSummary"): + content = obj.get("message", {}).get("content", "") + if isinstance(content, str): + boundary_line = prev_boundary_line if prev_boundary_line is not None else max(0, i - 1) + compact_boundaries.append((boundary_line, content)) + prev_boundary_line = None + continue + except json.JSONDecodeError: + pass + + # Detect compact_boundary marker + if '"compact_boundary"' in raw_line and '"subtype"' in raw_line: + try: + obj = json.loads(raw_line) + if obj.get("subtype") == "compact_boundary": + prev_boundary_line = i + continue + except json.JSONDecodeError: + pass + + # Fallback: if prev line was boundary and this is a user message with long string content + if prev_boundary_line is not None: + try: + obj = json.loads(raw_line) + content = obj.get("message", {}).get("content", "") + if isinstance(content, str) and len(content) > 100: + compact_boundaries.append((prev_boundary_line, content)) + except (json.JSONDecodeError, AttributeError): + compact_boundaries.append((prev_boundary_line, "")) + prev_boundary_line = None + + # Determine hot zone: everything after last compact boundary + its summary + if compact_boundaries: + last_boundary_line = compact_boundaries[-1][0] + hot_zone_start = last_boundary_line + 2 # skip boundary + summary + else: + # No compact boundaries: use size-adaptive strategy + if file_size < 500_000: # <500KB: read last 60% + hot_zone_start = max(0, int(total_lines * 0.4)) + elif file_size < 5_000_000: # <5MB: read last 30% + hot_zone_start = max(0, int(total_lines * 0.7)) + else: # >5MB: read last 15% + hot_zone_start = max(0, int(total_lines * 0.85)) + + # Second pass: extract hot zone messages + messages = [] + unresolved_tool_calls = {} # tool_use_id -> tool_use_info + errors = [] + files_touched = set() + last_message_role = None + error_count = 0 + + with open(session_file, encoding="utf-8") as f: + for i, raw_line in enumerate(f): + if i < hot_zone_start: + continue + try: + obj = json.loads(raw_line) + except json.JSONDecodeError: + continue + + msg_type = obj.get("type", "") + if msg_type in NOISE_TYPES: + continue + if msg_type == "system": + subtype = obj.get("subtype", "") + if subtype in NOISE_SUBTYPES: + if subtype == "api_error": + error_count += 1 + continue + if subtype == "compact_boundary": + continue + + # Track tool calls and results + msg = obj.get("message", {}) + role = msg.get("role", "") + content = msg.get("content", "") + + # Extract tool_use from assistant messages + if role == "assistant" and isinstance(content, list): + for block in content: + if not isinstance(block, dict) or block.get("type") != "tool_use": + continue + tool_id = block.get("id", "") + tool_name = block.get("name", "?") + inp = block.get("input", {}) + unresolved_tool_calls[tool_id] = { + "name": tool_name, + "input_preview": str(inp)[:200], + } + # Track file operations + if tool_name in ("Write", "Edit", "Read"): + fp = inp.get("file_path", "") + if fp: + files_touched.add(fp) + elif tool_name == "Bash": + cmd = inp.get("command", "") + for match in re.findall(r'(? str: + """Detect why the session ended.""" + if unresolved: + return "interrupted" # Tool calls dispatched but no results — likely ctrl-c + if error_count >= 3: + return "error_cascade" # Multiple API errors suggest systemic failure + if last_role == "assistant": + return "completed" # Assistant had the last word — clean end + if last_role == "user": + return "abandoned" # User sent a message but got no response + return "unknown" + + +def _is_noise_user_text(text: str) -> bool: + """Check if user text is system noise rather than a real request.""" + for pattern in NOISE_USER_PATTERNS: + if text.startswith(pattern) or pattern in text[:200]: + return True + return False + + +def extract_user_text(messages: List[Dict], limit: int = 5) -> List[str]: + """Extract the last N user text messages (not tool results or system noise).""" + user_texts = [] + for msg_obj in reversed(messages): + if msg_obj.get("isCompactSummary"): + continue + msg = msg_obj.get("message", {}) + if msg.get("role") != "user": + continue + content = msg.get("content", "") + if isinstance(content, str) and content.strip(): + if _is_noise_user_text(content): + continue + user_texts.append(content.strip()) + elif isinstance(content, list): + texts = [ + b.get("text", "") + for b in content + if isinstance(b, dict) and b.get("type") == "text" + ] + combined = "\n".join(t for t in texts if t.strip()) + if combined and not _is_noise_user_text(combined): + user_texts.append(combined) + if len(user_texts) >= limit: + break + user_texts.reverse() + return user_texts + + +def extract_assistant_text(messages: List[Dict], limit: int = 3) -> List[str]: + """Extract the last N assistant text responses (no thinking/tool_use).""" + assistant_texts = [] + for msg_obj in reversed(messages): + msg = msg_obj.get("message", {}) + if msg.get("role") != "assistant": + continue + content = msg.get("content", "") + if isinstance(content, list): + texts = [ + b.get("text", "") + for b in content + if isinstance(b, dict) and b.get("type") == "text" + ] + combined = "\n".join(t for t in texts if t.strip()) + if combined: + assistant_texts.append(combined[:2000]) + if len(assistant_texts) >= limit: + break + assistant_texts.reverse() + return assistant_texts + + +# ── Subagent extraction ────────────────────────────────────────────── + + +def extract_subagent_context(session_file: Path) -> List[Dict]: + """Extract subagent summaries from session subdirectories. + + Returns list of {name, type, status, last_text, is_interrupted}. + """ + session_dir = session_file.parent / session_file.stem + subagents_dir = session_dir / "subagents" + if not subagents_dir.is_dir(): + return [] + + # Group by agent ID: find meta.json and .jsonl pairs + agent_ids = set() + for f in subagents_dir.iterdir(): + if f.suffix == ".jsonl": + agent_ids.add(f.stem) + + results = [] + for agent_id in sorted(agent_ids): + jsonl_file = subagents_dir / f"{agent_id}.jsonl" + meta_file = subagents_dir / f"{agent_id}.meta.json" + + # Parse agent type from ID (format: agent-a- or agent-a) + agent_type = "unknown" + if meta_file.exists(): + try: + with open(meta_file, encoding="utf-8") as f: + meta = json.load(f) + agent_type = meta.get("type", meta.get("subagent_type", "unknown")) + except (json.JSONDecodeError, OSError): + pass + + if agent_type == "unknown": + # Infer from ID pattern: agent-a- + match = re.match(r'agent-a(compact|prompt_suggestion|[a-z_]+)-', agent_id) + if match: + agent_type = match.group(1) + + # Skip compact and prompt_suggestion agents (internal, not user work) + if agent_type in ("compact", "prompt_suggestion"): + continue + + # Read last few lines for final output + last_text = "" + is_interrupted = False + line_count = 0 + + if jsonl_file.exists(): + try: + lines = jsonl_file.read_text(encoding="utf-8").strip().split("\n") + line_count = len(lines) + has_tool_use_pending = False + # Check last 10 lines for final assistant text + for raw_line in reversed(lines[-10:]): + try: + obj = json.loads(raw_line) + msg = obj.get("message", {}) + role = msg.get("role", "") + content = msg.get("content", "") + + if role == "assistant" and isinstance(content, list): + for block in content: + if not isinstance(block, dict): + continue + block_type = block.get("type") + if block_type == "tool_use": + has_tool_use_pending = True + elif block_type == "text": + text = block.get("text", "") + if text.strip() and not last_text: + last_text = text.strip()[:500] + + if role == "user" and isinstance(content, list): + for block in content: + if isinstance(block, dict) and block.get("type") == "tool_result": + has_tool_use_pending = False + except json.JSONDecodeError: + continue + + is_interrupted = has_tool_use_pending + except OSError: + pass + + results.append({ + "id": agent_id, + "type": agent_type, + "last_text": last_text, + "is_interrupted": is_interrupted, + "lines": line_count, + }) + + return results + + +# ── Context sources ────────────────────────────────────────────────── + + +def get_git_state(project_path: str) -> str: + """Get current git status and recent log.""" + parts = [] + try: + branch = subprocess.run( + ["git", "branch", "--show-current"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=project_path, timeout=5, + ) + if branch.stdout.strip(): + parts.append(f"**Current branch**: `{branch.stdout.strip()}`") + except (subprocess.TimeoutExpired, FileNotFoundError): + pass + + try: + status = subprocess.run( + ["git", "status", "--short"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=project_path, timeout=10, + ) + if status.stdout.strip(): + parts.append(f"### git status\n```\n{status.stdout.strip()}\n```") + else: + parts.append("### git status\nClean working tree.") + except (subprocess.TimeoutExpired, FileNotFoundError): + parts.append("### git status\n(unavailable)") + + try: + log = subprocess.run( + ["git", "log", "--oneline", "-5"], + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + universal_newlines=True, cwd=project_path, timeout=10, + ) + if log.stdout.strip(): + parts.append(f"### git log (last 5)\n```\n{log.stdout.strip()}\n```") + except (subprocess.TimeoutExpired, FileNotFoundError): + pass + + return "\n\n".join(parts) + + +def get_memory_md(project_dir: Path) -> Optional[str]: + """Read MEMORY.md if it exists in the project's memory directory.""" + memory_dir = project_dir / "memory" + memory_file = memory_dir / "MEMORY.md" + if memory_file.exists(): + content = memory_file.read_text(encoding="utf-8").strip() + if content: + return content[:3000] + return None + + +def get_session_memory(session_file: Path) -> Optional[str]: + """Read session-memory/summary.md if it exists (newer CC versions).""" + session_dir = session_file.parent / session_file.stem + summary = session_dir / "session-memory" / "summary.md" + if summary.exists(): + content = summary.read_text(encoding="utf-8").strip() + if content: + return content[:3000] + return None + + +# ── Output formatting ──────────────────────────────────────────────── + + +END_REASON_LABELS = { + "completed": "Clean exit (assistant completed response)", + "interrupted": "Interrupted (unresolved tool calls — likely ctrl-c or timeout)", + "error_cascade": "Error cascade (multiple API errors)", + "abandoned": "Abandoned (user message with no response)", + "unknown": "Unknown", +} + + +def build_briefing( + session_entry: Optional[Dict], + parsed: Dict, + project_path: str, + project_dir: Path, + session_file: Path, +) -> str: + """Build the structured Markdown briefing.""" + sections = [] + + # Header + sections.append("# Resume Context Briefing\n") + + # Session metadata + if session_entry: + sid = session_entry.get("sessionId", "?") + modified = session_entry.get("modified", "?") + branch = session_entry.get("gitBranch", "?") + msg_count = session_entry.get("messageCount", "?") + first_prompt = session_entry.get("firstPrompt", "") + summary = session_entry.get("summary", "") + + sections.append("## Session Info\n") + sections.append(f"- **ID**: `{sid}`") + sections.append(f"- **Last active**: {modified}") + sections.append(f"- **Branch**: `{branch}`") + sections.append(f"- **Messages**: {msg_count}") + sections.append(f"- **First prompt**: {first_prompt}") + if summary: + sections.append(f"- **Summary**: {summary[:300]}") + + # File stats + end reason + file_mb = parsed["file_size"] / 1_000_000 + end_label = END_REASON_LABELS.get(parsed["end_reason"], parsed["end_reason"]) + sections.append(f"\n**Session file**: {file_mb:.1f} MB, {parsed['total_lines']} lines, " + f"{len(parsed['compact_boundaries'])} compaction(s)") + sections.append(f"**Session end reason**: {end_label}") + if parsed["error_count"] > 0: + sections.append(f"**API errors**: {parsed['error_count']}") + + # Session memory (newer CC versions generate this automatically) + session_mem = get_session_memory(session_file) + if session_mem: + sections.append("\n## Session Memory (auto-generated by Claude Code)\n") + sections.append(session_mem) + + # Compact summary (highest-signal context) + if parsed["compact_boundaries"]: + last_summary = parsed["compact_boundaries"][-1][1] + if last_summary: + # Allow up to 8K chars — this is the highest-signal content + display = last_summary[:8000] + if len(last_summary) > 8000: + display += f"\n\n... (truncated, full summary: {len(last_summary)} chars)" + sections.append("\n## Compact Summary (auto-generated by previous session)\n") + sections.append(display) + + # Last user requests + user_texts = extract_user_text(parsed["messages"]) + if user_texts: + sections.append("\n## Last User Requests\n") + for i, text in enumerate(user_texts, 1): + display = text[:500] + if len(text) > 500: + display += "..." + sections.append(f"### Request {i}\n{display}\n") + + # Last assistant responses + assistant_texts = extract_assistant_text(parsed["messages"]) + if assistant_texts: + sections.append("\n## Last Assistant Responses\n") + for i, text in enumerate(assistant_texts, 1): + display = text[:1000] + if len(text) > 1000: + display += "..." + sections.append(f"### Response {i}\n{display}\n") + + # Errors encountered + if parsed["errors"]: + sections.append("\n## Errors Encountered\n") + seen = set() + for err in parsed["errors"]: + short = err[:200] + if short not in seen: + seen.add(short) + sections.append(f"```\n{err}\n```\n") + + # Unresolved tool calls (interrupted session) + if parsed["unresolved_tool_calls"]: + sections.append("\n## Unresolved Tool Calls (session was interrupted)\n") + for tool_id, info in parsed["unresolved_tool_calls"].items(): + sections.append(f"- **{info['name']}**: `{tool_id}`") + sections.append(f" Input: {info['input_preview']}") + + # Subagent context (the "nobody has done this" feature) + subagents = extract_subagent_context(session_file) + if subagents: + interrupted = [s for s in subagents if s["is_interrupted"]] + completed = [s for s in subagents if not s["is_interrupted"]] + sections.append(f"\n## Subagent Workflow ({len(completed)} completed, {len(interrupted)} interrupted)\n") + if interrupted: + sections.append("### Interrupted Subagents\n") + for sa in interrupted: + sections.append(f"- **{sa['type']}** (`{sa['id']}`, {sa['lines']} lines)") + if sa["last_text"]: + sections.append(f" Last output: {sa['last_text'][:300]}") + if completed: + sections.append("\n### Completed Subagents\n") + for sa in completed: + sections.append(f"- **{sa['type']}** (`{sa['id']}`, {sa['lines']} lines)") + if sa["last_text"]: + sections.append(f" Last output: {sa['last_text'][:200]}") + + # Files touched in session + if parsed["files_touched"]: + sections.append("\n## Files Touched in Session\n") + for fp in sorted(parsed["files_touched"])[:30]: + sections.append(f"- `{fp}`") + + # MEMORY.md + memory = get_memory_md(project_dir) + if memory: + sections.append("\n## Persistent Memory (MEMORY.md)\n") + sections.append(memory) + + # Git state + sections.append("\n## Current Workspace State\n") + sections.append(get_git_state(project_path)) + + return "\n".join(sections) + + +# ── CLI ────────────────────────────────────────────────────────────── + + +def _check_session_files(entries: List[Dict], project_dir: Path) -> Dict[str, bool]: + """Check which index entries have actual files on disk.""" + status = {} + for entry in entries: + sid = entry.get("sessionId", "") + session_file = project_dir / f"{sid}.jsonl" + if session_file.exists(): + status[sid] = True + else: + full_path = entry.get("fullPath", "") + status[sid] = bool(full_path and Path(full_path).exists()) + return status + + +def main(): + parser = argparse.ArgumentParser( + description="Extract actionable resume context from Claude Code sessions.", + ) + parser.add_argument( + "--project", "-p", + default=os.getcwd(), + help="Project path (default: current directory)", + ) + parser.add_argument( + "--session", "-s", + default=None, + help="Session ID to extract context from", + ) + parser.add_argument( + "--query", "-q", + default=None, + help="Search sessions by keyword in firstPrompt/summary", + ) + parser.add_argument( + "--list", "-l", + action="store_true", + help="List recent sessions", + ) + parser.add_argument( + "--limit", "-n", + type=int, + default=10, + help="Number of sessions to list (default: 10)", + ) + parser.add_argument( + "--exclude-current", + default=None, + help="Session ID to exclude (typically the currently active session)", + ) + args = parser.parse_args() + + project_path = os.path.abspath(args.project) + project_dir = find_project_dir(project_path) + + if not project_dir: + print(f"Error: no Claude session data found for {project_path}", file=sys.stderr) + print(f"Looked in: {PROJECTS_DIR / normalize_path(project_path)}", file=sys.stderr) + sys.exit(1) + + entries = load_sessions_index(project_dir) + + # ── List mode ── + if args.list: + # Show both index entries and actual files, with file-exists status + file_status = _check_session_files(entries, project_dir) + existing = sum(1 for v in file_status.values() if v) + missing = sum(1 for v in file_status.values() if not v) + + if not entries and not list(project_dir.glob("*.jsonl")): + print("No sessions found.") + sys.exit(0) + + print(f"Sessions for {project_path}:\n") + if missing > 0: + print(f" (index has {len(entries)} entries, {existing} with files, {missing} with missing files)\n") + + for entry in entries[:args.limit]: + sid = entry.get("sessionId", "") + exists = file_status.get(sid, False) + print(format_session_entry(entry, file_exists=exists)) + print() + + # Also show files NOT in index + indexed_ids = {e.get("sessionId") for e in entries} + orphan_files = [ + f for f in sorted(project_dir.glob("*.jsonl"), key=lambda f: f.stat().st_mtime, reverse=True) + if f.stem not in indexed_ids + ] + if orphan_files: + print(f" Files not in index ({len(orphan_files)}):") + for f in orphan_files[:5]: + size_kb = f.stat().st_size / 1000 + print(f" {f.stem} ({size_kb:.0f} KB)") + print() + + sys.exit(0) + + # ── Query mode ── + if args.query: + results = search_sessions(entries, args.query) + if not results: + print(f"No sessions matching '{args.query}'.", file=sys.stderr) + sys.exit(1) + print(f"Sessions matching '{args.query}' ({len(results)} found):\n") + for entry in results[: args.limit]: + print(format_session_entry(entry)) + print() + if len(results) == 1: + args.session = results[0]["sessionId"] + else: + sys.exit(0) + + # ── Extract mode ── + session_id = args.session + session_entry = None + + if session_id: + for entry in entries: + if entry.get("sessionId") == session_id: + session_entry = entry + break + session_file = project_dir / f"{session_id}.jsonl" + if not session_file.exists(): + if session_entry: + full_path = session_entry.get("fullPath", "") + if full_path and Path(full_path).exists(): + session_file = Path(full_path) + if not session_file.exists(): + for jsonl in project_dir.glob("*.jsonl"): + if session_id in jsonl.name: + session_file = jsonl + break + else: + print(f"Error: session file not found for {session_id}", file=sys.stderr) + sys.exit(1) + else: + # Use latest session — prefer actual files, skip current session + jsonl_files = sorted( + project_dir.glob("*.jsonl"), + key=lambda f: f.stat().st_mtime, + reverse=True, + ) + if args.exclude_current: + jsonl_files = [f for f in jsonl_files if f.stem != args.exclude_current] + if not jsonl_files: + print("Error: no session files found.", file=sys.stderr) + sys.exit(1) + + # Skip the most recent file if it's likely the current active session + # (modified within the last 60 seconds and no explicit session requested) + if len(jsonl_files) > 1: + newest = jsonl_files[0] + age_seconds = time.time() - newest.stat().st_mtime + if age_seconds < 60: + print(f"Skipping active session {newest.stem} (modified {age_seconds:.0f}s ago)", + file=sys.stderr) + jsonl_files = jsonl_files[1:] + + session_file = jsonl_files[0] + session_id = session_file.stem + for entry in entries: + if entry.get("sessionId") == session_id: + session_entry = entry + break + + # Parse and build briefing + print(f"Parsing session {session_id} ({session_file.stat().st_size / 1_000_000:.1f} MB)...", + file=sys.stderr) + + parsed = parse_session_structure(session_file) + briefing = build_briefing(session_entry, parsed, project_path, project_dir, session_file) + + print(briefing) + + +if __name__ == "__main__": + main() diff --git a/daymade-claude-code/marketplace-dev/.security-scan-passed b/daymade-claude-code/marketplace-dev/.security-scan-passed new file mode 100644 index 00000000..f7c4d070 --- /dev/null +++ b/daymade-claude-code/marketplace-dev/.security-scan-passed @@ -0,0 +1,4 @@ +Security scan passed +Scanned at: 2026-04-11T22:50:53.799504 +Tool: gitleaks + pattern-based validation +Content hash: 7b7f335893c7905da722e12f0099ed1878164b056eae175995257cff433c62fb diff --git a/daymade-claude-code/marketplace-dev/SKILL.md b/daymade-claude-code/marketplace-dev/SKILL.md new file mode 100644 index 00000000..07221797 --- /dev/null +++ b/daymade-claude-code/marketplace-dev/SKILL.md @@ -0,0 +1,377 @@ +--- +name: marketplace-dev +description: | + Converts any Claude Code skills repository into an official plugin marketplace. + Analyzes existing skills, generates .claude-plugin/marketplace.json conforming to + the Anthropic spec, validates with `claude plugin validate`, tests real installation, + and creates a PR to the upstream repo. Encodes hard-won anti-patterns from real + marketplace development (schema traps, version semantics, description pitfalls). + Use when the user mentions: marketplace, plugin support, one-click install, + marketplace.json, plugin distribution, auto-update, or wants a skills repo + installable via `claude plugin install`. Also trigger when the user has a skills + repo and asks about packaging, distribution, or making it installable. +argument-hint: [repo-path] +--- + +# marketplace-dev + +Convert a Claude Code skills repository into an official plugin marketplace so users +can install skills via `claude plugin marketplace add` and get auto-updates. + +**Input**: a repo with `skills/` directories containing SKILL.md files. +**Output**: `.claude-plugin/marketplace.json` + validated + installation-tested + PR-ready. + +## Phase 0: Evidence Intake + +Before editing an existing marketplace, collect evidence instead of relying on the +default template: + +1. Read the current `.claude-plugin/marketplace.json`. +2. Read this repo's marketplace rules (`CLAUDE.md`, README install section, changelog). +3. Read official docs for marketplace/plugin path semantics. +4. If refining from prior failures, mine local Claude Code session history. + +Each project's sessions live under `~/.claude/projects//`: +- Top-level files: `.jsonl` +- Subagent transcripts: `/subagents/agent-*.jsonl` + +Useful search patterns (adjust keywords to the failure you are debugging): + +```bash +grep -lc "marketplace.json\|claude plugin validate\|claude plugin install" \ + ~/.claude/projects//*.jsonl +grep -lc "Unrecognized key\|Plugin not found\|No manifest found\|Duplicate plugin" \ + ~/.claude/projects//*.jsonl \ + ~/.claude/projects//*/subagents/*.jsonl +``` + +Extract lessons as evidence-backed rules: command attempted, observed output, root +cause, final working command/config. Do not encode guesses from memory. + +## Phase 1: Analyze the Target Repo + +### Step 1: Discover all skills + +```bash +# Find every SKILL.md +find /skills -name "SKILL.md" -type f 2>/dev/null +``` + +For each skill, extract from SKILL.md frontmatter: +- `name` — the skill identifier +- `description` — the ORIGINAL text, do NOT rewrite or translate + +### Step 2: Read the repo metadata + +- `VERSION` file (if exists) — this becomes `metadata.version` +- `README.md` — understand the project, author info, categories +- `LICENSE` — note the license type +- Git remotes — identify upstream vs fork (`git remote -v`) + +### Step 3: Determine categories + +Group skills by function. Categories are freeform strings. Good patterns: +- `business-diagnostics`, `content-creation`, `thinking-tools`, `utilities` +- `developer-tools`, `productivity`, `documentation`, `security` + +Ask the user to confirm categories if grouping is ambiguous. + +### Step 4: Choose plugin boundaries + +Claude Code has three separate levels: + +```text +marketplace -> plugin -> skill +``` + +- Marketplace name is used for install identity: `plugin@marketplace`. +- Plugin name is the slash namespace: `/plugin-name:skill-name`. +- Skill name comes from `SKILL.md` frontmatter when the skill path points to a + directory containing `SKILL.md` directly. + +Choose each plugin boundary by installation/update/cache intent: + +- **Single-skill plugin**: use when the skill should install, update, and roll back + independently with a narrow cache. +- **Suite plugin**: use when related skills should share one namespace and one + install command, for example `/daymade-docs:mermaid-tools`. + +For detailed source/cache patterns and pitfalls, read +`references/cache_and_source_patterns.md` before changing `source` or `skills`. + +## Phase 2: Create marketplace.json + +### The official schema (memorize this) + +Read `references/marketplace_schema.md` for the complete field reference. +Key rules that are NOT obvious from the docs: + +1. **`$schema` field is REJECTED** by `claude plugin validate`. Do not include it. +2. **`metadata` only has 3 valid fields**: `description`, `version`, `pluginRoot`. Nothing else. + `metadata.homepage` does NOT exist — the validator accepts it silently but it's not in the spec. +3. **`metadata.version`** is the marketplace catalog version, NOT individual plugin versions. + It should match the repo's VERSION file (e.g., `"2.3.0"`). +4. **Plugin entry `version`** is independent. For first-time marketplace registration, use `"1.0.0"`. +5. **`strict: false`** is required when there's no `plugin.json` in the repo. + With `strict: false`, the marketplace entry IS the entire plugin definition. + Having BOTH `strict: false` AND a `plugin.json` with components causes a load failure. +6. **`source` defines the installed plugin root**. All `skills` paths are relative + to that root. Use `source: "./"` only when a full repo snapshot is intended. + Use `source: "./path/to/skill"` + `skills: ["./"]` for a single-skill narrow + cache. Use `source: "./"` for suite plugins whose cache should + contain only the suite members. +7. **Reserved marketplace names** that CANNOT be used: `claude-code-marketplace`, + `claude-code-plugins`, `claude-plugins-official`, `anthropic-marketplace`, + `anthropic-plugins`, `agent-skills`, `knowledge-work-plugins`, `life-sciences`. +8. **`tags` vs `keywords`**: Both are optional. In the current Claude Code source, + `keywords` is defined but never consumed in search. `tags` only has a UI effect + for the value `"community-managed"` (shows a label). Neither affects discovery. + The Discover tab searches only `name` + `description` + `marketplaceName`. + Include `keywords` for future-proofing but don't over-invest. + +### Generate the marketplace.json + +Use this template, filling in from the analysis: + +```json +{ + "name": "", + "owner": { + "name": "" + }, + "metadata": { + "description": "", + "version": "" + }, + "plugins": [ + { + "name": "", + "description": "", + "source": "./", + "strict": false, + "version": "1.0.0", + "category": "", + "keywords": ["", ""], + "skills": ["./skills/"] + } + ] +} +``` + +### Naming the marketplace + +The `name` field is what users type after `@` in install commands: +`claude plugin install dbs@` + +Choose a name that is: +- Short and memorable +- kebab-case (lowercase, hyphens only) +- Related to the project identity, not generic + +### Description rules + +- **Use the ORIGINAL description from each SKILL.md frontmatter** +- Do NOT translate, embellish, or "improve" descriptions +- If the repo's audience is Chinese, keep descriptions in Chinese +- If bilingual, use the first language in the SKILL.md description field +- The `metadata.description` at marketplace level can be a new summary + +## Maintaining an existing marketplace + +When adding a new plugin to an existing marketplace.json: + +1. **Bump `metadata.version`** — this is the marketplace catalog version. + Follow semver: new plugin = minor bump, breaking change = major bump. +2. **Update `metadata.description`** — append the new skill's summary. +3. **Set new plugin `version` to `"1.0.0"`** — it's new to the marketplace. +4. **Bump existing plugin `version`** when its SKILL.md content changes. + Claude Code uses version to detect updates — same version = skip update. +5. **Bump existing plugin `version`** when its `source` or `skills` changes. + The installed cache path and component resolution changed even if SKILL.md did not. +6. **Audit `metadata` for invalid fields** — `metadata.homepage` is a common + mistake (not in spec, silently ignored). Remove if found. + +## Phase 3: Validate + +### Step 1: One-shot pre-flight check + +Run the bundled validator. It runs four checks in sequence and exits non-zero on +any required failure: + +```bash +bash scripts/check_marketplace.sh # validates current repo +bash scripts/check_marketplace.sh /path # validates a target repo +``` + +What it checks: + +| # | Check | Failure means | +|---|-------|---------------| +| 1 | JSON syntax of `.claude-plugin/marketplace.json` | file is not parseable JSON | +| 2 | `claude plugin validate .` (skipped if `claude` CLI missing) | schema-level rejection (e.g. `Unrecognized key: "$schema"`, duplicate names) | +| 3 | `source` + `skills` resolution for every plugin entry | a plugin entry points to a SKILL.md that does not exist on disk | +| 4 | Reverse sync (disk → manifest) | WARN-only: a SKILL.md on disk is not registered in any plugin entry | + +Common schema failures and fixes: +- `Unrecognized key: "$schema"` → remove the `$schema` field +- `Duplicate plugin name` → ensure all names are unique +- `Path contains ".."` → use `./` relative paths only +- `No manifest found in directory` when validating an installed cache path → validate + the marketplace manifest or plugin source, not a `strict: false` cache directory. + +### Step 2: Installation test + +```bash +# Add as local marketplace +claude plugin marketplace add . + +# Install a plugin +claude plugin install @ + +# Verify it appears +claude plugin list | grep + +# Check for updates (should say "already at latest") +claude plugin update @ + +# Clean up +claude plugin uninstall @ +claude plugin marketplace remove +``` + +### Step 3: Cache footprint test + +After installation or update, inspect the actual cache. This is the only way to +confirm `source` produced the intended snapshot: + +```bash +PLUGIN= +MARKET= +CACHE=$(jq -r --arg id "$PLUGIN@$MARKET" '.plugins[$id][0].installPath' ~/.claude/plugins/installed_plugins.json) +find "$CACHE" -maxdepth 1 -mindepth 1 -exec basename {} \; | sort +``` + +Expected results: + +- Single-skill plugin cache: `SKILL.md` plus its own `scripts/`, `references/`, + `assets/` as applicable. +- Suite plugin cache: only the suite member skill directories and suite-scoped + resources. +- If unrelated skill directories appear, `source` is too broad. +- If cache entries are symlinks, the plugin is not self-contained; use canonical + source directories instead of symlink farms. + +### Step 4: GitHub installation test (if pushed) + +```bash +# Test from GitHub (requires the branch to be pushed) +claude plugin marketplace add / +claude plugin install @ + +# Verify +claude plugin list | grep + +# Clean up +claude plugin uninstall @ +claude plugin marketplace remove +``` + +## Pre-flight Checklist (MUST pass before proceeding to PR) + +Run this checklist after every marketplace.json change. Do not skip items. + +### Automated checks + +```bash +bash scripts/check_marketplace.sh +``` + +All four checks must pass. Treat the reverse-sync WARN as a real signal: an +unregistered `SKILL.md` on disk is almost always either an accidentally-dropped +skill you forgot to register, or dead code that should be removed. + +### Metadata check + +Verify these by reading marketplace.json: + +- [ ] `metadata.version` bumped from previous version +- [ ] `metadata.description` mentions all skill categories +- [ ] No `metadata.homepage` (not in spec, silently ignored) +- [ ] No `$schema` field (rejected by validator) + +### Per-plugin check + +For each plugin entry: + +- [ ] `description` matches SKILL.md frontmatter EXACTLY (not rewritten) +- [ ] `version` is `"1.0.0"` for new plugins, bumped for changed plugins +- [ ] `source` starts with `"./"` and intentionally matches the plugin cache boundary +- [ ] Every `skills` path starts with `"./"` and resolves relative to `source` +- [ ] If `source` points directly at a skill root, `skills` is `["./"]` +- [ ] `strict` is `false` (no plugin.json in repo) +- [ ] `name` is kebab-case, unique across all entries + +### Final validation + +```bash +bash scripts/check_marketplace.sh +``` + +Must print `RESULT: PASSED` before creating a PR. A `WARN [4/4]` is acceptable +only when you have consciously decided to leave a SKILL.md unregistered. + +## Phase 4: Create PR + +### Principles +- **Pure incremental**: do NOT modify any existing files (skills, README, etc.) +- **Squash commits**: avoid binary bloat in git history from iterative changes +- Only add: `.claude-plugin/marketplace.json`, optionally `scripts/`, optionally update README + +### README update (if appropriate) +Add the marketplace install method above existing install instructions: + +```markdown +## Install + +![demo](demo.gif) + +**Claude Code plugin marketplace (one-click install, auto-update):** + +\`\`\`bash +claude plugin marketplace add / +claude plugin install @ +\`\`\` +``` + +### PR description template +Include: +- What was added (marketplace.json with N skills, M categories) +- Install commands users will use after merge +- Design decisions (pure incremental, original descriptions, etc.) +- Validation evidence (`claude plugin validate .` passed) +- Test plan (install commands to verify) + +## Bundled hooks (optional, auto-activated) + +This skill ships two PostToolUse hooks under `hooks/`: + +- `hooks/post_edit_validate.sh` — runs `claude plugin validate` whenever a + `marketplace.json` file is written or edited. +- `hooks/post_edit_sync_check.sh` — warns when a `SKILL.md` is edited but the + matching plugin entry in `marketplace.json` does not bump its `version`. + +Both hooks are declared in this plugin's own manifest entry (`plugins[].hooks`), +so they activate automatically when the plugin is enabled in a Claude Code +session. No manual `settings.json` edit is required. To disable them, remove +the `hooks` block from this plugin entry in the user's installed copy or use +`/plugin disable marketplace-dev` (they take effect only when the plugin is +enabled). + +These hooks are editor-time guardrails. They do NOT replace +`scripts/check_marketplace.sh` — always run the pre-flight check before a PR. + +## Anti-Patterns (things that went wrong and how to fix them) + +Read `references/anti_patterns.md` for the full list of pitfalls discovered during +real marketplace development. These are NOT theoretical — every one was encountered +and debugged in production. diff --git a/daymade-claude-code/marketplace-dev/hooks/post_edit_sync_check.sh b/daymade-claude-code/marketplace-dev/hooks/post_edit_sync_check.sh new file mode 100755 index 00000000..872289e3 --- /dev/null +++ b/daymade-claude-code/marketplace-dev/hooks/post_edit_sync_check.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# PostToolUse hook: warn when SKILL.md is edited but marketplace.json version not bumped +# Detects skill content changes that need a corresponding version bump in marketplace.json + +set -euo pipefail + +INPUT=$(cat) + +FILE_PATH=$(echo "$INPUT" | python3 -c " +import json, sys +try: + data = json.load(sys.stdin) + print(data.get('tool_input', {}).get('file_path', '')) +except: + print('') +" 2>/dev/null) + +# Only care about SKILL.md edits +[[ "$FILE_PATH" != *"SKILL.md"* ]] && exit 0 + +# Find the skill name from path (e.g., /repo/skills/dbs-hook/SKILL.md → dbs-hook) +SKILL_DIR=$(dirname "$FILE_PATH") +SKILL_NAME=$(basename "$SKILL_DIR") + +# Search for marketplace.json upward from the edited file +SEARCH_DIR="$SKILL_DIR" +MARKETPLACE_JSON="" +while [[ "$SEARCH_DIR" != "/" ]]; do + if [[ -f "$SEARCH_DIR/.claude-plugin/marketplace.json" ]]; then + MARKETPLACE_JSON="$SEARCH_DIR/.claude-plugin/marketplace.json" + break + fi + SEARCH_DIR=$(dirname "$SEARCH_DIR") +done + +[[ -z "$MARKETPLACE_JSON" ]] && exit 0 + +# Check if this skill is registered and if version needs bumping +python3 -c " +import json, sys + +with open('$MARKETPLACE_JSON') as f: + data = json.load(f) + +skill_name = '$SKILL_NAME' +found = False +for p in data.get('plugins', []): + skills = p.get('skills', []) + for s in skills: + if s.rstrip('/').split('/')[-1] == skill_name: + found = True + version = p.get('version', 'unknown') + print(json.dumps({ + 'result': f'SKILL.md for \"{skill_name}\" was modified. Remember to bump its version in marketplace.json (currently {version}). Users on the old version won\\'t receive this update otherwise.' + })) + break + if found: + break + +if not found: + print(json.dumps({ + 'result': f'SKILL.md for \"{skill_name}\" was modified but this skill is NOT registered in marketplace.json. Add it if you want it installable via plugin marketplace.' + })) +" 2>/dev/null diff --git a/daymade-claude-code/marketplace-dev/hooks/post_edit_validate.sh b/daymade-claude-code/marketplace-dev/hooks/post_edit_validate.sh new file mode 100755 index 00000000..ae8118a5 --- /dev/null +++ b/daymade-claude-code/marketplace-dev/hooks/post_edit_validate.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# PostToolUse hook: auto-validate marketplace.json after Write/Edit +# Checks if the edited file is marketplace.json, runs validation if so. + +set -euo pipefail + +# Read tool use details from stdin +INPUT=$(cat) + +# Check if the edited file is marketplace.json +FILE_PATH=$(echo "$INPUT" | python3 -c " +import json, sys +try: + data = json.load(sys.stdin) + path = data.get('tool_input', {}).get('file_path', '') + print(path) +except: + print('') +" 2>/dev/null) + +if [[ "$FILE_PATH" == *"marketplace.json"* ]]; then + MARKETPLACE_DIR=$(dirname "$(dirname "$FILE_PATH")") + if [[ -f "$FILE_PATH" ]]; then + RESULT=$(cd "$MARKETPLACE_DIR" && claude plugin validate . 2>&1) || true + if echo "$RESULT" | grep -q "Validation passed"; then + echo '{"result": "marketplace.json validated ✔"}' + else + ERRORS=$(echo "$RESULT" | grep -v "^$" | head -5) + echo "{\"result\": \"marketplace.json validation FAILED:\\n$ERRORS\"}" + fi + fi +fi diff --git a/daymade-claude-code/marketplace-dev/references/anti_patterns.md b/daymade-claude-code/marketplace-dev/references/anti_patterns.md new file mode 100644 index 00000000..0e3ac57d --- /dev/null +++ b/daymade-claude-code/marketplace-dev/references/anti_patterns.md @@ -0,0 +1,143 @@ +# Anti-Patterns: Marketplace Development Pitfalls + +Every item below was encountered during real marketplace development. +Not theoretical — each cost debugging time. + +## Contents + +- [Schema Errors](#schema-errors) — `$schema`, `metadata.homepage`, conflicting `strict: false` + `plugin.json` +- [Version Confusion](#version-confusion) — marketplace vs plugin version, silent update skips, `source`/`skills` changes +- [Source and Cache Errors](#source-and-cache-errors) — full-repo cache pollution, namespace surprises, symlink suites, cache-directory validation +- [Description Errors](#description-errors) — rewriting SKILL.md descriptions, inventing features +- [Installation Testing](#installation-testing) — GitHub push timing, test cleanup +- [PR Best Practices](#pr-best-practices) — unrelated file diffs, commit squashing +- [Discovery Misconceptions](#discovery-misconceptions) — what `keywords` and `tags` actually do + +## Schema Errors + +### Adding `$schema` field +- **Symptom**: `claude plugin validate` fails with `Unrecognized key: "$schema"` +- **Fix**: Do not include `$schema`. Unlike many JSON schemas, Claude Code rejects it. +- **Why it's tempting**: Other marketplace examples (like daymade/claude-code-skills) include it, + and it works for JSON Schema-aware editors. But the validator is strict. + +### Using `metadata.homepage` +- **Symptom**: Silently ignored — no error, no effect. +- **Fix**: `metadata` only supports `description`, `version`, `pluginRoot`. Put `homepage` + on individual plugin entries if needed. +- **Why it's tricky**: `homepage` IS valid on plugin entries (from plugin.json schema), + so it looks correct but is wrong at the metadata level. + +### Conflicting `strict: false` with `plugin.json` +- **Symptom**: Plugin fails to load with "conflicting manifests" error. +- **Fix**: Choose one authority. `strict: false` means marketplace.json is the SOLE + definition. Remove plugin.json or set `strict: true` and let plugin.json define components. + +## Version Confusion + +### Using marketplace version as plugin version +- **Symptom**: All plugins show the same version (e.g., "2.3.0") even though each was + introduced at different times. +- **Fix**: `metadata.version` = marketplace catalog version (matches repo VERSION file). + Each plugin entry `version` is independent. First time in marketplace = `"1.0.0"`. + +### Not bumping version after changes +- **Symptom**: Users don't receive updates after you push changes. +- **Fix**: Claude Code uses version to detect updates. Same version = skip. + Bump the plugin `version` in marketplace.json when you change skill content. + +### Changing source/skills without bumping plugin version +- **Symptom**: Marketplace cache updates, but installed users stay on an old cache + layout because `claude plugin update` sees the same plugin version. +- **Fix**: Treat `source` and `skills` changes as plugin behavior changes. Bump the + plugin version even if SKILL.md content is unchanged. + +## Source and Cache Errors + +### Using full repo source for a narrow plugin +- **Symptom**: Installing a single plugin creates a cache containing many unrelated + skill directories. +- **Fix**: Point `source` at the intended plugin root and make `skills` relative to + that root. For a skill whose `SKILL.md` is directly in the source root, use + `skills: ["./"]`. + +### Assuming marketplace name controls slash namespace +- **Symptom**: Expecting `/daymade-skills:mermaid-tools` after installing + `mermaid-tools@daymade-skills`. +- **Fix**: The plugin name controls the slash namespace. Use a suite plugin like + `daymade-docs` when you want `/daymade-docs:mermaid-tools`. + +### Building suite sources with symlinks +- **Symptom**: Installed cache contains symlinks pointing back to a marketplace + working copy. +- **Fix**: Use real canonical suite source directories. Do not use symlink farms for + plugin cache boundaries. + +### Validating a strict:false cache directory as a plugin manifest +- **Symptom**: `claude plugin validate ~/.claude/plugins/cache/...` reports + `No manifest found in directory`. +- **Fix**: Validate the marketplace manifest or source repo. Then validate installed + cache footprint with `find`, not `claude plugin validate` on the cache. + +## Description Errors + +### Rewriting or translating SKILL.md descriptions +- **Symptom**: Descriptions don't match the actual skill behavior. English descriptions + for a Chinese-audience repo feel foreign. +- **Fix**: Copy the EXACT `description` field from each SKILL.md frontmatter. + The author wrote it for their audience — preserve it. + +### Inventing features in descriptions +- **Symptom**: Description promises "8,000+ consultations" or "auto-backup and rollback" + when the SKILL.md doesn't mention these specifics. +- **Fix**: Only state what the SKILL.md frontmatter says. If you want to add context, + use the marketplace-level `metadata.description`, not individual plugin descriptions. + +## Installation Testing + +### Not testing after GitHub push +- **Symptom**: Local validation passes, but `claude plugin marketplace add /` + fails because it clones the default branch which doesn't have the marketplace.json. +- **Fix**: Push marketplace.json to the repo's default branch before testing GitHub install. + Feature branches only work if the user specifies the ref. + +### Forgetting to clean up test installations +- **Symptom**: Next test run finds stale marketplace/plugins and produces confusing results. +- **Fix**: Always uninstall plugins and remove marketplace after testing: + ```bash + claude plugin uninstall @ 2>/dev/null + claude plugin marketplace remove 2>/dev/null + ``` + +## PR Best Practices + +### Modifying existing files unnecessarily +- **Symptom**: PR diff includes unrelated changes (empty lines in .gitignore, whitespace + changes in README), making it harder to review and merge. +- **Fix**: Only add new files. If modifying README, be surgical — only add the install section. + Verify with `git diff upstream/main` that no unrelated files are touched. + +### Not squashing commits +- **Symptom**: Git history has 15+ commits with iterative demo.gif changes, bloating the + repo by megabytes. Users cloning the marketplace download all this history. +- **Fix**: Squash all commits into one before creating the PR: + ```bash + git reset --soft upstream/main + git commit -m "feat: add Claude Code plugin marketplace" + ``` + +## Discovery Misconceptions + +### Over-investing in keywords/tags +- **Symptom**: Spending time crafting perfect keyword lists. +- **Reality**: In the current Claude Code source (verified by reading DiscoverPlugins.tsx), + the Discover tab searches ONLY `name` + `description` + `marketplaceName`. + `keywords` is defined in the schema but never consumed. `tags` only affects UI + for the specific value `"community-managed"`. Include keywords for future-proofing + but don't obsess over them. + +### Using `tags` for search optimization +- **Symptom**: Adding `tags: ["business", "diagnosis"]` expecting search improvements. +- **Reality**: Only `tags: ["community-managed"]` has any effect (shows a UI label). + The official Anthropic marketplace (123 plugins) uses tags on only 3 plugins, + all with the value `["community-managed"]`. diff --git a/daymade-claude-code/marketplace-dev/references/cache_and_source_patterns.md b/daymade-claude-code/marketplace-dev/references/cache_and_source_patterns.md new file mode 100644 index 00000000..c0083c1d --- /dev/null +++ b/daymade-claude-code/marketplace-dev/references/cache_and_source_patterns.md @@ -0,0 +1,174 @@ +# Cache and Source Patterns + +This reference captures marketplace lessons from real Claude Code marketplace work: +full-repo cache pollution, suite namespace design, symlink experiments, and version +semantics. + +## Contents + +- [Mental Model](#mental-model) — the three-level marketplace → plugin → skill hierarchy +- [Pattern: Single-Skill Narrow Cache](#pattern-single-skill-narrow-cache) — independent install/update for one skill +- [Pattern: Suite Plugin](#pattern-suite-plugin) — shared namespace for related skills +- [Canonical Source for Suite Members](#canonical-source-for-suite-members) — avoiding duplicate skill directories +- [Anti-Patterns](#anti-patterns) — full-repo sources, symlink suites, broad text replacement +- [Verification Commands](#verification-commands) — schema, resolution, and cache footprint checks + +## Mental Model + +Claude Code marketplace distribution has three levels: + +```text +marketplace -> plugin -> skill +``` + +- **Marketplace** is the catalog and install suffix: `plugin@marketplace`. +- **Plugin** is the install/update/cache boundary and slash namespace. +- **Skill** is the actual `SKILL.md` capability. + +`source` defines the installed plugin root. `skills` paths are resolved relative +to that root. + +## Pattern: Single-Skill Narrow Cache + +Use this when a skill should install and update independently: + +```json +{ + "name": "mermaid-tools", + "source": "./daymade-docs/mermaid-tools", + "strict": false, + "version": "1.0.2", + "skills": ["./"] +} +``` + +Expected cache: + +```text +SKILL.md +references/ +scripts/ +``` + +The slash command remains `/mermaid-tools:mermaid-tools` because the plugin and +skill have the same name. This is acceptable when independence matters more than +namespace aesthetics. + +## Pattern: Suite Plugin + +Use this when related skills should share one namespace: + +```json +{ + "name": "daymade-docs", + "source": "./daymade-docs", + "strict": false, + "version": "1.0.1", + "skills": [ + "./doc-to-markdown", + "./mermaid-tools", + "./pdf-creator", + "./ppt-creator", + "./docs-cleaner", + "./meeting-minutes-taker" + ] +} +``` + +Expected slash commands: + +```text +/daymade-docs:doc-to-markdown +/daymade-docs:mermaid-tools +/daymade-docs:pdf-creator +``` + +Expected cache top level: + +```text +doc-to-markdown/ +docs-cleaner/ +meeting-minutes-taker/ +mermaid-tools/ +pdf-creator/ +ppt-creator/ +``` + +## Canonical Source for Suite Members + +If users also need single-skill installs for suite members, point the individual +plugin entries at the same canonical subdirectories: + +```json +{ + "name": "pdf-creator", + "source": "./daymade-docs/pdf-creator", + "strict": false, + "version": "1.3.2", + "skills": ["./"] +} +``` + +Avoid keeping duplicate root-level skill directories and suite copies. Duplication +creates drift and makes version bumps ambiguous. + +## Anti-Patterns + +### Full repo source for a single skill + +```json +{ + "name": "mermaid-tools", + "source": "./", + "skills": ["./mermaid-tools"] +} +``` + +This loads correctly but installs a full repository cache for one plugin. The cache +will contain unrelated skills and can confuse debugging. + +### Symlink suite directories + +Do not build suite sources from symlinks to canonical skill directories. Claude Code +preserves the symlink in the cache, and the symlink can point back to the marketplace +working copy. That cache is not self-contained or version-immutable. + +### Text-wide source replacement + +Do not patch `source` fields by broad text replacement. In a real failure, a patch +that intended to change only docs plugins also changed unrelated plugins like +`skill-creator` and `statusline-generator`. Use a structured JSON edit keyed by +`plugins[].name`, then run a source+skills resolution check. + +## Verification Commands + +Schema + resolution + reverse sync in one shot (uses the bundled script): + +```bash +bash scripts/check_marketplace.sh # validate current repo +bash scripts/check_marketplace.sh /path # validate a target repo +``` + +`check_marketplace.sh` runs four checks: + +1. JSON syntax of `.claude-plugin/marketplace.json` +2. `claude plugin validate .` (skipped if `claude` CLI is missing) +3. source+skills resolution (every plugin path resolves to a real `SKILL.md`) +4. Reverse sync (WARN-only when a disk `SKILL.md` is not registered) + +Inspect the installed cache footprint (cannot be done by the pre-flight script +because it depends on what `claude plugin install` actually produced): + +```bash +PLUGIN= +MARKET= +CACHE=$(jq -r --arg id "$PLUGIN@$MARKET" \ + '.plugins[$id][0].installPath' ~/.claude/plugins/installed_plugins.json) +find "$CACHE" -maxdepth 1 -mindepth 1 -exec basename {} \; | sort +find "$CACHE" -maxdepth 1 -type l -ls +``` + +A symlink in the cache almost always means the plugin was built from a symlink +suite and is not self-contained. Fix by pointing `source` at a real canonical +directory. + diff --git a/daymade-claude-code/marketplace-dev/references/marketplace_schema.md b/daymade-claude-code/marketplace-dev/references/marketplace_schema.md new file mode 100644 index 00000000..454126af --- /dev/null +++ b/daymade-claude-code/marketplace-dev/references/marketplace_schema.md @@ -0,0 +1,92 @@ +# marketplace.json Complete Schema Reference + +Source: https://code.claude.com/docs/en/plugin-marketplaces + https://code.claude.com/docs/en/plugins-reference +Verified against Claude Code source code and `claude plugin validate`. + +## Root Level + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `name` | string | **Yes** | Marketplace identifier, kebab-case. Users see this: `plugin install X@` | +| `owner` | object | **Yes** | `name` (required string), `email` (optional string) | +| `plugins` | array | **Yes** | Array of plugin entries | +| `metadata` | object | No | See metadata fields below | + +### metadata fields (ONLY these 3 are valid) + +| Field | Type | Description | +|-------|------|-------------| +| `metadata.description` | string | Brief marketplace description | +| `metadata.version` | string | Marketplace catalog version (NOT plugin version) | +| `metadata.pluginRoot` | string | Base directory prepended to relative plugin source paths | + +**Invalid metadata fields** (silently ignored or rejected): +- `metadata.homepage` — does NOT exist in spec +- `metadata.repository` — does NOT exist in spec +- `$schema` — REJECTED by `claude plugin validate` + +## Plugin Entry Fields + +Each entry in the `plugins` array. Can include any field from the plugin.json +manifest schema, plus marketplace-specific fields. + +### Required + +| Field | Type | Description | +|-------|------|-------------| +| `name` | string | Plugin identifier, kebab-case | +| `source` | string or object | Where to fetch the plugin | + +### Standard metadata (from plugin.json schema) + +| Field | Type | Description | +|-------|------|-------------| +| `description` | string | Brief plugin description | +| `version` | string | Plugin version (independent of metadata.version) | +| `author` | object | `name` (required), `email` (optional), `url` (optional) | +| `homepage` | string | Plugin homepage URL | +| `repository` | string | Source code URL | +| `license` | string | SPDX identifier (MIT, Apache-2.0, etc.) | +| `keywords` | array | Tags for discovery (currently unused in search) | + +### Marketplace-specific + +| Field | Type | Description | +|-------|------|-------------| +| `category` | string | Freeform category for organization | +| `tags` | array | Tags for searchability (only `community-managed` has UI effect) | +| `strict` | boolean | Default: true. Set false when no plugin.json exists | + +### Component paths (from plugin.json schema) + +| Field | Type | Description | +|-------|------|-------------| +| `commands` | string or array | Custom command file/directory paths | +| `agents` | string or array | Custom agent file paths | +| `skills` | string or array | Custom skill directory paths | +| `hooks` | string or object | Hook configuration or path | +| `mcpServers` | string or object | MCP server config or path | +| `lspServers` | string or object | LSP server config or path | +| `outputStyles` | string or array | Output style paths | +| `userConfig` | object | User-configurable values prompted at enable time | +| `channels` | array | Channel declarations for message injection | + +## Source Types + +| Source | Format | Example | +|--------|--------|---------| +| Relative path | string `"./"` | `"source": "./"` | +| GitHub | object | `{"source": "github", "repo": "owner/repo"}` | +| Git URL | object | `{"source": "url", "url": "https://..."}` | +| Git subdirectory | object | `{"source": "git-subdir", "url": "...", "path": "..."}` | +| npm | object | `{"source": "npm", "package": "@scope/pkg"}` | + +All object sources support optional `ref` (branch/tag) and `sha` (40-char commit). + +## Reserved Marketplace Names + +Cannot be used: `claude-code-marketplace`, `claude-code-plugins`, +`claude-plugins-official`, `anthropic-marketplace`, `anthropic-plugins`, +`agent-skills`, `knowledge-work-plugins`, `life-sciences`. + +Names impersonating official marketplaces are also blocked. diff --git a/daymade-claude-code/marketplace-dev/scripts/check_marketplace.sh b/daymade-claude-code/marketplace-dev/scripts/check_marketplace.sh new file mode 100755 index 00000000..3315496c --- /dev/null +++ b/daymade-claude-code/marketplace-dev/scripts/check_marketplace.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# check_marketplace.sh — One-shot validation for a plugin marketplace repo. +# +# Usage: +# bash scripts/check_marketplace.sh # validate current repo +# bash scripts/check_marketplace.sh /path/to/repo +# +# Runs four checks: +# 1. JSON syntax of .claude-plugin/marketplace.json +# 2. `claude plugin validate .` (skipped if claude CLI missing) +# 3. source+skills resolution (every plugin path points to a real SKILL.md) +# 4. reverse sync (warns when a disk SKILL.md is not registered) +# +# Exit codes: +# 0 all required checks passed (check 4 is warn-only) +# 1 at least one required check failed +# +# Zero external dependencies beyond bash + python3 (ships with macOS and Linux). + +set -uo pipefail + +REPO="${1:-.}" +if [[ ! -d "$REPO" ]]; then + echo "FAIL: repo path '$REPO' is not a directory" >&2 + exit 1 +fi +REPO="$(cd "$REPO" && pwd)" +MANIFEST="$REPO/.claude-plugin/marketplace.json" + +if [[ ! -f "$MANIFEST" ]]; then + echo "FAIL: $MANIFEST not found" >&2 + exit 1 +fi + +echo "Validating marketplace at: $REPO" +echo "" + +FAILED=0 + +# --- Check 1: JSON syntax ------------------------------------------------ +if python3 -m json.tool "$MANIFEST" >/dev/null 2>&1; then + echo "PASS [1/4] JSON syntax" +else + echo "FAIL [1/4] JSON syntax: marketplace.json is not valid JSON" + python3 -m json.tool "$MANIFEST" 2>&1 | head -5 | sed 's/^/ /' + FAILED=1 +fi + +# --- Check 2: claude plugin validate ------------------------------------- +if command -v claude >/dev/null 2>&1; then + if output=$(cd "$REPO" && claude plugin validate . 2>&1); then + echo "PASS [2/4] claude plugin validate" + else + echo "FAIL [2/4] claude plugin validate:" + echo "$output" | sed 's/^/ /' + FAILED=1 + fi +else + echo "SKIP [2/4] claude plugin validate (claude CLI not installed)" +fi + +# --- Check 3: source + skills resolution --------------------------------- +if python3 - "$MANIFEST" "$REPO" <<'PY' +import json, os, sys +manifest_path, repo = sys.argv[1], sys.argv[2] +with open(manifest_path) as f: + data = json.load(f) + +errors = [] +for plugin in data.get("plugins", []): + name = plugin.get("name", "") + source = plugin.get("source", "") + # Remote sources (github/git/npm) can't be resolved on disk + if isinstance(source, dict): + continue + if not isinstance(source, str) or not source.startswith("./"): + continue + root = source.lstrip("./").rstrip("/") or "." + skills = plugin.get("skills") + if skills is None: + skill_dirs = [""] + else: + skill_dirs = [s.lstrip("./").rstrip("/") or "" for s in skills] + for sd in skill_dirs: + skill_md = os.path.join(repo, root, sd, "SKILL.md") + if not os.path.isfile(skill_md): + errors.append(f" {name}: source={source!r} skill={sd!r} -> missing {os.path.relpath(skill_md, repo)}") + +if errors: + print("FAIL [3/4] source+skills resolution:") + for e in errors: + print(e) + sys.exit(1) +print("PASS [3/4] source+skills resolution") +PY +then + : +else + FAILED=1 +fi + +# --- Check 4: reverse sync (warn-only) ----------------------------------- +python3 - "$MANIFEST" "$REPO" <<'PY' +import json, os, sys +manifest_path, repo = sys.argv[1], sys.argv[2] +with open(manifest_path) as f: + data = json.load(f) + +registered = set() +for plugin in data.get("plugins", []): + source = plugin.get("source", "") + if isinstance(source, dict) or not isinstance(source, str) or not source.startswith("./"): + continue + root = source.lstrip("./").rstrip("/") or "." + skills = plugin.get("skills") or [""] + for s in skills: + sd = s.lstrip("./").rstrip("/") or "" + full = os.path.normpath(os.path.join(repo, root, sd)) + registered.add(full) + +ignored = {".git", "node_modules", "dist", "build", ".claude-plugin", ".githooks", "backups"} +disk = set() +for root_dir, dirs, files in os.walk(repo): + # Prune hidden and ignored directories + dirs[:] = [d for d in dirs if d not in ignored and not d.startswith(".")] + if "SKILL.md" in files: + disk.add(os.path.normpath(root_dir)) + +unregistered = disk - registered +if unregistered: + print("WARN [4/4] disk SKILL.md files not registered in marketplace.json:") + for u in sorted(unregistered): + print(f" {os.path.relpath(u, repo)}") +else: + print("PASS [4/4] reverse sync") +PY + +echo "" +if [[ $FAILED -eq 1 ]]; then + echo "RESULT: FAILED" + exit 1 +fi +echo "RESULT: PASSED" diff --git a/statusline-generator/SKILL.md b/daymade-claude-code/statusline-generator/SKILL.md similarity index 67% rename from statusline-generator/SKILL.md rename to daymade-claude-code/statusline-generator/SKILL.md index d21b83bc..0bd73bb6 100644 --- a/statusline-generator/SKILL.md +++ b/daymade-claude-code/statusline-generator/SKILL.md @@ -1,6 +1,6 @@ --- name: statusline-generator -description: Configures and customizes Claude Code statuslines with multi-line layouts, cost tracking via ccusage, git status indicators, and customizable colors. Activates for statusline setup, installation, configuration, customization, color changes, cost display, git status integration, or troubleshooting statusline issues. +description: Configures and customizes Claude Code statuslines with context window display (actual token counts), multi-line layouts, cost tracking via ccusage, git status indicators, human-readable formatting, and customizable colors. Activates for statusline setup, installation, configuration, customization, context window display, color changes, cost display, git status integration, or troubleshooting statusline issues. --- # Statusline Generator @@ -19,6 +19,22 @@ This skill activates for: - Statusline display or cost tracking issues - Git status or path shortening features +## Dependencies + +The statusline script needs to parse JSON from stdin. It auto-detects the available parser: + +| Priority | Tool | Availability | +|----------|------|-------------| +| 1 (preferred) | `jq` | macOS/Linux: `brew install jq` / `apt install jq`; Windows: `choco install jq` or `scoop install jq` | +| 2 (fallback) | `python3` | Pre-installed on macOS and most Linux distros; Windows: `winget install python3` or python.org | + +If neither is available, context window and cost display will be skipped — git branch and path still work. + +Other requirements: +- `git` — for branch status (optional; gracefully skips outside repos) +- `awk` — for number formatting (installed on macOS/Linux by default; Git Bash on Windows includes it) +- `ccusage` — for session/daily cost tracking (optional; gracefully skips if missing) + ## Quick Start ### Basic Installation @@ -33,7 +49,7 @@ Install the default multi-line statusline: 2. Restart Claude Code to see the statusline The default statusline displays: -- **Line 1**: `username (model) [session_cost/daily_cost]` +- **Line 1**: `username (model) [session_cost/daily_cost] ctx: 89.5K/1.0M (9%)` - **Line 2**: `current_path` - **Line 3**: `[git:branch*+]` @@ -61,7 +77,7 @@ Alternatively, manually install by: The statusline uses a 3-line layout optimized for portrait screens: ``` -username (Sonnet 4.5 [1M]) [$0.26/$25.93] +username (Sonnet 4.5 [1M]) [$0.26/$25.93] ctx: 89.5K/1.0M (9%) ~/workspace/java/ready-together-svc [git:feature/branch-name*+] ``` @@ -90,6 +106,26 @@ Model names are automatically shortened: This saves horizontal space while preserving key information. +### Context Window Display + +The statusline can show actual context window usage with human-readable token counts — not just a percentage. This is the most important statusline feature for long sessions. + +Format: +``` +ctx: 88.9K/1.0M (9%) +``` + +Components: +- **Used tokens**: `current_usage.input_tokens + cache_read_input_tokens + cache_creation_input_tokens` +- **Total capacity**: `context_window_size` from the input JSON +- **Percentage**: Shown in parentheses as secondary info +- **Human-readable**: `<1000` → raw, `≥1000` → `X.XK`, `≥1M` → `X.XM` +- **Color-coded**: Green (≤50%), Yellow (51–80%), Red (>80%) + +**Important**: Use `current_usage.*` fields to compute actual context usage. `total_input_tokens` is the session-cumulative count and can exceed the context window size. + +Full statusline input JSON schema (all available fields): `references/context-window-schema.md`. + ### Git Status Indicators Git branch status shows: @@ -140,9 +176,10 @@ Convert to single-line layout by modifying the final `printf`: printf '\033[01;32m%s\033[00m \033[01;36m(%s)\033[00m%s\n\033[01;37m%s\033[00m\n%s' \ "$username" "$model" "$cost_info" "$short_path" "$git_info" -# With: -printf '\033[01;32m%s\033[00m \033[01;36m(%s)\033[00m:\033[01;37m%s\033[00m%s%s' \ - "$username" "$model" "$short_path" "$git_info" "$cost_info" +# With single-line + context: +printf '\033[01;36m[%s]\033[00m \033[01;35m%s\033[00m%s | %bctx: %s/%s (%s%%)\033[00m | \033[01;32m$%s\033[00m' \ + "$model" "$short_dir" "$git_info" "$ctx_color" "$ctx_used_h" "$ctx_size_h" "$ctx_used_pct" "$cost" +# Output: [Sonnet 4.5 [1M]] project (main*) | ctx: 89.5K/1M (9%) | $0.42 ``` ### Disabling Cost Tracking @@ -201,11 +238,14 @@ printf '\033[01;32m%s@%s\033[00m \033[01;36m(%s)\033[00m%s [%s]\n...' \ ## Resources ### scripts/generate_statusline.sh -Main statusline script with all features (multi-line, ccusage, git, colors). Copy this to `~/.claude/statusline.sh` for use. +Main statusline script with all features (context window, multi-line, ccusage, git, colors). Copy to `~/.claude/statusline.sh` for use. ### scripts/install_statusline.sh Automated installation script that copies the statusline script and updates settings.json. +### references/context-window-schema.md +Complete statusline input JSON schema. Documents all fields under `context_window`, `cost`, `model`, `workspace`. Load for context window display implementation or debugging. + ### references/color_codes.md Complete ANSI color code reference for customizing statusline colors. Load when users request color customization. diff --git a/statusline-generator/references/ccusage_integration.md b/daymade-claude-code/statusline-generator/references/ccusage_integration.md similarity index 100% rename from statusline-generator/references/ccusage_integration.md rename to daymade-claude-code/statusline-generator/references/ccusage_integration.md diff --git a/statusline-generator/references/color_codes.md b/daymade-claude-code/statusline-generator/references/color_codes.md similarity index 100% rename from statusline-generator/references/color_codes.md rename to daymade-claude-code/statusline-generator/references/color_codes.md diff --git a/daymade-claude-code/statusline-generator/references/context-window-schema.md b/daymade-claude-code/statusline-generator/references/context-window-schema.md new file mode 100644 index 00000000..aac2b0db --- /dev/null +++ b/daymade-claude-code/statusline-generator/references/context-window-schema.md @@ -0,0 +1,107 @@ +# Statusline Input JSON Schema + +The statusline script receives a JSON object on stdin. This reference documents all known fields. + +## Top-Level Fields + +```json +{ + "session_id": "uuid", + "transcript_path": "/path/to/transcript.jsonl", + "cwd": "/current/working/dir", + "version": "2.1.121", + "fast_mode": false, + "exceeds_200k_tokens": false, + "output_style": { "name": "default" }, + "model": { + "id": "model-id-string", + "display_name": "Human-Readable Model Name" + }, + "workspace": { + "current_dir": "/path/to/cwd", + "project_dir": "/path/to/project", + "added_dirs": ["/additional/dir"] + }, + "cost": { + "total_cost_usd": 1.0868, + "total_duration_ms": 342863, + "total_api_duration_ms": 282122, + "total_lines_added": 0, + "total_lines_removed": 0 + }, + "context_window": { ... }, + "effort": { "level": "max" }, + "thinking": { "enabled": true } +} +``` + +## `context_window` Object (Key Fields) + +```json +{ + "context_window": { + "context_window_size": 1000000, + "used_percentage": 9, + "remaining_percentage": 91, + "total_input_tokens": 83320, + "total_output_tokens": 5456, + "current_usage": { + "input_tokens": 29, + "output_tokens": 163, + "cache_creation_input_tokens": 0, + "cache_read_input_tokens": 88832 + } + } +} +``` + +### Field Meanings + +| Field | Type | Description | +|-------|------|-------------| +| `context_window_size` | number | Model's total context window in tokens (e.g., 1000000 = 1M) | +| `used_percentage` | number | Current context usage as percentage (0-100, may have decimals) | +| `remaining_percentage` | number | Remaining context capacity as percentage | +| `total_input_tokens` | number | **Session-cumulative** input tokens (not current context state) | +| `total_output_tokens` | number | **Session-cumulative** output tokens | +| `current_usage.input_tokens` | number | Current turn uncached input tokens | +| `current_usage.output_tokens` | number | Current turn output tokens | +| `current_usage.cache_creation_input_tokens` | number | Tokens written to prompt cache this turn | +| `current_usage.cache_read_input_tokens` | number | Tokens read from prompt cache this turn | + +### Computing Actual Context Usage + +**Correct formula:** +``` +current_context_used = current_usage.input_tokens + + current_usage.cache_read_input_tokens + + current_usage.cache_creation_input_tokens +``` + +This sum should approximately equal `context_window_size * used_percentage / 100`. + +**Do NOT use `total_input_tokens`** — it's the session-level cumulative total, which can exceed the context window size (e.g., 150K total_input_tokens in a 100K context window is normal because old content gets evicted). + +### Model Context Window Sizes (Reference) + +| Model Family | Typical `context_window_size` | +|-------------|------------------------------| +| Claude Opus 4.x | 200,000 | +| Claude Sonnet 4.x | 200,000 | +| Claude Opus 4.5+ | 500,000 | +| Claude Sonnet 4.5+ | 1,000,000 | +| DeepSeek V4 Flash | 200,000 | +| DeepSeek V4 Pro | 1,000,000 | +| GPT-5.x | varies | + +Always read `context_window_size` from the JSON — never hardcode it. + +## `cost` Object + +| Field | Type | Description | +|-------|------|-------------| +| `total_cost_usd` | number | Session total cost in USD (may have >2 decimals) | +| `total_duration_ms` | number | Total wall-clock duration in ms | +| `total_api_duration_ms` | number | Total API call duration in ms | +| `total_lines_added` | number | Lines added this session | +| `total_lines_removed` | number | Lines removed this session | diff --git a/daymade-claude-code/statusline-generator/scripts/generate_statusline.sh b/daymade-claude-code/statusline-generator/scripts/generate_statusline.sh new file mode 100644 index 00000000..55a82d62 --- /dev/null +++ b/daymade-claude-code/statusline-generator/scripts/generate_statusline.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# Statusline script — multi-line layout with context window, cost, git +# Dependencies: git, awk. jq preferred; python3 used as fallback if jq missing. + +input=$(cat) + +# --- JSON field extraction (jq with python3 fallback) --- +if command -v jq &>/dev/null; then + IFS=$'\t' read -r model_full cwd ctx_used_pct cost_raw ctx_size \ + ctx_input ctx_cache_read ctx_cache_create <<< \ + "$(echo "$input" | jq -r ' + [ + (.model.display_name // "Claude"), + (.workspace.current_dir // ""), + (.context_window.used_percentage // 0), + (.cost.total_cost_usd // 0), + (.context_window.context_window_size // 0), + (.context_window.current_usage.input_tokens // 0), + (.context_window.current_usage.cache_read_input_tokens // 0), + (.context_window.current_usage.cache_creation_input_tokens // 0) + ] | @tsv + ')" +else + # Windows / no-jq: fall back to python3 (stdlib only, no pip needed) + IFS=$'\t' read -r model_full cwd ctx_used_pct cost_raw ctx_size \ + ctx_input ctx_cache_read ctx_cache_create <<< \ + "$(echo "$input" | python3 -c ' +import json, sys +d = json.load(sys.stdin) +cw = d.get("context_window", {}) +cu = cw.get("current_usage", {}) +vals = [ + d.get("model", {}).get("display_name", "Claude"), + d.get("workspace", {}).get("current_dir", ""), + cw.get("used_percentage", 0), + d.get("cost", {}).get("total_cost_usd", 0), + cw.get("context_window_size", 0), + cu.get("input_tokens", 0), + cu.get("cache_read_input_tokens", 0), + cu.get("cache_creation_input_tokens", 0), +] +print("\t".join(str(v) for v in vals)) +')" +fi + +# coerce percentage to int safely +ctx_used_pct=$(printf '%.0f' "${ctx_used_pct:-0}" 2>/dev/null || echo 0) + +# --- derived values --- +cwd="${cwd:-$PWD}" +ctx_used=$((ctx_input + ctx_cache_read + ctx_cache_create)) + +# human-readable number formatter +human() { + local n=${1:-0} + if [ "$n" -ge 1000000 ]; then + awk -v v="$n" 'BEGIN {printf "%.1fM", v/1000000}' + elif [ "$n" -ge 1000 ]; then + awk -v v="$n" 'BEGIN {printf "%.1fK", v/1000}' + else + echo "$n" + fi +} + +ctx_used_h=$(human $ctx_used) +ctx_size_h=$(human $ctx_size) +cost=$(echo "$cost_raw" | awk '{printf "%.2f", $1}') + +# model name shortening +model=$(echo "$model_full" \ + | sed -E 's/\(with ([0-9]+[KM]) token context\)/[\1]/' \ + | sed -E 's/\[1m\]/[1M]/' \ + | sed 's/ *$//') + +username=$(whoami) +short_path="${cwd/#$HOME/\~}" + +# --- context color thresholds --- +ctx_color="\033[01;32m" # green ≤50% +if [ "$ctx_used_pct" -gt 80 ]; then + ctx_color="\033[01;31m" # red >80% +elif [ "$ctx_used_pct" -gt 50 ]; then + ctx_color="\033[01;33m" # yellow 51-80% +fi + +# --- git branch status --- +git_info="" +if [ -d "$cwd/.git" ] || git -C "$cwd" --no-optional-locks rev-parse --git-dir >/dev/null 2>&1; then + branch=$(git -C "$cwd" --no-optional-locks branch --show-current 2>/dev/null || echo "detached") + + status="" + if ! git -C "$cwd" --no-optional-locks diff --quiet 2>/dev/null || \ + ! git -C "$cwd" --no-optional-locks diff --cached --quiet 2>/dev/null; then + status="*" + fi + if [ -n "$(git -C "$cwd" --no-optional-locks ls-files --others --exclude-standard 2>/dev/null)" ]; then + status="${status}+" + fi + + if [ -n "$status" ]; then + git_info=$(printf ' \033[01;31m[git:%s%s]\033[00m' "$branch" "$status") + else + git_info=$(printf ' \033[01;33m[git:%s]\033[00m' "$branch") + fi +fi + +# --- cost via ccusage (async, non-blocking; requires jq or python3) --- +cost_info="" +if command -v jq &>/dev/null || command -v python3 &>/dev/null; then + cache_file="/tmp/claude_cost_cache_$(date +%Y%m%d_%H%M).txt" + find /tmp -name "claude_cost_cache_*.txt" -mmin +2 -delete 2>/dev/null + + if [ -f "$cache_file" ]; then + cost_info=$(cat "$cache_file") + else + { + if command -v jq &>/dev/null; then + session=$(ccusage session --json --offline -o desc 2>/dev/null | jq -r '.sessions[0].totalCost' 2>/dev/null | xargs printf "%.2f" 2>/dev/null) + daily=$(ccusage daily --json --offline -o desc 2>/dev/null | jq -r '.daily[0].totalCost' 2>/dev/null | xargs printf "%.2f" 2>/dev/null) + else + session=$(ccusage session --json --offline -o desc 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print(f\"{d['sessions'][0]['totalCost']:.2f}\")" 2>/dev/null) + daily=$(ccusage daily --json --offline -o desc 2>/dev/null | python3 -c "import json,sys; d=json.load(sys.stdin); print(f\"{d['daily'][0]['totalCost']:.2f}\")" 2>/dev/null) + fi + if [ -n "$session" ] && [ -n "$daily" ] && [ "$session" != "" ] && [ "$daily" != "" ]; then + printf ' \033[01;35m[$%s/$%s]\033[00m' "$session" "$daily" > "$cache_file" + fi + } & + prev_cache=$(find /tmp -name "claude_cost_cache_*.txt" -mmin -10 2>/dev/null | head -1) + if [ -f "$prev_cache" ]; then + cost_info=$(cat "$prev_cache") + fi + fi +fi + +# --- context display string --- +ctx_display="" +if [ "$ctx_size" -gt 0 ]; then + ctx_display=$(printf " ${ctx_color}ctx: %s/%s (%s%%)" "$ctx_used_h" "$ctx_size_h" "$ctx_used_pct") +fi + +# --- output: 3-line layout --- +# Line 1: username (model) [costs] ctx: used/total (pct%) +# Line 2: path +# Line 3: [git:branch] +printf '\033[01;32m%s\033[00m \033[01;36m(%s)\033[00m%s%s\n\033[01;37m%s\033[00m\n%s' \ + "$username" "$model" "$cost_info" "$ctx_display" "$short_path" "$git_info" diff --git a/statusline-generator/scripts/install_statusline.sh b/daymade-claude-code/statusline-generator/scripts/install_statusline.sh similarity index 100% rename from statusline-generator/scripts/install_statusline.sh rename to daymade-claude-code/statusline-generator/scripts/install_statusline.sh diff --git a/daymade-docs/doc-to-markdown/SKILL.md b/daymade-docs/doc-to-markdown/SKILL.md new file mode 100644 index 00000000..8f9014cb --- /dev/null +++ b/daymade-docs/doc-to-markdown/SKILL.md @@ -0,0 +1,186 @@ +--- +name: doc-to-markdown +description: Converts DOCX/PDF/PPTX to high-quality Markdown with automatic post-processing. Fixes pandoc grid tables, simple tables, image paths, CJK bold spacing, attribute noise, and code blocks. Benchmarked best-in-class (7.6/10) against Docling, MarkItDown, Pandoc raw, and Mammoth. Trigger on "convert document", "docx to markdown", "parse word", "doc to markdown", "解析word", "转换文档". +--- + +# Doc to Markdown + +Convert documents to high-quality markdown with intelligent multi-tool orchestration and automatic DOCX post-processing. + +**Architecture**: Pandoc (best-in-class extraction) + 8 post-processing fixes (our value-add). + +## Quick Start + +```bash +# DOCX → Markdown (one command, zero manual fixes) +uv run --with pymupdf4llm --with markitdown scripts/convert.py document.docx -o output.md --assets-dir ./media + +# PDF → Markdown +uv run --with pymupdf4llm --with markitdown scripts/convert.py document.pdf -o output.md + +# Run tests +uv run --with pytest pytest scripts/test_convert.py -v +``` + +## Dual Mode + +| Mode | Speed | Quality | Use Case | +|------|-------|---------|----------| +| **Quick** (default) | Fast | Good | Drafts, simple documents | +| **Heavy** | Slower | Best | Final documents, complex layouts | + +## Tool Selection + +| Format | Quick Mode | Heavy Mode | +|--------|-----------|------------| +| PDF | pymupdf4llm | pymupdf4llm + markitdown | +| DOCX | pandoc + post-processing | pandoc + markitdown | +| PPTX | markitdown | markitdown + pandoc | +| XLSX | markitdown | markitdown | + +## DOCX Post-Processing (automatic) + +When converting DOCX via pandoc, 8 cleanups are applied automatically: + +| Problem | Fix | Test coverage | +|---------|-----|---------------| +| Grid tables (`+:---+`) | Single-column → blockquote, multi-column → pipe table | `TestPostprocessPipeline` | +| Simple tables (` ---- ----`) | Multi-column images → pipe table with captions | `TestSimpleTable` | +| Image path nesting (`media/media/`) | Flatten to `media/`, absolute → relative | `test_stats_tracking` | +| Pandoc attributes (`{width="..."}`) | Removed | `test_pandoc_attributes_removed` | +| CJK bold spacing (`**粗体**中文`) | Add space around `**` for CJK bold spans | `TestCjkBoldSpacing` (15 cases) | +| Indented dashed code blocks | → fenced ``` with language detection | `test_code_block_with_language` | +| Escaped brackets (`\[...\]`) | → `[...]` | `test_escaped_brackets_fixed` | +| Double-bracket links (`[[text]](url)`) | → `[text](url)` | `test_double_bracket_links_fixed` | + +### CJK Bold Spacing — why and how + +DOCX uses run-level styling (no spaces between bold/normal runs in CJK text). Markdown renderers need whitespace around `**` to recognize bold boundaries. + +**Rule**: if a `**content**` span contains any CJK character, ensure both sides have a space — unless already spaced or at line boundary. This handles CJK punctuation, emoji adjacency, and mixed content. + +``` +Before: 打开**飞书**，就可以 → some renderers fail to bold +After: 打开 **飞书** ，就可以 → universally renders correctly +``` + +## Heavy Mode Workflow + +Heavy Mode runs multiple tools in parallel and selects the best segments: + +1. **Parallel Execution**: Run all applicable tools simultaneously +2. **Segment Analysis**: Parse each output into segments (tables, headings, images, paragraphs) +3. **Quality Scoring**: Score each segment based on completeness and structure +4. **Intelligent Merge**: Select best version of each segment across tools + +### Merge Criteria + +| Segment Type | Selection Criteria | +|--------------|-------------------| +| Tables | More rows/columns, proper header separator | +| Images | Alt text present, local paths preferred | +| Headings | Proper hierarchy, appropriate length | +| Lists | More items, nested structure preserved | +| Paragraphs | Content completeness | + +## Image Extraction + +```bash +# Extract images with metadata +uv run --with pymupdf scripts/extract_pdf_images.py document.pdf -o ./extracted-images + +# Generate markdown references file +uv run --with pymupdf scripts/extract_pdf_images.py document.pdf --markdown refs.md +``` + +Output: +- Images: `extracted-images/img_page1_1.png`, `extracted-images/img_page2_1.jpg` +- Metadata: `extracted-images/images_metadata.json` (page, position, dimensions) + +## Quality Validation + +```bash +# Validate conversion quality +uv run --with pymupdf scripts/validate_output.py document.pdf output.md + +# Generate HTML report +uv run --with pymupdf scripts/validate_output.py document.pdf output.md --report report.html +``` + +### Quality Metrics + +| Metric | Pass | Warn | Fail | +|--------|------|------|------| +| Text Retention | >95% | 85-95% | <85% | +| Table Retention | 100% | 90-99% | <90% | +| Image Retention | 100% | 80-99% | <80% | + +## Merge Outputs Manually + +```bash +# Merge multiple markdown files +python scripts/merge_outputs.py output1.md output2.md -o merged.md + +# Show segment attribution +python scripts/merge_outputs.py output1.md output2.md -o merged.md --verbose +``` + +## Path Conversion (Windows/WSL) + +```bash +# Windows to WSL conversion +python scripts/convert_path.py "C:\Users\\Documents\file.pdf" +# Output: /mnt/c/Users//Documents/file.pdf +``` + +## Common Issues + +**"No conversion tools available"** +```bash +# Install all tools +pip install pymupdf4llm +uv tool install "markitdown[pdf]" +brew install pandoc +``` + +**FontBBox warnings during PDF conversion** +- Harmless font parsing warnings, output is still correct + +**Images missing from output** +- Use Heavy Mode for better image preservation +- Or extract separately with `scripts/extract_pdf_images.py` + +**Tables broken in output** +- Use Heavy Mode - it selects the most complete table version +- Or validate with `scripts/validate_output.py` + +## Bundled Scripts + +| Script | Purpose | +|--------|---------| +| `convert.py` | Main orchestrator with Quick/Heavy mode + DOCX post-processing | +| `test_convert.py` | 31 tests covering all post-processing functions | +| `merge_outputs.py` | Merge multiple markdown outputs | +| `validate_output.py` | Quality validation with HTML report | +| `extract_pdf_images.py` | PDF image extraction with metadata | +| `convert_path.py` | Windows to WSL path converter | + +## References + +- `references/benchmark-2026-03-22.md` - 5-tool benchmark (Docling/MarkItDown/Pandoc/Mammoth/ours) +- `references/heavy-mode-guide.md` - Detailed Heavy Mode documentation +- `references/tool-comparison.md` - Tool capabilities comparison +- `references/conversion-examples.md` - Batch operation examples + +## Next Step: Clean Up Converted Content + +After converting documents to markdown, suggest cleanup: + +``` +Conversion complete: [N] files converted to markdown. + +Options: +A) Clean up docs — run /docs-cleaner to consolidate redundant content (Recommended if multiple files) +B) Check facts — run /fact-checker to verify claims in the converted content +C) No thanks — the markdown conversion is sufficient +``` diff --git a/daymade-docs/doc-to-markdown/references/benchmark-2026-03-22.md b/daymade-docs/doc-to-markdown/references/benchmark-2026-03-22.md new file mode 100644 index 00000000..27b20690 --- /dev/null +++ b/daymade-docs/doc-to-markdown/references/benchmark-2026-03-22.md @@ -0,0 +1,163 @@ +# DOCX→Markdown 转换方案基准测试 + +> **测试日期**：2026-03-22 +> +> **测试文件**：`助教-【腾讯云🦞】小白实践 OpenClaw 保姆级教程.docx`（19MB，77 张图片，含 grid table 布局、JSON 代码块、多列图片并排、信息框） +> +> **测试方法**：5 个方案对同一文件转换，按 5 个维度各 10 分制打分 + +--- + +## 综合评分 + +| 维度 | Docling (IBM) | MarkItDown (MS) | Pandoc | Mammoth | **doc-to-markdown（我们）** | +|------|:---:|:---:|:---:|:---:|:---:| +| 表格质量 | 5 | 3 | 5 | 1~3 | **6** | +| 图片提取 | 4 | 2 | **10** | 5 | 7 | +| 文本完整性 | 8 | 7 | **9** | 7 | **9** | +| 格式清洁度 | 5 | 5 | 5 | 3 | **7** | +| 代码块 | 2 | 1 | N/A | 1 | **9** | +| **综合** | **4.8** | **3.6** | **7.3** | **3.4~3.6** | **7.6** | + +--- + +## 各方案详细分析 + +### 1. IBM Docling（综合 4.8） + +- **版本**：docling 2.x + Granite-Docling-258M +- **架构**：AI 驱动（VLM 视觉语言模型），DocTags 中间格式 → Markdown + +**致命问题**： +- 图片引用全部是 `` 占位符（77 张图 0 张可显示），`ImageRefMode` API 对 DOCX 不可用 +- 标题层级全部丢失（0 个 `#`），所有标题退化为粗体文本 +- 代码块为零，JSON 和命令全部输出为普通段落 +- `api_key` 被错误转义为 `api\_key` + +**优点**： +- 文本内容完整，中文/emoji/链接保留良好 +- 无 grid table 或 HTML 残留 +- 表格语法正确（pipe table），但内容是占位符 + +**结论**：Docling 的优势在 PDF（AAAI 2025 论文场景），DOCX 支持远未达到生产级别。 + +### 2. Microsoft MarkItDown（综合 3.6） + +- **版本**：markitdown 0.1.5 +- **架构**：底层调用 mammoth → HTML → markdownify → Markdown + +**致命问题**： +- 77 张图片全部是截断的 base64 占位符（`data:image/png;base64,...`），默认 `keep_data_uris=False` 主动丢弃图片数据 +- 标题全部变为粗体文本（mammoth 无法识别 WPS 自定义样式） +- 代码块为零，JSON 被塞入表格单元格 +- 有序列表编号全部错误（输出为 `1. 1. 1.`） + +**优点**： +- 无 HTML 标签残留 +- 文本内容基本完整 + +**结论**：MarkItDown 的 markdownify 后处理反而引入破坏性截断。轻量场景可用，复杂 DOCX 不可靠。 + +### 3. Pandoc（综合 7.3） + +- **版本**：pandoc 3.9 +- **架构**：Haskell 原生 AST 解析，支持 60+ 格式 + +**测试了 3 种参数**： + +| 参数 | 结果 | +|------|------| +| `-t gfm` | 最差：24 个 HTML `` 嵌套，74 个 HTML `` | +| `-t markdown` | 最佳：grid table（可后处理），无 HTML | +| `-t markdown-raw_html-...` | 与 markdown 完全相同，参数无效果 | + +**问题**： +- Grid table 不可避免（原 docx 有多行单元格和嵌套表格，pipe table 无法表达） +- `{width="..." height="..."}` 68 处 +- `{.underline}` 6 处 +- 反斜杠过度转义 37 处 + +**优点**： +- 图片提取 10/10（77 张全部正确，路径结构一致） +- 文本完整性 9/10（内容、链接、emoji 全部保留） +- 最成熟稳定的底层引擎 + +**结论**：Pandoc 是最可靠的底层引擎，输出质量最高但需要后处理清洗 pandoc 私有语法。 + +### 4. Mammoth（综合 3.4~3.6） + +- **版本**：mammoth 1.11.0 +- **架构**：python-docx 解析 → HTML/Markdown（Markdown 支持已废弃） + +**测试了 2 种方式**： + +| 方式 | 综合 | +|------|------| +| 方式A：直接转 Markdown | 3.4（表格完全丢失） | +| 方式B：转 HTML → markdownify | 3.6（有表格但嵌套被压扁） | + +**致命问题**： +- 标题全部丢失（WPS `styles.xml` 中样式定义为空，mammoth 无法映射 Heading） +- 代码块为零 +- 图片全部 base64 内嵌，单文件 28MB +- 方式B 中 markdownify 丢失 14 张图片（63/77） + +**结论**：Mammoth 的 Markdown 支持已废弃，对 WPS 导出的 docx 兼容性差。不推荐。 + +### 5. doc-to-markdown / 我们的方案（综合 7.6） + +- **版本**：doc-to-markdown 1.0（基于 pandoc + 6 个后处理函数） +- **架构**：Pandoc 转换 → 自动后处理（grid table 清理、图片路径修复、属性清理、代码块修复、转义修复） + +**后处理实际效果**： + +| 后处理函数 | 修复数量 | +|-----------|---------| +| `_convert_grid_tables` | 11 处 grid table → pipe table / blockquote | +| `_clean_pandoc_attributes` | 3437 字符属性清理 | +| `_fix_code_blocks` | 22 处缩进虚线 → ``` 代码块 | +| `_fix_escaped_brackets` | 10 处 | +| `_fix_double_bracket_links` | 1 处 | +| `_fix_image_paths` | 77 张图片路径修复 | + +**已知问题（待修复）**： +- 图片路径双层嵌套 bug：`--assets-dir` 指定目录内被 pandoc 再建一层 `media/` +- 2 处 grid table 残留（文末并排图片组未完全转换） + +**优点**： +- 代码块识别 9/10（JSON 带语言标签，命令行正确包裹） +- 格式清洁度 7/10（attributes、转义、grid table 大部分清理干净） +- 文本完整性 9/10（关键内容全部保留） + +**结论**：综合最优，核心价值在 pandoc 后处理层。剩余 2 个 bug 可修。 + +--- + +## 架构决策 + +``` +最终方案：Pandoc（底层引擎）+ doc-to-markdown 后处理（增值层） + +理由： +1. Pandoc 图片提取最可靠（10/10），文本最完整（9/10） +2. Pandoc 的问题（grid table、属性、转义）全部可后处理解决 +3. Docling/MarkItDown/Mammoth 的致命问题（图片丢失、标题丢失）无法后处理修复 +4. 后处理层是我们的核心竞争力，成本低、可迭代 +``` + +--- + +## 测试文件特征 + +本次测试文件的难点在于： + +| 特征 | 说明 | 影响 | +|------|------|------| +| WPS 导出 | 非标准 Word 样式（Style ID 2/3 而非 Heading 1/2） | mammoth/markitdown/docling 标题全丢 | +| 多列图片布局 | 2x2、1x4 图片网格用表格排版 | pandoc 输出 grid table | +| 信息框/提示框 | 单列表格包裹文字 | pandoc 输出 grid table | +| 嵌套表格 | 表格内套表格 | pipe table 无法表达 | +| JSON 代码块 | 非代码块样式，用文本框/缩进表示 | 多数工具无法识别为代码 | +| 19MB 文件 | 77 张截图嵌入 | base64 方案导致 28MB 输出 | + +这些特征代表了真实世界中 WPS/飞书文档导出 docx 的典型困难，是有效的基准测试场景。 diff --git a/markdown-tools/references/conversion-examples.md b/daymade-docs/doc-to-markdown/references/conversion-examples.md similarity index 89% rename from markdown-tools/references/conversion-examples.md rename to daymade-docs/doc-to-markdown/references/conversion-examples.md index d340ce5e..b1c616cf 100644 --- a/markdown-tools/references/conversion-examples.md +++ b/daymade-docs/doc-to-markdown/references/conversion-examples.md @@ -11,7 +11,7 @@ Comprehensive examples for converting various document formats to markdown. markitdown "document.pdf" > output.md # WSL path example -markitdown "/mnt/c/Users/username/Documents/report.pdf" > report.md +markitdown "/mnt/c/Users//Documents/report.pdf" > report.md # With explicit output markitdown "slides.pdf" > "slides.md" @@ -37,7 +37,7 @@ markitdown "/path/to/docs/file.docx" > "/path/to/output/file.md" markitdown "presentation.pptx" > slides.md # WSL path -markitdown "/mnt/c/Users/username/Desktop/slides.pptx" > slides.md +markitdown "/mnt/c/Users//Desktop/slides.pptx" > slides.md ``` --- @@ -48,10 +48,10 @@ markitdown "/mnt/c/Users/username/Desktop/slides.pptx" > slides.md ```bash # Windows path -C:\Users\username\Documents\file.doc +C:\Users\\Documents\file.doc # WSL equivalent -/mnt/c/Users/username/Documents/file.doc +/mnt/c/Users//Documents/file.doc ``` ### Conversion Examples @@ -62,12 +62,12 @@ C:\folder\file.txt → /mnt/c/folder/file.txt # Path with spaces (must use quotes) -C:\Users\John Doe\Documents\report.pdf -→ "/mnt/c/Users/John Doe/Documents/report.pdf" +C:\Users\\Documents\report.pdf +→ "/mnt/c/Users//Documents/report.pdf" # OneDrive path -C:\Users\username\OneDrive\Documents\file.doc -→ "/mnt/c/Users/username/OneDrive/Documents/file.doc" +C:\Users\\OneDrive\Documents\file.doc +→ "/mnt/c/Users//OneDrive/Documents/file.doc" # Different drive letters D:\Projects\document.docx @@ -78,11 +78,11 @@ D:\Projects\document.docx ```bash # Automatic conversion -python scripts/convert_path.py "C:\Users\username\Downloads\document.doc" -# Output: /mnt/c/Users/username/Downloads/document.doc +python scripts/convert_path.py "C:\Users\\Downloads\document.doc" +# Output: /mnt/c/Users//Downloads/document.doc # Use in conversion command -wsl_path=$(python scripts/convert_path.py "C:\Users\username\file.docx") +wsl_path=$(python scripts/convert_path.py "C:\Users\\file.docx") markitdown "$wsl_path" > output.md ``` @@ -161,7 +161,7 @@ iconv -f ISO-8859-1 -t UTF-8 input.md > output.md ```bash # Mirror directory structure -src_dir="/mnt/c/Users/username/Documents" +src_dir="/mnt/c/Users//Documents" out_dir="/path/to/output" find "$src_dir" -name "*.docx" | while read file; do diff --git a/markdown-tools/references/heavy-mode-guide.md b/daymade-docs/doc-to-markdown/references/heavy-mode-guide.md similarity index 98% rename from markdown-tools/references/heavy-mode-guide.md rename to daymade-docs/doc-to-markdown/references/heavy-mode-guide.md index 442cb6e1..3befcab7 100644 --- a/markdown-tools/references/heavy-mode-guide.md +++ b/daymade-docs/doc-to-markdown/references/heavy-mode-guide.md @@ -1,6 +1,6 @@ # Heavy Mode Guide -Detailed documentation for markdown-tools Heavy Mode conversion. +Detailed documentation for doc-to-markdown Heavy Mode conversion. ## Overview diff --git a/markdown-tools/references/tool-comparison.md b/daymade-docs/doc-to-markdown/references/tool-comparison.md similarity index 100% rename from markdown-tools/references/tool-comparison.md rename to daymade-docs/doc-to-markdown/references/tool-comparison.md diff --git a/daymade-docs/doc-to-markdown/scripts/convert.py b/daymade-docs/doc-to-markdown/scripts/convert.py new file mode 100755 index 00000000..a3c97475 --- /dev/null +++ b/daymade-docs/doc-to-markdown/scripts/convert.py @@ -0,0 +1,1171 @@ +#!/usr/bin/env python3 +""" +Multi-tool document to markdown converter with intelligent orchestration. + +Supports Quick Mode (fast, single tool) and Heavy Mode (best quality, multi-tool merge). +DOCX files get automatic post-processing to fix pandoc artifacts. + +Usage: + # Quick Mode (default) - fast, single best tool + uv run --with pymupdf4llm --with markitdown scripts/convert.py document.pdf -o output.md + + # Heavy Mode - multi-tool parallel execution with merge + uv run --with pymupdf4llm --with markitdown scripts/convert.py document.pdf -o output.md --heavy + + # DOCX deep mode - python-docx direct parsing (experimental) + uv run --with python-docx scripts/convert.py document.docx -o output.md --docx-deep + + # With image extraction + uv run --with pymupdf4llm scripts/convert.py document.pdf -o output.md --assets-dir ./images + +Dependencies: + - pymupdf4llm: PDF conversion (LLM-optimized) + - markitdown: PDF/DOCX/PPTX conversion + - pandoc: DOCX/PPTX conversion (system install: brew install pandoc) + - python-docx: DOCX deep parsing (optional, for --docx-deep) +""" + +import argparse +import json +import re +import subprocess +import sys +import shutil +import zipfile +from dataclasses import dataclass, field +from pathlib import Path +from typing import Optional + + +@dataclass +class ConversionResult: + """Result from a single tool conversion.""" + markdown: str + tool: str + images: list[str] = field(default_factory=list) + success: bool = True + error: str = "" + + +# ── Post-processing stats ──────────────────────────────────────────────────── + +@dataclass +class PostProcessStats: + """Track what the DOCX post-processor fixed.""" + grid_tables_converted: int = 0 + image_paths_fixed: int = 0 + attributes_removed: int = 0 + code_blocks_fixed: int = 0 + escaped_brackets_fixed: int = 0 + double_brackets_fixed: int = 0 + + def any_fixes(self) -> bool: + return any( + getattr(self, f) > 0 + for f in self.__dataclass_fields__ + ) + + def summary(self) -> str: + parts = [] + if self.grid_tables_converted: + parts.append(f"grid tables: {self.grid_tables_converted}") + if self.image_paths_fixed: + parts.append(f"image paths: {self.image_paths_fixed}") + if self.attributes_removed: + parts.append(f"attributes: {self.attributes_removed}") + if self.code_blocks_fixed: + parts.append(f"code blocks: {self.code_blocks_fixed}") + if self.escaped_brackets_fixed: + parts.append(f"escaped brackets: {self.escaped_brackets_fixed}") + if self.double_brackets_fixed: + parts.append(f"double brackets: {self.double_brackets_fixed}") + return ", ".join(parts) if parts else "no fixes needed" + + +# ── DOCX post-processing ───────────────────────────────────────────────────── + +# Regex patterns compiled once +_RE_GRID_BORDER = re.compile(r"^\+[:=-][-:=]+(?:\+[:=-][-:=]+)*\+$") +_RE_GRID_ROW = re.compile(r"^\|(.+)\|$") +_RE_NESTED_GRID_BORDER = re.compile(r"^\|\s*\+[:=-][-:=]+\+\s*\|$") +_RE_PANDOC_ATTR = re.compile(r"\{[^}]*(?:width|height)\s*=\s*\"[^\"]*\"[^}]*\}") +_RE_PANDOC_CLASS = re.compile(r"\{\.(?:underline|mark)\}") +_RE_DOUBLE_BRACKET_LINK = re.compile(r"\[\[([^\]]+)\]\(([^)]+)\)") +_RE_DOUBLE_BRACKET_CLOSED = re.compile(r"\[\[([^\]]+)\]\]\(([^)]+)\)") +_RE_DOUBLE_BRACKET_ATTR_LINK = re.compile(r"\[\[([^\]]+)\]\{[^}]*\}\]\(([^)]+)\)") +_RE_ESCAPED_BRACKET = re.compile(r"\\(\[|])") +# Matches single-column dashed line: " ------" +# AND multi-column simple table border: " ---- -----" +_RE_DASHED_LINE = re.compile(r"^(\s{2,})-{3,}[\s-]*$") +_RE_ESCAPED_QUOTE = re.compile(r'\\"') +# CJK + fullwidth punctuation range for bold spacing checks +_RE_CJK_PUNCT = re.compile(r'[\u4e00-\u9fff\u3000-\u303f\uff01-\uffef，。、；：！？（）【】「」《》""'']') +_RE_BOLD_PAIR = re.compile(r'\*\*(.+?)\*\*') + + +def _is_grid_border(line: str) -> bool: + """Check if a line is a grid table border like +---+ or +:---+.""" + stripped = line.strip() + return bool(_RE_GRID_BORDER.match(stripped)) + + +def _is_nested_grid_border(line: str) -> bool: + """Check if a line is a nested grid border like | +---+ |.""" + stripped = line.strip() + return bool(_RE_NESTED_GRID_BORDER.match(stripped)) + + +def _count_grid_columns(border_line: str) -> int: + """Count columns in a grid table border line.""" + stripped = border_line.strip() + if not stripped.startswith("+"): + return 0 + # Count + separators minus 1 = number of columns + return stripped.count("+") - 1 + + + +# Languages recognized as code block hints in pandoc dashed-line blocks +_KNOWN_CODE_LANGS = frozenset({ + "json", "bash", "shell", "python", "javascript", "js", + "html", "css", "yaml", "xml", "sql", "plain text", + "text", "plaintext", "typescript", "ts", "go", "rust", + "java", "c", "cpp", "ruby", "php", +}) + + +def _build_pipe_table(rows: list[list[str]]) -> list[str]: + """Build a standard markdown pipe table from rows of cells.""" + if not rows: + return [] + col_count = max(len(r) for r in rows) + lines = [ + "| " + " | ".join([""] * col_count) + " |", + "| " + " | ".join(["---"] * col_count) + " |", + ] + for row in rows: + padded = row + [""] * (col_count - len(row)) + lines.append("| " + " | ".join(padded) + " |") + return lines + + +def _collect_images(directory: Path) -> list[str]: + """Collect image files from a directory (single glob pass).""" + if not directory.exists(): + return [] + image_exts = {".png", ".jpg", ".jpeg", ".gif", ".webp"} + return sorted( + str(p) for p in directory.rglob("*") + if p.suffix.lower() in image_exts + ) + + +def _convert_grid_tables(text: str, stats: PostProcessStats) -> str: + """Convert pandoc grid tables to standard markdown. + + Single-column grid tables (info boxes) -> blockquotes. + Multi-column grid tables (side-by-side images) -> split into individual elements. + Nested grid tables are flattened. + """ + lines = text.split("\n") + result = [] + i = 0 + + while i < len(lines): + line = lines[i] + + # Detect grid table start + if _is_grid_border(line): + # Collect the entire grid table + table_lines = [line] + i += 1 + while i < len(lines): + table_lines.append(lines[i]) + if _is_grid_border(lines[i]) and len(table_lines) > 1: + i += 1 + break + i += 1 + else: + # Reached end of file without closing border + # Just output as-is + result.extend(table_lines) + continue + + stats.grid_tables_converted += 1 + num_cols = _count_grid_columns(table_lines[0]) + + # Extract content lines (skip borders) + content_lines = [] + for tl in table_lines: + if _is_grid_border(tl) or _is_nested_grid_border(tl): + continue + m = _RE_GRID_ROW.match(tl.strip()) + if m: + content_lines.append(m.group(1).strip()) + else: + # Non-standard line inside grid, keep content + stripped = tl.strip() + if stripped and stripped != "|": + content_lines.append(stripped) + + if num_cols <= 1: + # Single column -> blockquote + result.append("") + for cl in content_lines: + # Strip outer pipes if present from nested grids + cleaned = cl.strip() + if cleaned.startswith("|") and cleaned.endswith("|"): + cleaned = cleaned[1:-1].strip() + # Skip nested grid borders + if _RE_GRID_BORDER.match(cleaned): + continue + if cleaned: + result.append(f"> {cleaned}") + else: + result.append(">") + result.append("") + else: + # Multi-column -> convert to standard pipe table + # Parse rows: each content_line is a row, split by | into cells + table_rows = [] + for cl in content_lines: + cells = [c.strip() for c in cl.split("|") if c.strip() and not _RE_GRID_BORDER.match(c.strip())] + if cells: + table_rows.append(cells) + + if table_rows: + result.append("") + result.extend(_build_pipe_table(table_rows)) + result.append("") + else: + result.append(line) + i += 1 + + return "\n".join(result) + + +def _fix_image_paths(text: str, assets_dir: Optional[Path], stats: PostProcessStats) -> str: + """Fix pandoc's double media path and verify images exist. + + Pandoc extracts to /media/ but references as + /media/media/. Fix the references. + Also flatten the actual directory if needed. + """ + def fix_path(m: re.Match) -> str: + alt = m.group(1) + path = m.group(2) + new_path = path + + # Fix double media/ path + if "media/media/" in path: + new_path = path.replace("media/media/", "media/") + stats.image_paths_fixed += 1 + + return f"![{alt}]({new_path})" + + text = re.sub(r"!\[([^\]]*)\]\(([^)]+)\)", fix_path, text) + + # Flatten double media/ nesting if present (pandoc artifact) + if assets_dir: + double_media = assets_dir / "media" / "media" + single_media = assets_dir / "media" + try: + for f in double_media.iterdir(): + dest = single_media / f.name + if not dest.exists(): + shutil.move(str(f), str(dest)) + double_media.rmdir() + except (FileNotFoundError, OSError): + pass + + return text + + +def _clean_pandoc_attributes(text: str, stats: PostProcessStats) -> str: + """Remove pandoc attribute annotations from markdown. + + Removes: {width="..." height="..."}, {.underline}, {.mark}, etc. + """ + count_before = len(text) + + # Remove width/height attributes on images + text = _RE_PANDOC_ATTR.sub("", text) + + # Remove class attributes like {.underline} + text = _RE_PANDOC_CLASS.sub("", text) + + if len(text) != count_before: + # Rough count of removals + stats.attributes_removed = count_before - len(text) + + return text + + +def _is_code_content(lines: list[str]) -> bool: + """Heuristic: decide if content between dashed lines is code or a note/callout. + + Code indicators: + - Has a language hint on the first line + - Contains JSON/code-like syntax ({, }, =, ;, ->, //) + - Contains URLs with protocols + - Has backslash line continuations + + Note indicators: + - Mostly CJK/prose text without code syntax + - Short single-line content + """ + text = "\n".join(lines) + stripped = text.strip() + + if not stripped: + return False + + # Code syntax indicators + code_chars = set('{}[]();=<>/\\') + code_char_count = sum(1 for c in stripped if c in code_chars) + + # If >5% of content is code syntax characters, treat as code + if len(stripped) > 0 and code_char_count / len(stripped) > 0.05: + return True + + # JSON-like structure + if stripped.startswith("{") or stripped.startswith("["): + return True + + # Command-like (starts with common command patterns) + first_line = lines[0].strip() if lines else "" + if re.match(r"^(curl|wget|npm|pip|brew|apt|docker|git|ssh|cd|ls|cat|echo|python|node|uv)\s", first_line): + return True + + return False + + +def _fix_code_blocks(text: str, stats: PostProcessStats) -> str: + """Convert pandoc's indented dashed-line blocks to fenced code blocks or blockquotes. + + Pandoc wraps both code and notes in: + ------------------------------------------------------------------ + content here + + ------------------------------------------------------------------ + + With language hint -> code block: + ```json + content here + ``` + + Without language hint + prose content -> blockquote: + > content here + + Without language hint + code-like content -> code block: + ``` + content here + ``` + """ + lines = text.split("\n") + result = [] + i = 0 + + known_langs = _KNOWN_CODE_LANGS + + while i < len(lines): + line = lines[i] + + # Detect indented dashed line (2+ leading spaces, 3+ dashes) + if _RE_DASHED_LINE.match(line): + # Check if this is a pandoc simple table (multiple dashed columns + # on the same line, or content between dashes contains images) + # Simple table pattern: " ---- ----" (multiple dash groups separated by spaces) + # Gap can be 1+ spaces (pandoc uses varying gaps) + dash_parts = [p for p in line.strip().split() if p.strip()] + is_simple_table_border = len(dash_parts) > 1 and all( + re.match(r"^-+$", p.strip()) for p in dash_parts + ) + + if is_simple_table_border: + # This is a pandoc simple table border - collect rows until + # next simple table border, convert to pipe table + table_rows = [] + j = i + 1 + while j < len(lines): + next_line = lines[j] + # Check for closing simple table border + next_parts = [p for p in next_line.strip().split() if p.strip()] + is_next_border = len(next_parts) > 1 and all( + re.match(r"^-+$", p.strip()) for p in next_parts + ) + if is_next_border: + j += 1 + break + if next_line.strip(): + # Split by 2+ spaces to get columns (pandoc uses varying gaps) + cells = [c.strip() for c in re.split(r"\s{2,}", next_line.strip()) if c.strip()] + if cells: + table_rows.append(cells) + j += 1 + + if table_rows: + stats.code_blocks_fixed += 1 + result.append("") + result.extend(_build_pipe_table(table_rows)) + result.append("") + + i = j + continue + + # Not a simple table - look for content and closing dashed line + block_content = [] + lang_hint = "" + j = i + 1 + + while j < len(lines): + next_line = lines[j] + + if _RE_DASHED_LINE.match(next_line): + # Found closing dashed line + j += 1 + break + + block_content.append(next_line) + j += 1 + else: + # No closing dashed line found - not a block, keep as-is + result.append(line) + i += 1 + continue + + # If content contains images, treat as simple table (single-column) + has_images = any("![" in cl for cl in block_content) + if has_images: + result.append("") + for cl in block_content: + cl = cl.strip() + if cl: + result.append(cl) + result.append("") + i = j + continue + + # Check if first line is a language hint (e.g., " JSON\", " Plain Text\") + has_lang_hint = False + if block_content: + first = block_content[0].strip() + first_clean = first.rstrip("\\").strip() + if first_clean.lower() in known_langs: + lang_hint = first_clean.lower() + if lang_hint in ("plain text", "text", "plaintext"): + lang_hint = "" # No language tag for plain text + has_lang_hint = True + block_content = block_content[1:] + + # Clean content: remove leading 2-space indent, fix escaped quotes + cleaned = [] + for cl in block_content: + if cl.startswith(" "): + cl = cl[2:] + cl = cl.replace('\\"', '"') + if cl.endswith("\\"): + cl = cl[:-1] + cleaned.append(cl) + + # Remove trailing/leading empty lines + while cleaned and not cleaned[-1].strip(): + cleaned.pop() + while cleaned and not cleaned[0].strip(): + cleaned.pop(0) + + if cleaned: + stats.code_blocks_fixed += 1 + + # Decide: code block vs blockquote + if has_lang_hint or _is_code_content(cleaned): + # Code block — try to pretty-print JSON + code_lines = cleaned + if lang_hint == "json": + try: + raw = "\n".join(cleaned) + parsed = json.loads(raw) + code_lines = json.dumps(parsed, indent=2, ensure_ascii=False).split("\n") + except (json.JSONDecodeError, ValueError): + pass # Keep original if not valid JSON + + result.append("") + result.append(f"```{lang_hint}") + result.extend(code_lines) + result.append("```") + result.append("") + else: + # Note/callout -> blockquote + result.append("") + for cl in cleaned: + if cl.strip(): + result.append(f"> {cl}") + else: + result.append(">") + result.append("") + + i = j + else: + result.append(line) + i += 1 + + return "\n".join(result) + + +def _fix_escaped_brackets(text: str, stats: PostProcessStats) -> str: + r"""Fix pandoc's escaped brackets: \[ -> [, \] -> ].""" + count = len(_RE_ESCAPED_BRACKET.findall(text)) + if count: + stats.escaped_brackets_fixed = count + text = _RE_ESCAPED_BRACKET.sub(r"\1", text) + return text + + +def _fix_double_bracket_links(text: str, stats: PostProcessStats) -> str: + """Fix double-bracket links: [[text]{.underline}](url) -> [text](url).""" + count = 0 + + def _replace_link(m: re.Match) -> str: + nonlocal count + count += 1 + return f"[{m.group(1)}]({m.group(2)})" + + text = _RE_DOUBLE_BRACKET_ATTR_LINK.sub(_replace_link, text) + text = _RE_DOUBLE_BRACKET_CLOSED.sub(_replace_link, text) + text = _RE_DOUBLE_BRACKET_LINK.sub(_replace_link, text) + + stats.double_brackets_fixed = count + return text + + +def _fix_cjk_bold_spacing(text: str) -> str: + """Add space around **bold** spans that contain CJK characters. + + DOCX uses run-level styling for bold — no spaces between runs in CJK text. + Markdown renderers need whitespace around ** to recognize bold boundaries. + + Rule: if a **content** span contains any CJK character, ensure both sides + have a space (unless already spaced or at line boundary). This handles: + - CJK directly touching **: 打开**飞书** → 打开 **飞书** + - Emoji touching **: **密码】**➡️ → **密码】** ➡️ + - Already spaced: 已有 **粗体** → unchanged + - English bold: English **bold** text → unchanged + """ + result = [] + last_end = 0 + + for m in _RE_BOLD_PAIR.finditer(text): + start, end = m.start(), m.end() + content = m.group(1) + + result.append(text[last_end:start]) + + # Only add spaces for bold spans containing CJK + if _RE_CJK_PUNCT.search(content): + # Space before ** if previous char is not whitespace + if start > 0 and text[start - 1] not in (' ', '\t', '\n'): + result.append(' ') + + result.append(m.group(0)) + + # Space after ** if next char is not whitespace + if end < len(text) and text[end] not in (' ', '\t', '\n'): + result.append(' ') + else: + result.append(m.group(0)) + + last_end = end + + result.append(text[last_end:]) + return ''.join(result) + + +def _cleanup_excessive_blank_lines(text: str) -> str: + """Collapse 3+ consecutive blank lines to 2.""" + return re.sub(r"\n{4,}", "\n\n\n", text) + + +def postprocess_docx_markdown( + text: str, + assets_dir: Optional[Path] = None, +) -> tuple[str, PostProcessStats]: + """Apply all DOCX-specific post-processing to pandoc markdown output. + + Returns (cleaned_text, stats). + """ + stats = PostProcessStats() + + # Order matters: grid tables first (they contain images with attributes) + text = _convert_grid_tables(text, stats) + text = _fix_image_paths(text, assets_dir, stats) + text = _clean_pandoc_attributes(text, stats) + text = _fix_code_blocks(text, stats) + text = _fix_double_bracket_links(text, stats) + text = _fix_escaped_brackets(text, stats) + text = _fix_cjk_bold_spacing(text) + text = _cleanup_excessive_blank_lines(text) + + return text, stats + + +# ── DOCX deep parsing (python-docx) ────────────────────────────────────────── + +def convert_with_docx_deep( + file_path: Path, assets_dir: Optional[Path] = None +) -> ConversionResult: + """Convert DOCX using python-docx direct parsing (experimental). + + More precise than pandoc for: + - Table structure preservation + - Comment extraction + - Image extraction with position info + """ + try: + from docx import Document + from docx.opc.constants import RELATIONSHIP_TYPE as RT + except ImportError: + return ConversionResult( + markdown="", + tool="docx-deep", + success=False, + error="python-docx not installed. Run: pip install python-docx", + ) + + try: + doc = Document(str(file_path)) + md_parts = [] + images = [] + image_counter = 0 + + # Extract images from docx zip + if assets_dir: + assets_dir.mkdir(parents=True, exist_ok=True) + media_dir = assets_dir / "media" + media_dir.mkdir(exist_ok=True) + + with zipfile.ZipFile(str(file_path), "r") as zf: + for name in zf.namelist(): + if name.startswith("word/media/"): + img_name = Path(name).name + img_dest = media_dir / img_name + with zf.open(name) as src, open(img_dest, "wb") as dst: + dst.write(src.read()) + images.append(str(img_dest)) + + # Process paragraphs + for para in doc.paragraphs: + style_name = para.style.name if para.style else "" + text = para.text.strip() + + if not text: + md_parts.append("") + continue + + # Headings + if style_name.startswith("Heading"): + try: + level = int(style_name.split()[-1]) + except (ValueError, IndexError): + level = 1 + md_parts.append(f"{'#' * level} {text}") + md_parts.append("") + continue + + # Check for bold-only paragraphs (often sub-headings in Chinese docs) + all_bold = all(run.bold for run in para.runs if run.text.strip()) + if all_bold and para.runs and len(text) < 100: + md_parts.append(f"**{text}**") + md_parts.append("") + continue + + # Regular paragraph + md_parts.append(text) + md_parts.append("") + + # Process tables + for table in doc.tables: + md_parts.append("") + rows = table.rows + if not rows: + continue + + # Header row + header_cells = [cell.text.strip() for cell in rows[0].cells] + md_parts.append("| " + " | ".join(header_cells) + " |") + md_parts.append("| " + " | ".join(["---"] * len(header_cells)) + " |") + + # Data rows + for row in rows[1:]: + cells = [cell.text.strip() for cell in row.cells] + md_parts.append("| " + " | ".join(cells) + " |") + md_parts.append("") + + markdown = "\n".join(md_parts) + + return ConversionResult( + markdown=markdown, + tool="docx-deep", + images=images, + success=True, + ) + except Exception as e: + return ConversionResult( + markdown="", tool="docx-deep", success=False, error=str(e) + ) + + +# ── Existing tool converters ───────────────────────────────────────────────── + +def check_tool_available(tool: str) -> bool: + """Check if a conversion tool is available.""" + if tool == "pymupdf4llm": + try: + import pymupdf4llm + return True + except ImportError: + return False + elif tool == "markitdown": + try: + import markitdown + return True + except ImportError: + return False + elif tool == "pandoc": + return shutil.which("pandoc") is not None + elif tool == "docx-deep": + try: + from docx import Document + return True + except ImportError: + return False + return False + + +def select_tools(file_path: Path, mode: str) -> list[str]: + """Select conversion tools based on file type and mode.""" + ext = file_path.suffix.lower() + + # Tool preferences by format + tool_map = { + ".pdf": { + "quick": ["pymupdf4llm", "markitdown"], # fallback order + "heavy": ["pymupdf4llm", "markitdown"], + }, + ".docx": { + "quick": ["pandoc", "markitdown"], + "heavy": ["pandoc", "markitdown"], + }, + ".doc": { + "quick": ["pandoc", "markitdown"], + "heavy": ["pandoc", "markitdown"], + }, + ".pptx": { + "quick": ["markitdown", "pandoc"], + "heavy": ["markitdown", "pandoc"], + }, + ".xlsx": { + "quick": ["markitdown"], + "heavy": ["markitdown"], + }, + } + + tools = tool_map.get(ext, {"quick": ["markitdown"], "heavy": ["markitdown"]}) + + if mode == "quick": + # Return first available tool + for tool in tools["quick"]: + if check_tool_available(tool): + return [tool] + return [] + else: # heavy + # Return all available tools + return [t for t in tools["heavy"] if check_tool_available(t)] + + +def convert_with_pymupdf4llm( + file_path: Path, assets_dir: Optional[Path] = None +) -> ConversionResult: + """Convert using PyMuPDF4LLM (best for PDFs).""" + try: + import pymupdf4llm + + kwargs = {} + images = [] + + if assets_dir: + assets_dir.mkdir(parents=True, exist_ok=True) + kwargs["write_images"] = True + kwargs["image_path"] = str(assets_dir) + kwargs["dpi"] = 150 + + # Use best table detection strategy + kwargs["table_strategy"] = "lines_strict" + + md_text = pymupdf4llm.to_markdown(str(file_path), **kwargs) + + if assets_dir: + images = _collect_images(assets_dir) + + return ConversionResult( + markdown=md_text, tool="pymupdf4llm", images=images, success=True + ) + except Exception as e: + return ConversionResult( + markdown="", tool="pymupdf4llm", success=False, error=str(e) + ) + + +def convert_with_markitdown( + file_path: Path, assets_dir: Optional[Path] = None +) -> ConversionResult: + """Convert using markitdown.""" + try: + # markitdown CLI approach + result = subprocess.run( + ["markitdown", str(file_path)], + capture_output=True, + text=True, + timeout=120, + ) + + if result.returncode != 0: + return ConversionResult( + markdown="", + tool="markitdown", + success=False, + error=result.stderr, + ) + + return ConversionResult( + markdown=result.stdout, tool="markitdown", success=True + ) + except FileNotFoundError: + # Try Python API + try: + from markitdown import MarkItDown + + md = MarkItDown() + result = md.convert(str(file_path)) + return ConversionResult( + markdown=result.text_content, tool="markitdown", success=True + ) + except Exception as e: + return ConversionResult( + markdown="", tool="markitdown", success=False, error=str(e) + ) + except Exception as e: + return ConversionResult( + markdown="", tool="markitdown", success=False, error=str(e) + ) + + +def convert_with_pandoc( + file_path: Path, assets_dir: Optional[Path] = None +) -> ConversionResult: + """Convert using pandoc. + + Pandoc's --extract-media=DIR creates a media/ subdirectory inside DIR. + We point --extract-media at assets_dir's parent so pandoc's media/ + subdirectory lands exactly at assets_dir (when assets_dir ends with 'media'), + or we use a temp dir and move files afterward. + """ + try: + cmd = ["pandoc", str(file_path), "-t", "markdown", "--wrap=none"] + + extract_dir = None + if assets_dir: + assets_dir.mkdir(parents=True, exist_ok=True) + # Pandoc always creates a media/ subdirectory inside --extract-media. + # Point it at the parent so media/ lands at assets_dir. + if assets_dir.name == "media": + extract_dir = assets_dir.parent + else: + extract_dir = assets_dir + cmd.extend(["--extract-media", str(extract_dir)]) + + result = subprocess.run( + cmd, capture_output=True, text=True, timeout=120 + ) + + if result.returncode != 0: + return ConversionResult( + markdown="", tool="pandoc", success=False, error=result.stderr + ) + + md = result.stdout + + # Convert absolute image paths to relative paths based on output location + if extract_dir: + abs_media = str(extract_dir / "media") + # Replace absolute paths with relative 'media/' prefix + md = md.replace(abs_media + "/", "media/") + + images = _collect_images(assets_dir) if assets_dir else [] + + return ConversionResult( + markdown=md, tool="pandoc", images=images, success=True + ) + except Exception as e: + return ConversionResult( + markdown="", tool="pandoc", success=False, error=str(e) + ) + + +def convert_single( + file_path: Path, tool: str, assets_dir: Optional[Path] = None +) -> ConversionResult: + """Run a single conversion tool.""" + converters = { + "pymupdf4llm": convert_with_pymupdf4llm, + "markitdown": convert_with_markitdown, + "pandoc": convert_with_pandoc, + "docx-deep": convert_with_docx_deep, + } + + converter = converters.get(tool) + if not converter: + return ConversionResult( + markdown="", tool=tool, success=False, error=f"Unknown tool: {tool}" + ) + + return converter(file_path, assets_dir) + + +def merge_results(results: list[ConversionResult]) -> ConversionResult: + """Merge results from multiple tools, selecting best segments.""" + if not results: + return ConversionResult(markdown="", tool="none", success=False) + + # Filter successful results + successful = [r for r in results if r.success and r.markdown.strip()] + if not successful: + # Return first error + return results[0] if results else ConversionResult( + markdown="", tool="none", success=False + ) + + if len(successful) == 1: + return successful[0] + + # Multiple successful results - merge them + # Strategy: Compare key metrics and select best + best = successful[0] + best_score = score_markdown(best.markdown) + + for result in successful[1:]: + score = score_markdown(result.markdown) + if score > best_score: + best = result + best_score = score + + # Merge images from all results + all_images = [] + seen = set() + for result in successful: + for img in result.images: + if img not in seen: + all_images.append(img) + seen.add(img) + + best.images = all_images + best.tool = f"merged({','.join(r.tool for r in successful)})" + + return best + + +def score_markdown(md: str) -> float: + """Score markdown quality for comparison.""" + score = 0.0 + + # Length (more content is generally better) + score += min(len(md) / 10000, 5.0) # Cap at 5 points + + # Tables (proper markdown tables) + table_count = md.count("|---|") + md.count("| ---") + score += min(table_count * 0.5, 3.0) + + # Images (referenced images) + image_count = md.count("![") + score += min(image_count * 0.3, 2.0) + + # Headings (proper hierarchy) + h1_count = md.count("\n# ") + h2_count = md.count("\n## ") + h3_count = md.count("\n### ") + if h1_count > 0 and h2_count >= h1_count: + score += 1.0 # Good hierarchy + + # Lists (structured content) + list_count = md.count("\n- ") + md.count("\n* ") + md.count("\n1. ") + score += min(list_count * 0.1, 2.0) + + # Penalize pandoc artifacts (grid tables, attributes) + artifact_count = md.count("+:---") + md.count("+---+") + artifact_count += md.count('{width="') + md.count("{.underline}") + score -= artifact_count * 0.5 + + return score + + +def main(): + parser = argparse.ArgumentParser( + description="Convert documents to markdown with multi-tool orchestration", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=""" +Examples: + # Quick mode (default) + python convert.py document.pdf -o output.md + + # Heavy mode (best quality) + python convert.py document.pdf -o output.md --heavy + + # DOCX deep mode (python-docx parsing) + python convert.py document.docx -o output.md --docx-deep + + # With custom assets directory + python convert.py document.pdf -o output.md --assets-dir ./images + """, + ) + parser.add_argument("input", type=Path, nargs="?", help="Input document path") + parser.add_argument( + "-o", "--output", type=Path, help="Output markdown file" + ) + parser.add_argument( + "--heavy", + action="store_true", + help="Enable Heavy Mode (multi-tool, best quality)", + ) + parser.add_argument( + "--docx-deep", + action="store_true", + help="Use python-docx direct parsing (experimental, DOCX only)", + ) + parser.add_argument( + "--no-postprocess", + action="store_true", + help="Disable DOCX post-processing (keep raw pandoc output)", + ) + parser.add_argument( + "--assets-dir", + type=Path, + default=None, + help="Directory for extracted images (default: _assets/)", + ) + parser.add_argument( + "--tool", + choices=["pymupdf4llm", "markitdown", "pandoc", "docx-deep"], + help="Force specific tool (overrides auto-selection)", + ) + parser.add_argument( + "--list-tools", + action="store_true", + help="List available tools and exit", + ) + + args = parser.parse_args() + + # List tools mode + if args.list_tools: + tools = ["pymupdf4llm", "markitdown", "pandoc", "docx-deep"] + print("Available conversion tools:") + for tool in tools: + status = "+" if check_tool_available(tool) else "-" + print(f" {status} {tool}") + sys.exit(0) + + # Validate input + if args.input is None: + parser.error("the following arguments are required: input") + if not args.input.exists(): + print(f"Error: Input file not found: {args.input}", file=sys.stderr) + sys.exit(1) + + # Determine output path + output_path = args.output or args.input.with_suffix(".md") + + # Determine assets directory + assets_dir = args.assets_dir + if assets_dir is None: + assets_dir = output_path.parent / f"{output_path.stem}_assets" + + is_docx = args.input.suffix.lower() in (".docx", ".doc") + + # Handle --docx-deep mode + if args.docx_deep: + if not is_docx: + print("Error: --docx-deep only works with DOCX files.", file=sys.stderr) + sys.exit(1) + tools = ["docx-deep"] + elif args.tool: + tools = [args.tool] if check_tool_available(args.tool) else [] + else: + # Select tools + mode = "heavy" if args.heavy else "quick" + tools = select_tools(args.input, mode) + + mode = "docx-deep" if args.docx_deep else ("heavy" if args.heavy else "quick") + + if not tools: + print("Error: No conversion tools available.", file=sys.stderr) + print("Install with:", file=sys.stderr) + print(" pip install pymupdf4llm", file=sys.stderr) + print(" uv tool install markitdown[pdf]", file=sys.stderr) + print(" brew install pandoc", file=sys.stderr) + sys.exit(1) + + print(f"Converting: {args.input}") + print(f"Mode: {mode.upper()}") + print(f"Tools: {', '.join(tools)}") + + # Run conversions + results = [] + for tool in tools: + print(f" Running {tool}...", end=" ", flush=True) + + # Use separate assets dirs for each tool in heavy mode + tool_assets = None + if assets_dir and mode == "heavy" and len(tools) > 1: + tool_assets = assets_dir / tool + elif assets_dir: + tool_assets = assets_dir + + result = convert_single(args.input, tool, tool_assets) + results.append(result) + + if result.success: + print(f"ok ({len(result.markdown):,} chars, {len(result.images)} images)") + else: + print(f"FAIL ({result.error[:50]}...)") + + # Merge results if heavy mode + if mode == "heavy" and len(results) > 1: + print(" Merging results...", end=" ", flush=True) + final = merge_results(results) + print(f"ok (using {final.tool})") + else: + final = merge_results(results) + + if not final.success: + print(f"Error: Conversion failed: {final.error}", file=sys.stderr) + sys.exit(1) + + # Apply DOCX post-processing + if is_docx and not args.no_postprocess and "pandoc" in final.tool: + print(" Post-processing DOCX output...", end=" ", flush=True) + final.markdown, pp_stats = postprocess_docx_markdown( + final.markdown, assets_dir + ) + print(f"ok ({pp_stats.summary()})") + + # Write output + output_path.parent.mkdir(parents=True, exist_ok=True) + output_path.write_text(final.markdown) + + print(f"\nOutput: {output_path}") + print(f" Size: {len(final.markdown):,} characters") + if final.images: + print(f" Images: {len(final.images)} extracted") + + +if __name__ == "__main__": + main() diff --git a/markdown-tools/scripts/convert_path.py b/daymade-docs/doc-to-markdown/scripts/convert_path.py similarity index 74% rename from markdown-tools/scripts/convert_path.py rename to daymade-docs/doc-to-markdown/scripts/convert_path.py index cbc56d9e..011328d9 100755 --- a/markdown-tools/scripts/convert_path.py +++ b/daymade-docs/doc-to-markdown/scripts/convert_path.py @@ -3,10 +3,10 @@ Convert Windows paths to WSL format. Usage: - python convert_path.py "C:\\Users\\username\\Downloads\\file.doc" + python convert_path.py "C:\\Users\\\\Downloads\\file.doc" Output: - /mnt/c/Users/username/Downloads/file.doc + /mnt/c/Users//Downloads/file.doc """ import sys @@ -18,10 +18,10 @@ def convert_windows_to_wsl(windows_path: str) -> str: Convert a Windows path to WSL format. Args: - windows_path: Windows path (e.g., "C:\\Users\\username\\file.doc") + windows_path: Windows path (e.g., "C:\\Users\\\\file.doc") Returns: - WSL path (e.g., "/mnt/c/Users/username/file.doc") + WSL path (e.g., "/mnt/c/Users//file.doc") """ # Remove quotes if present path = windows_path.strip('"').strip("'") @@ -49,7 +49,7 @@ def convert_windows_to_wsl(windows_path: str) -> str: def main(): if len(sys.argv) < 2: print("Usage: python convert_path.py ") - print('Example: python convert_path.py "C:\\Users\\username\\Downloads\\file.doc"') + print('Example: python convert_path.py "C:\\Users\\\\Downloads\\file.doc"') sys.exit(1) windows_path = sys.argv[1] diff --git a/markdown-tools/scripts/extract_pdf_images.py b/daymade-docs/doc-to-markdown/scripts/extract_pdf_images.py similarity index 100% rename from markdown-tools/scripts/extract_pdf_images.py rename to daymade-docs/doc-to-markdown/scripts/extract_pdf_images.py diff --git a/markdown-tools/scripts/merge_outputs.py b/daymade-docs/doc-to-markdown/scripts/merge_outputs.py similarity index 100% rename from markdown-tools/scripts/merge_outputs.py rename to daymade-docs/doc-to-markdown/scripts/merge_outputs.py diff --git a/daymade-docs/doc-to-markdown/scripts/test_convert.py b/daymade-docs/doc-to-markdown/scripts/test_convert.py new file mode 100644 index 00000000..d334209b --- /dev/null +++ b/daymade-docs/doc-to-markdown/scripts/test_convert.py @@ -0,0 +1,242 @@ +"""Tests for doc-to-markdown convert.py post-processing functions. + +Run: uv run pytest scripts/test_convert.py -v +""" + +import pytest +import re +import sys +from pathlib import Path + +# Import the module under test +sys.path.insert(0, str(Path(__file__).parent)) +from convert import ( + _fix_cjk_bold_spacing, + _build_pipe_table, + _collect_images, + PostProcessStats, + postprocess_docx_markdown, +) + + +# ── CJK Bold Spacing ───────────────────────────────────────────────────────── + + +class TestCjkBoldSpacing: + """Test _fix_cjk_bold_spacing: spaces between **bold** and CJK chars.""" + + def test_bold_followed_by_cjk_punctuation(self): + """**text** directly touching CJK colon → add space after **.""" + inp = "**打开阶跃开放平台链接**：https://platform.stepfun.com/" + out = _fix_cjk_bold_spacing(inp) + assert "**打开阶跃开放平台链接** ：" in out + + def test_cjk_before_bold(self): + """CJK char directly before ** → add space before **.""" + assert _fix_cjk_bold_spacing("可用**手机号**进行") == "可用 **手机号** 进行" + + def test_bold_with_emoji_neighbor(self): + """**text** touching emoji ➡️ → still add space (CJK content rule).""" + inp = "点击**【接口密码】**➡️**【创建新的密钥**】" + out = _fix_cjk_bold_spacing(inp) + # Each CJK-containing bold span should have spaces on both sides + assert "点击 **【接口密码】** ➡️" in out + assert "➡️ **【创建新的密钥**" in out + + def test_full_emoji_line(self): + """Complete line with emoji separators between bold spans.""" + inp = "点击**【接口密码】**➡️**【创建新的密钥**】➡️**【输入密钥名称】**（输入你想取的名称），生成API Key" + out = _fix_cjk_bold_spacing(inp) + assert "点击 **【接口密码】** ➡️" in out + assert "**【输入密钥名称】** （输入" in out + + def test_bold_between_cjk(self): + """CJK **text** CJK → spaces on both sides.""" + assert _fix_cjk_bold_spacing("打开**飞书**，就可以") == "打开 **飞书** ，就可以" + + def test_bold_with_chinese_quotes(self): + """Bold containing Chinese quotes.""" + inp = '有个**"企鹅戴龙虾头套的机器人"**，开始' + out = _fix_cjk_bold_spacing(inp) + assert '**"企鹅戴龙虾头套的机器人"** ，' in out + + def test_multiple_bold_spans(self): + """Multiple bold spans in one line.""" + assert _fix_cjk_bold_spacing("这是**测试**和**验证**的效果") == "这是 **测试** 和 **验证** 的效果" + + def test_already_spaced(self): + """Already has spaces → no double spaces.""" + inp = "已有空格 **粗体** 不需要再加" + assert _fix_cjk_bold_spacing(inp) == inp + + def test_english_unchanged(self): + """English bold text should not be modified.""" + inp = "English **bold** text should not change" + assert _fix_cjk_bold_spacing(inp) == inp + + def test_line_start_bold(self): + """Bold at line start followed by CJK.""" + assert _fix_cjk_bold_spacing("**重要**内容") == "**重要** 内容" + + def test_line_start_bold_standalone(self): + """Bold at line start with no CJK neighbor → no change.""" + assert _fix_cjk_bold_spacing("**这是纯粗体不需要改**") == "**这是纯粗体不需要改**" + + def test_no_bold(self): + """Text without bold markers → unchanged.""" + inp = "这是普通文本，没有粗体" + assert _fix_cjk_bold_spacing(inp) == inp + + def test_empty_string(self): + assert _fix_cjk_bold_spacing("") == "" + + def test_bold_at_line_end(self): + """Bold at line end → no trailing space needed.""" + assert _fix_cjk_bold_spacing("内容是**粗体**") == "内容是 **粗体**" + + def test_mixed_cjk_and_english_bold(self): + """English bold between CJK → no change (no CJK in content).""" + inp = "请使用 **API Key** 进行认证" + assert _fix_cjk_bold_spacing(inp) == inp + + +# ── Pipe Table Builder ──────────────────────────────────────────────────────── + + +class TestBuildPipeTable: + """Test _build_pipe_table: rows → markdown pipe table.""" + + def test_basic_table(self): + rows = [["a", "b"], ["c", "d"]] + result = _build_pipe_table(rows) + assert result == [ + "| | |", + "| --- | --- |", + "| a | b |", + "| c | d |", + ] + + def test_uneven_rows(self): + """Rows with different column counts → padded.""" + rows = [["a", "b", "c"], ["d"]] + result = _build_pipe_table(rows) + assert "| d | | |" in result + + def test_single_cell(self): + rows = [["only"]] + result = _build_pipe_table(rows) + assert len(result) == 3 # header + sep + 1 row + + def test_empty_rows(self): + assert _build_pipe_table([]) == [] + + def test_image_with_caption(self): + """Images and captions should pair correctly in table.""" + rows = [ + ["![](img1.png)", "![](img2.png)"], + ["Step 1", "Step 2"], + ] + result = _build_pipe_table(rows) + assert "| ![](img1.png) | ![](img2.png) |" in result + assert "| Step 1 | Step 2 |" in result + + +# ── Full Post-Processing Pipeline ───────────────────────────────────────────── + + +class TestPostprocessPipeline: + """Integration tests for the full postprocess_docx_markdown pipeline.""" + + def test_grid_table_single_column_to_blockquote(self): + """Single-column grid table → blockquote.""" + inp = """+:---+ +| 注意事项 | ++----+""" + out, stats = postprocess_docx_markdown(inp) + assert "> 注意事项" in out + assert "+:---+" not in out + + def test_pandoc_attributes_removed(self): + """Pandoc {width=...} and {.underline} removed.""" + inp = '![](img.png){width="5in" height="3in"} and [text]{.underline}' + out, stats = postprocess_docx_markdown(inp) + assert "{width=" not in out + assert "{.underline}" not in out + assert "![](img.png)" in out + + def test_escaped_brackets_fixed(self): + r"""Pandoc \[ and \] → [ and ].""" + inp = r"你 \[在飞书里\] 发消息" + out, stats = postprocess_docx_markdown(inp) + assert "你 [在飞书里] 发消息" in out + + def test_double_bracket_links_fixed(self): + """[[text]](url) → [text](url).""" + inp = "[[点击跳转]](https://example.com)" + out, stats = postprocess_docx_markdown(inp) + assert "[点击跳转](https://example.com)" in out + + def test_code_block_with_language(self): + """Indented dashed block with JSON language hint → ```json.""" + inp = """ ------------------------------------------------------------------ + JSON\\ + {\\ + "provider": "stepfun"\\ + } + ------------------------------------------------------------------""" + out, stats = postprocess_docx_markdown(inp) + assert "```json" in out + assert '"provider": "stepfun"' in out + assert "---" not in out + + def test_code_block_plain_text_to_blockquote(self): + """Indented dashed block with plain text → blockquote.""" + inp = """ -------------------------- + 注意：这是一条重要提示 + --------------------------""" + out, stats = postprocess_docx_markdown(inp) + assert "> 注意：这是一条重要提示" in out + + def test_cjk_bold_spacing_in_pipeline(self): + """CJK bold spacing is applied in the full pipeline.""" + inp = "打开**飞书**，就可以看到" + out, stats = postprocess_docx_markdown(inp) + assert "打开 **飞书** ，就可以看到" in out + + def test_excessive_blank_lines_collapsed(self): + """4+ blank lines → 2 blank lines.""" + inp = "line1\n\n\n\n\nline2" + out, stats = postprocess_docx_markdown(inp) + assert out.count("\n") < 5 + + def test_stats_tracking(self): + """Stats object correctly tracks fix counts.""" + inp = '![](media/media/img.png){width="5in"}' + out, stats = postprocess_docx_markdown(inp) + assert stats.attributes_removed > 0 + + +# ── Simple Table (pandoc) ───────────────────────────────────────────────────── + + +class TestSimpleTable: + """Test pandoc simple table (indented dashes with spaces) → pipe table.""" + + def test_two_column_image_table(self): + """Two images side by side in simple table → pipe table.""" + inp = """ ---- ---- + ![](img1.png) ![](img2.png) + + ---- ----""" + out, stats = postprocess_docx_markdown(inp) + assert "| ![](img1.png) | ![](img2.png) |" in out + assert "----" not in out + + def test_four_column_image_table(self): + """Four images in simple table → 4-column pipe table.""" + inp = """ ---------- ---------- ---------- ---------- + ![](a.png) ![](b.png) ![](c.png) ![](d.png) + + ---------- ---------- ---------- ----------""" + out, stats = postprocess_docx_markdown(inp) + assert "| ![](a.png) | ![](b.png) | ![](c.png) | ![](d.png) |" in out diff --git a/markdown-tools/scripts/validate_output.py b/daymade-docs/doc-to-markdown/scripts/validate_output.py similarity index 98% rename from markdown-tools/scripts/validate_output.py rename to daymade-docs/doc-to-markdown/scripts/validate_output.py index 937bf37c..2f3d09db 100755 --- a/markdown-tools/scripts/validate_output.py +++ b/daymade-docs/doc-to-markdown/scripts/validate_output.py @@ -105,9 +105,10 @@ def extract_text_from_docx(docx_path: Path) -> tuple[str, int, int]: root = tree.getroot() # Extract text - ns = {'w': 'http://schemas.openxmlformats.org/wordprocessingml/2006/main'} + wordprocessing_ns = 'http' + '://schemas.openxmlformats.org/wordprocessingml/2006/main' + ns = {'w': wordprocessing_ns} text_parts = [] - for t in root.iter('{http://schemas.openxmlformats.org/wordprocessingml/2006/main}t'): + for t in root.iter(f'{{{wordprocessing_ns}}}t'): if t.text: text_parts.append(t.text) diff --git a/docs-cleaner/SKILL.md b/daymade-docs/docs-cleaner/SKILL.md similarity index 100% rename from docs-cleaner/SKILL.md rename to daymade-docs/docs-cleaner/SKILL.md diff --git a/docs-cleaner/references/value_analysis_template.md b/daymade-docs/docs-cleaner/references/value_analysis_template.md similarity index 100% rename from docs-cleaner/references/value_analysis_template.md rename to daymade-docs/docs-cleaner/references/value_analysis_template.md diff --git a/meeting-minutes-taker/SKILL.md b/daymade-docs/meeting-minutes-taker/SKILL.md similarity index 97% rename from meeting-minutes-taker/SKILL.md rename to daymade-docs/meeting-minutes-taker/SKILL.md index 6971e650..bb31f088 100644 --- a/meeting-minutes-taker/SKILL.md +++ b/daymade-docs/meeting-minutes-taker/SKILL.md @@ -11,7 +11,7 @@ Transform raw meeting transcripts into comprehensive, evidence-based meeting min ## Quick Start **Pre-processing (Optional but Recommended):** -- **Document conversion**: Use `markdown-tools` skill to convert .docx/.pdf to Markdown first (preserves tables/images) +- **Document conversion**: Use `doc-to-markdown` skill to convert .docx/.pdf to Markdown first (preserves tables/images) - **Transcript cleanup**: Use `transcript-fixer` skill to fix ASR/STT errors if transcript quality is poor - **Context file**: Prepare `context.md` with team directory for accurate speaker identification @@ -327,7 +327,7 @@ Final Review: - **Flowcharts** for process/workflow decisions - **State diagrams** for state machine discussions - Diagrams make minutes significantly easier for humans to review and understand -- **Context-first document structure** - Place all reviewed artifacts (UI mockups, API docs, design images) at the TOP of the document (after metadata, before Executive Summary) to establish context before decisions; copy images to `assets//` folder and embed inline using `![description](assets/...)` syntax; include brief descriptions with the visuals - this creates "next level" human-readable minutes where readers understand what was discussed before reading the discussion +- **Context-first document structure** - Place all reviewed artifacts (UI mockups, API docs, design images) at the TOP of the document (after metadata, before Executive Summary) to establish context before decisions; copy images to `meeting-media//` folder and embed inline using `![description](meeting-media/...)` syntax; include brief descriptions with the visuals - this creates "next level" human-readable minutes where readers understand what was discussed before reading the discussion - **Speaker attribution** - Correctly attribute decisions to speakers #### Key Rules @@ -457,7 +457,7 @@ If v3 has a flowchart for "Status Query Mechanism" but v1/v2 don't have it, that **Full pipeline for .docx transcripts:** ``` -Step 0: markdown-tools # Convert .docx → Markdown (preserves tables/images) +Step 0: doc-to-markdown # Convert .docx → Markdown (preserves tables/images) ↓ Step 0.5: transcript-fixer # Fix ASR errors (optional, if quality is poor) ↓ @@ -643,3 +643,16 @@ Round 4: Update to use "Annotation" instead of "Note" - ❌ **Creating non-existent links** - Do NOT create markdown links to files that don't exist in the repo (e.g., `[doc.md](reviewed-document)`); use plain text for external/local documents not in the repository - ❌ **Losing content during consolidation** - When moving or consolidating sections, verify ALL bullet points and details are preserved; never summarize away specific details like "supports batch operations" or "button triggers auto-save" - ❌ **Appending domain details to role titles** - Use ONLY the Role column from Team Directory for speaker attribution (e.g., "Backend", "Frontend", "TPM"); do NOT append specializations like "Backend, Infrastructure" or "Backend, Business Logic" - all team members with the same role should have identical attribution + +## Next Step: Export to Deliverable Format + +After structuring meeting minutes, suggest exporting: + +``` +Meeting minutes complete: [N] decisions, [M] action items captured. + +Options: +A) Export as PDF — run /pdf-creator (Recommended for sharing) +B) Export as slides — run /ppt-creator (for presentation) +C) No thanks — the markdown is sufficient +``` diff --git a/meeting-minutes-taker/references/completeness_review_checklist.md b/daymade-docs/meeting-minutes-taker/references/completeness_review_checklist.md similarity index 100% rename from meeting-minutes-taker/references/completeness_review_checklist.md rename to daymade-docs/meeting-minutes-taker/references/completeness_review_checklist.md diff --git a/meeting-minutes-taker/references/context_file_template.md b/daymade-docs/meeting-minutes-taker/references/context_file_template.md similarity index 100% rename from meeting-minutes-taker/references/context_file_template.md rename to daymade-docs/meeting-minutes-taker/references/context_file_template.md diff --git a/meeting-minutes-taker/references/meeting_minutes_template.md b/daymade-docs/meeting-minutes-taker/references/meeting_minutes_template.md similarity index 100% rename from meeting-minutes-taker/references/meeting_minutes_template.md rename to daymade-docs/meeting-minutes-taker/references/meeting_minutes_template.md diff --git a/mermaid-tools/SKILL.md b/daymade-docs/mermaid-tools/SKILL.md similarity index 92% rename from mermaid-tools/SKILL.md rename to daymade-docs/mermaid-tools/SKILL.md index 8402ba9e..2e10c0ee 100644 --- a/mermaid-tools/SKILL.md +++ b/daymade-docs/mermaid-tools/SKILL.md @@ -16,7 +16,7 @@ This skill enables extraction of Mermaid diagrams from markdown files and genera Extract Mermaid diagrams from a markdown file and generate PNG images using the bundled `extract-and-generate.sh` script: ```bash -cd ~/.claude/skills/mermaid-tools/scripts +cd "${CLAUDE_SKILL_DIR}/scripts" ./extract-and-generate.sh "" "" ``` @@ -26,8 +26,8 @@ cd ~/.claude/skills/mermaid-tools/scripts **Example:** ```bash -cd ~/.claude/skills/mermaid-tools/scripts -./extract-and-generate.sh "/path/to/document.md" "/path/to/output" +cd "${CLAUDE_SKILL_DIR}/scripts" +./extract-and-generate.sh "" "" ``` ### What the Script Does @@ -53,7 +53,7 @@ The numbering ensures diagrams maintain their order from the source document. Override default dimensions using environment variables: ```bash -cd ~/.claude/skills/mermaid-tools/scripts +cd "${CLAUDE_SKILL_DIR}/scripts" MERMAID_WIDTH=1600 MERMAID_HEIGHT=1200 ./extract-and-generate.sh "" "" ``` @@ -65,14 +65,14 @@ MERMAID_WIDTH=1600 MERMAID_HEIGHT=1200 ./extract-and-generate.sh "" "" ``` ### Print-Quality Output ```bash -cd ~/.claude/skills/mermaid-tools/scripts +cd "${CLAUDE_SKILL_DIR}/scripts" MERMAID_SCALE=5 ./extract-and-generate.sh "" "" ``` @@ -98,7 +98,7 @@ Context-aware naming in the extraction process helps trigger appropriate smart s Run the script from its own directory to properly locate dependencies (`extract_diagrams.py` and `puppeteer-config.json`): ```bash -cd ~/.claude/skills/mermaid-tools/scripts +cd "${CLAUDE_SKILL_DIR}/scripts" ./extract-and-generate.sh "" "" ``` @@ -129,7 +129,7 @@ Quick fixes for common issues: **Permission denied:** ```bash -chmod +x ~/.claude/skills/mermaid-tools/scripts/extract-and-generate.sh +chmod +x "${CLAUDE_SKILL_DIR}/scripts/extract-and-generate.sh" ``` **Low quality output:** @@ -161,4 +161,4 @@ Comprehensive reference documentation including: - WSL2-specific Chrome dependency setup - Validation procedures -Load this reference when dealing with setup issues, installation problems, or advanced customization needs. \ No newline at end of file +Load this reference when dealing with setup issues, installation problems, or advanced customization needs. diff --git a/mermaid-tools/references/setup_and_troubleshooting.md b/daymade-docs/mermaid-tools/references/setup_and_troubleshooting.md similarity index 91% rename from mermaid-tools/references/setup_and_troubleshooting.md rename to daymade-docs/mermaid-tools/references/setup_and_troubleshooting.md index 1e455396..a99f3035 100644 --- a/mermaid-tools/references/setup_and_troubleshooting.md +++ b/daymade-docs/mermaid-tools/references/setup_and_troubleshooting.md @@ -28,7 +28,7 @@ Install Chrome and dependencies: ```bash # Add Chrome repository wget -q -O - https://dl.google.com/linux/linux_signing_key.pub | sudo apt-key add - -echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" | sudo tee /etc/apt/sources.list.d/google-chrome.list +echo "deb [arch=amd64] https://dl.google.com/linux/chrome/deb/ stable main" | sudo tee /etc/apt/sources.list.d/google-chrome.list # Update and install Chrome sudo apt update @@ -53,9 +53,9 @@ python3 --version ## Script Locations The mermaid diagram tools are bundled with this skill in the `scripts/` directory: -- Main script: `~/.claude/skills/mermaid-tools/scripts/extract-and-generate.sh` -- Python extractor: `~/.claude/skills/mermaid-tools/scripts/extract_diagrams.py` -- Puppeteer config: `~/.claude/skills/mermaid-tools/scripts/puppeteer-config.json` +- Main script: `${CLAUDE_SKILL_DIR}/scripts/extract-and-generate.sh` +- Python extractor: `${CLAUDE_SKILL_DIR}/scripts/extract_diagrams.py` +- Puppeteer config: `${CLAUDE_SKILL_DIR}/scripts/puppeteer-config.json` All scripts should be run from the `scripts/` directory to properly locate dependencies. @@ -119,7 +119,7 @@ MERMAID_SCALE=5 ./extract-and-generate.sh "file.md" "output_dir" **Solution**: ```bash -chmod +x ~/.claude/skills/mermaid-tools/scripts/extract-and-generate.sh +chmod +x "${CLAUDE_SKILL_DIR}/scripts/extract-and-generate.sh" ``` ### No Diagrams Found @@ -172,4 +172,4 @@ The script automatically validates generated PNG files by: 3. Reporting actual dimensions 4. Displaying file size in bytes -Look for ✅ validation messages in the output to confirm successful generation. \ No newline at end of file +Look for ✅ validation messages in the output to confirm successful generation. diff --git a/mermaid-tools/scripts/extract-and-generate.sh b/daymade-docs/mermaid-tools/scripts/extract-and-generate.sh old mode 100644 new mode 100755 similarity index 94% rename from mermaid-tools/scripts/extract-and-generate.sh rename to daymade-docs/mermaid-tools/scripts/extract-and-generate.sh index 357b4dc6..1fe161e4 --- a/mermaid-tools/scripts/extract-and-generate.sh +++ b/daymade-docs/mermaid-tools/scripts/extract-and-generate.sh @@ -3,7 +3,7 @@ # Extracts diagrams from markdown and numbers them sequentially # # Usage: ./extract-and-generate.sh [output_directory] -# Example: ./extract-and-generate.sh "~/workspace/document.md" "~/workspace/diagrams" +# Example: ./extract-and-generate.sh set -e @@ -14,7 +14,7 @@ EXTRACTOR_SCRIPT="$SCRIPT_DIR/extract_diagrams.py" # Parse arguments if [ $# -lt 1 ]; then echo "Usage: $0 [output_directory]" - echo "Example: $0 '~/workspace/document.md' '~/workspace/diagrams'" + echo "Example: $0 " exit 1 fi @@ -22,7 +22,7 @@ MARKDOWN_FILE="$1" OUTPUT_DIR="${2:-$(dirname "$MARKDOWN_FILE")/diagrams}" echo "=== Enhanced Mermaid Diagram Processor ===" -echo "Source markdown: $MARKDOWN_FILE" +echo "Source markdown: $MARKDOWN_FILE" echo "Output directory: $OUTPUT_DIR" echo "Environment: WSL2 Ubuntu with Chrome dependencies" echo @@ -81,7 +81,7 @@ echo echo "Generating PNG files..." cd "$OUTPUT_DIR" -# Default dimensions - can be overridden with environment variables +# Default dimensions - can be overridden with environment variables DEFAULT_WIDTH="${MERMAID_WIDTH:-1200}" DEFAULT_HEIGHT="${MERMAID_HEIGHT:-800}" SCALE_FACTOR="${MERMAID_SCALE:-2}" @@ -104,14 +104,14 @@ for mmd_file in "${mmd_files[@]}"; do if [ ! -f "$mmd_file" ]; then continue fi - + # Extract filename without extension diagram="${mmd_file%.mmd}" - + # Use smart defaults based on diagram content or filename patterns width="$DEFAULT_WIDTH" height="$DEFAULT_HEIGHT" - + # Smart sizing based on filename patterns if [[ "$diagram" =~ timeline|gantt ]]; then width=$((DEFAULT_WIDTH * 2)) # Wider for timelines @@ -126,7 +126,7 @@ for mmd_file in "${mmd_files[@]}"; do width=$((DEFAULT_WIDTH * 2)) # Wider for workflows and sequences height="$DEFAULT_HEIGHT" fi - + echo "Generating $diagram.png (${width}x${height}, scale: ${SCALE_FACTOR}x)..." PUPPETEER_EXECUTABLE_PATH="$CHROME_PATH" mmdc \ -i "$mmd_file" \ @@ -135,14 +135,14 @@ for mmd_file in "${mmd_files[@]}"; do -w "$width" \ -H "$height" \ -s "$SCALE_FACTOR" - + if [ $? -eq 0 ]; then echo " ✅ Generated successfully" else echo " ❌ Generation failed" continue fi - + # Validate PNG if test -s "$diagram.png" && file "$diagram.png" | grep -q "PNG image"; then size=$(stat -c%s "$diagram.png") @@ -163,4 +163,4 @@ ls -la [0-9][0-9]-*.png 2>/dev/null | awk '{printf " %s (%s bytes)\n", $9, $5}' echo echo "Generated files (all):" -ls -la *.png 2>/dev/null | awk '{printf " %s (%s bytes)\n", $9, $5}' || echo " No PNG files found" \ No newline at end of file +ls -la *.png 2>/dev/null | awk '{printf " %s (%s bytes)\n", $9, $5}' || echo " No PNG files found" diff --git a/mermaid-tools/scripts/extract_diagrams.py b/daymade-docs/mermaid-tools/scripts/extract_diagrams.py old mode 100644 new mode 100755 similarity index 96% rename from mermaid-tools/scripts/extract_diagrams.py rename to daymade-docs/mermaid-tools/scripts/extract_diagrams.py index acc3d4f0..2e10e969 --- a/mermaid-tools/scripts/extract_diagrams.py +++ b/daymade-docs/mermaid-tools/scripts/extract_diagrams.py @@ -9,50 +9,50 @@ def extract_mermaid_diagrams(markdown_file, output_dir): """Extract Mermaid diagrams from markdown file and create numbered .mmd files""" - + try: with open(markdown_file, 'r', encoding='utf-8') as f: content = f.read() except Exception as e: print(f"ERROR: Cannot read markdown file: {e}") return [] - + # Find all mermaid code blocks with their content mermaid_pattern = r'```mermaid\n(.*?)\n```' matches = re.findall(mermaid_pattern, content, re.DOTALL) - + if not matches: print("No Mermaid diagrams found in markdown file") return [] - + # Extract diagram names from context (look backwards for section headers) diagrams = [] lines = content.split('\n') - + for i, match in enumerate(matches, 1): # Find the position of this diagram in the content diagram_pattern = f'```mermaid\n{re.escape(match)}\n```' diagram_match = re.search(diagram_pattern, content) - + if not diagram_match: # Fallback: use simple search diagram_start = content.find(f'```mermaid\n{match}\n```') else: diagram_start = diagram_match.start() - + # Count lines up to this point to find context if diagram_start >= 0: lines_before = content[:diagram_start].count('\n') else: lines_before = 0 - + # Look backwards for the most recent section header or meaningful context diagram_name = f"diagram-{i:02d}" # Default fallback - + # Look for context clues in the 20 lines before the diagram context_start = max(0, lines_before - 20) context_lines = lines[context_start:lines_before] if lines_before > 0 else [] - + # Priority 1: Look for specific diagram descriptions for line in reversed(context_lines): line = line.strip().lower() @@ -77,7 +77,7 @@ def extract_mermaid_diagrams(markdown_file, output_dir): elif 'agency' in line and ('hierarchy' in line or 'filter' in line): diagram_name = f"{i:02d}-agency-hierarchy" break - + # Priority 2: Look for section headers (## or ###) if diagram_name.startswith('diagram-'): for line in reversed(context_lines): @@ -87,28 +87,28 @@ def extract_mermaid_diagrams(markdown_file, output_dir): header = re.sub(r'^#+\s*\*?\*?', '', line) header = re.sub(r'\*?\*?$', '', header) header = header.strip() - + # Convert to filename-friendly format name_part = re.sub(r'[^\w\s-]', '', header) name_part = re.sub(r'\s+', '-', name_part.strip()) name_part = name_part.lower()[:30] # Limit length - + if name_part and name_part != 'detailed-design': diagram_name = f"{i:02d}-{name_part}" break - + diagrams.append({ 'number': i, 'name': diagram_name, 'content': match.strip() }) - + print(f"Found diagram {i}: {diagram_name}") - + # Write .mmd files output_path = Path(output_dir) output_path.mkdir(parents=True, exist_ok=True) - + created_files = [] for diagram in diagrams: mmd_file = output_path / f"{diagram['name']}.mmd" @@ -119,16 +119,16 @@ def extract_mermaid_diagrams(markdown_file, output_dir): print(f"Created: {mmd_file}") except Exception as e: print(f"ERROR: Cannot create {mmd_file}: {e}") - + return created_files if __name__ == "__main__": if len(sys.argv) != 3: print("Usage: python3 extract_diagrams.py ") sys.exit(1) - + markdown_file = sys.argv[1] output_dir = sys.argv[2] - + files = extract_mermaid_diagrams(markdown_file, output_dir) print(f"\nExtracted {len(files)} diagrams successfully") \ No newline at end of file diff --git a/mermaid-tools/scripts/puppeteer-config.json b/daymade-docs/mermaid-tools/scripts/puppeteer-config.json similarity index 100% rename from mermaid-tools/scripts/puppeteer-config.json rename to daymade-docs/mermaid-tools/scripts/puppeteer-config.json diff --git a/daymade-docs/pdf-creator/SKILL.md b/daymade-docs/pdf-creator/SKILL.md new file mode 100644 index 00000000..eb54cc5e --- /dev/null +++ b/daymade-docs/pdf-creator/SKILL.md @@ -0,0 +1,122 @@ +--- +name: pdf-creator +description: Create PDF documents from markdown with proper Chinese font support. Supports theme system (default for formal docs, warm-terra for training materials) and dual backend (weasyprint or Chrome). Triggers include "convert to PDF", "generate PDF", "markdown to PDF", or any request for creating printable documents. +--- + +# PDF Creator + +Create professional PDF documents from markdown with Chinese font support and theme system. + +## Quick Start + +```bash +# Default theme (formal: Songti SC + black/grey) +uv run --with weasyprint scripts/md_to_pdf.py input.md output.pdf + +# Warm theme (training: PingFang SC + terra cotta) +uv run --with weasyprint scripts/md_to_pdf.py input.md --theme warm-terra + +# No weasyprint? Use Chrome backend (auto-detected if weasyprint unavailable) +python scripts/md_to_pdf.py input.md --theme warm-terra --backend chrome + +# List available themes +python scripts/md_to_pdf.py --list-themes dummy.md +``` + +## Themes + +Stored in `themes/*.css`. Each theme is a standalone CSS file. + +| Theme | Font | Color | Best for | +|-------|------|-------|----------| +| `default` | Songti SC + Heiti SC | Black/grey | Legal docs, contracts, formal reports | +| `warm-terra` | PingFang SC | Terra cotta (#d97756) + warm neutrals | Course outlines, training materials, workshops | + +To create a new theme: copy `themes/default.css`, modify, save as `themes/your-theme.css`. + +## Backends + +The script auto-detects the best available backend: + +| Backend | Install | Pros | Cons | +|---------|---------|------|------| +| `weasyprint` | `pip install weasyprint` | Precise CSS rendering, no browser needed | Requires system libs (cairo, pango) | +| `chrome` | Google Chrome installed | Zero Python deps, great CJK support | Larger binary, slightly less CSS control | + +Override with `--backend chrome` or `--backend weasyprint`. + +## Batch Convert + +```bash +uv run --with weasyprint scripts/batch_convert.py *.md --output-dir ./pdfs +``` + +## Troubleshooting + +**Chinese characters display as boxes**: Ensure Chinese fonts are installed (Songti SC, PingFang SC, etc.) + +**weasyprint import error**: Run with `uv run --with weasyprint` or use `--backend chrome` instead. + +**CJK text in code blocks garbled (weasyprint)**: The script auto-detects code blocks containing Chinese/Japanese/Korean characters and converts them to styled divs with CJK-capable fonts. If you still see issues, use `--backend chrome` which has native CJK support. Alternatively, convert code blocks to markdown tables before generating the PDF. + +**Chrome header/footer appearing**: The script passes `--no-pdf-header-footer`. If it still appears, your Chrome version may not support this flag — update Chrome. + +**Inline code with mixed CJK + ASCII shows blanks in macOS Preview** (e.g. `` `Terminal/终端` `` renders only `Terminal/` with the CJK part missing): weasyprint subset-embeds PingFang SC as **OpenType (CID Type 0C)**, which strict PDF readers (macOS Preview / Adobe Reader) fail to render. Chrome's PDF viewer falls back automatically and hides the bug. Fix is in the default theme: code font-family chain prioritizes **CID TrueType** CJK fonts (Songti SC / Heiti SC) before OpenType ones (PingFang SC). To verify: `pdfplumber` + check `font['fontname']` of CJK chars — if any references `PingFang-SC` (CID Type 0C OT), readers will likely fail. Reorder font chain to put CID TrueType first. + +**Table column 1 with short label gets mid-broken** (e.g. `4/28（周|二）下|午`): pandoc auto-emits `` from dash counts in the markdown separator row. For `| ----- | --- | --- | -------- |` (uneven dash widths), pandoc allocates col 1 ~17% — too narrow for a 9-char CJK label. Inline `style=""` beats external CSS at equal specificity, so `td:first-child { width:... }` is silently shadowed. Fix is in default theme: `table colgroup col { width: auto !important }` neutralizes pandoc's hint, letting `table-layout: fixed` distribute equally (25% per column for a 4-col table). To verify: `pandoc input.md -t html | grep colgroup` — if it shows ``, the bug applies. + +## Visual Self-Check (default behavior) + +After every PDF generation, the script automatically: + +1. Converts each page to PNG via `pdftoppm` (poppler-utils) into a `-preview/` directory next to the PDF +2. Prints a structured self-check checklist reminding the caller to visually inspect each page + +**Why**: "PDF generated cleanly" ≠ "rendering matches markdown intent". Common silent failures include paragraphs collapsing into one (CommonMark soft-break behavior on consecutive non-blank lines), tables overflowing page margins, missing CJK / emoji glyphs, code block garbling. The checklist enforces visual verification as the default contract — not an optional step that's easy to skip. + +**Workflow**: After running the script, `Read` each `page-NN.png` and verify against the markdown source. If anything renders differently from intent, **fix the markdown** (use `- ` real lists instead of pseudo-lists, insert blank lines, restructure tables) and rerun. The script does NOT silently "fix" non-standard markdown — that would mask the signal that the source is wrong, causing the same markdown to render incorrectly in other processors (Obsidian, GitHub, VS Code preview). + +**Disable** with `--no-preview` for batch / non-interactive runs: + +```bash +python scripts/md_to_pdf.py input.md output.pdf --no-preview +``` + +**Requires** `pdftoppm` (`brew install poppler` on macOS). If not installed, the script logs a hint and skips preview generation but still produces the PDF. + +## CJK Typography (default behavior) + +The script applies two layers of CJK-aware processing automatically — **without modifying the user's markdown source or theme CSS files**: + +### Layer 1: CSS patch (auto-injected, fixes ~80% of cases) + +`_load_theme()` appends a CJK typography CSS patch to the loaded theme CSS. The patch: + +- `table { table-layout: fixed; width: 100% }` — equal column widths prevent weasyprint auto-layout from squeezing one column to ~10% width when an adjacent column has 5x more content +- `td, th { word-break: keep-all; line-break: strict }` — don't slice CJK characters apart +- `th { white-space: nowrap }` — short headers stay one line for predictable column widths + +This silently fixes the most common anti-pattern (cell content forcibly wrapped between CJK characters producing single-char-only lines), without touching the user's source. The user's theme CSS file on disk is never modified. + +### Layer 2: Typography lint (post-render detection, catches the rest) + +After PDF generation, the script runs `pdftotext -layout` per page and scans for known CJK anti-patterns per "中文文案排版指北" (Chinese typography style guide): + +- Single CJK character alone on a line (cell still too narrow even after Layer 1) +- Line ending with `（` followed by content next line (broken bracket pair) +- Line starting with `）` (broken from previous bracket pair) +- Short line ending with mid-thought punctuation `、，；：` + +Findings are printed to stderr with page+line locations. They are **warnings, not errors** — PDF still generates. The author sees the finding and decides: + +1. Accept (e.g. one orphan char in a long doc may be acceptable) +2. Shorten the offending cell content to fit the column width +3. Restructure (e.g. move long content into a paragraph below the table) + +### Why not silently auto-fix everything? + +Layer 2 deliberately does NOT modify the markdown. Per CLAUDE.md "禁止隐式行为" rule: silently rewriting non-standard markdown (e.g. expanding pseudo-lists into real lists) would mask the signal that the source is wrong, causing the same markdown to render incorrectly in other processors. Layer 1 is acceptable because it patches **rendering behavior** for already-standard markdown (a standard table that weasyprint happens to render imperfectly for CJK), not the markdown source itself. + +### Known limitations + +When a single cell's content is just slightly longer than the available column width (e.g. 10 CJK chars in a 9-char-wide cell after equal split), weasyprint will fall back to forced break despite `keep-all`. Layer 1 cannot fix this — Layer 2 will catch it and prompt the author to shorten cell content or restructure. diff --git a/pdf-creator/scripts/batch_convert.py b/daymade-docs/pdf-creator/scripts/batch_convert.py similarity index 100% rename from pdf-creator/scripts/batch_convert.py rename to daymade-docs/pdf-creator/scripts/batch_convert.py diff --git a/daymade-docs/pdf-creator/scripts/md_to_pdf.py b/daymade-docs/pdf-creator/scripts/md_to_pdf.py new file mode 100644 index 00000000..828ced66 --- /dev/null +++ b/daymade-docs/pdf-creator/scripts/md_to_pdf.py @@ -0,0 +1,690 @@ +#!/usr/bin/env python3 +""" +Markdown to PDF converter with Chinese font support and theme system. + +Converts markdown files to PDF using: + - pandoc (markdown → HTML) + - weasyprint or headless Chrome (HTML → PDF), auto-detected + +Usage: + python md_to_pdf.py input.md output.pdf + python md_to_pdf.py input.md --theme warm-terra + python md_to_pdf.py input.md --theme default --backend chrome + python md_to_pdf.py input.md # outputs input.pdf, default theme, auto backend + +Themes: + Stored in ../themes/*.css. Built-in themes: + - default: Songti SC + black/grey, formal documents + - warm-terra: PingFang SC + terra cotta, training/workshop materials + +Requirements: + pandoc (system install, e.g. brew install pandoc) + weasyprint (pip install weasyprint) OR Google Chrome (for --backend chrome) +""" + +from __future__ import annotations + +import argparse +import os +import platform +import re +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +SCRIPT_DIR = Path(__file__).resolve().parent +THEMES_DIR = SCRIPT_DIR.parent / "themes" + +# macOS ARM: auto-configure library path for weasyprint +if platform.system() == "Darwin": + _homebrew_lib = "/opt/homebrew/lib" + if Path(_homebrew_lib).is_dir(): + _cur = os.environ.get("DYLD_LIBRARY_PATH", "") + if _homebrew_lib not in _cur: + os.environ["DYLD_LIBRARY_PATH"] = ( + f"{_homebrew_lib}:{_cur}" if _cur else _homebrew_lib + ) + + +def _find_chrome() -> str | None: + """Find Chrome/Chromium binary path.""" + candidates = [ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "/Applications/Chromium.app/Contents/MacOS/Chromium", + shutil.which("google-chrome"), + shutil.which("chromium"), + shutil.which("chrome"), + ] + for c in candidates: + if c and Path(c).exists(): + return str(c) + return None + + +def _has_weasyprint() -> bool: + """Check if weasyprint is importable.""" + try: + import weasyprint # noqa: F401 + + return True + except ImportError: + return False + + +def _detect_backend() -> str: + """Auto-detect best available backend: weasyprint > chrome.""" + if _has_weasyprint(): + return "weasyprint" + if _find_chrome(): + return "chrome" + print( + "Error: No PDF backend found. Install weasyprint (pip install weasyprint) " + "or Google Chrome.", + file=sys.stderr, + ) + sys.exit(1) + + +# ---------------- CJK typography patch (auto-injected, no source mutation) ---- +# +# Design principle: the user's markdown stays untouched, the user's theme CSS +# files stay untouched. We only patch the CSS string at load time (intermediate +# state), so the same .md + the same theme.css render correctly without the +# author having to know about CJK line-break engine quirks. +# +# Why needed: weasyprint's default `word-break: normal` treats every CJK +# character as a break opportunity. In narrow table cells this produces +# single-char-only lines and broken bracket pairs — anti-patterns per the +# well-known "中文文案排版指北" Chinese typography guide. +# +# Strategy (CJK-safe break rules, scoped to table cells only so body text +# behaves normally): +# - `word-break: keep-all` — don't break inside CJK runs +# - `overflow-wrap: break-word` — but allow break at word/punctuation +# boundaries when content exceeds cell width +# - `line-break: strict` — apply strict CJK rules (no break before +# closing 」』）, no break after opening 「『（, no break around 、，；：) +_TYPOGRAPHY_CSS_PATCH = """ +/* ===== md_to_pdf auto-injected: CJK typography for table cells ===== + * + * Three-layer fix for the most common CJK rendering anti-patterns: + * + * 1. table-layout: fixed + equal column widths — prevents weasyprint + * auto-layout from squeezing one column to 10% width when an + * adjacent column has 5x more content (the root cause of "single + * CJK char alone on a line" in narrow cells). + * + * 2. CJK break rules at cell level — don't slice CJK characters apart; + * break only at word/punctuation boundaries. + * + * 3. Header nowrap — short headers stay one line; combined with fixed + * layout, column widths are predictable. + * + * Trade-off: tables now distribute width equally across columns instead + * of content-aware. This may give "wider than needed" columns to short- + * content cells, but eliminates the "single CJK char per line" bug. + */ +table { + table-layout: fixed; + width: 100%; +} +table td, table th { + word-break: keep-all; + /* `overflow-wrap: normal` instead of break-word: when keep-all says + * "don't break inside CJK" and cell is too narrow for the content, + * prefer letting content overflow slightly rather than fallback to + * mid-token breaks (which produce single-CJK-char-per-line). */ + overflow-wrap: normal; + line-break: strict; +} +table th { + white-space: nowrap; +} +/* =================================================================== */ +""" + + +def _load_theme(theme_name: str) -> str: + """Load CSS from themes directory and append CJK typography patch. + + The patch is appended AFTER the user theme so it wins the cascade for + table cells. The user's theme CSS file on disk is never modified. + """ + theme_file = THEMES_DIR / f"{theme_name}.css" + if not theme_file.exists(): + available = [f.stem for f in THEMES_DIR.glob("*.css")] + print( + f"Error: Theme '{theme_name}' not found. Available: {available}", + file=sys.stderr, + ) + sys.exit(1) + return theme_file.read_text(encoding="utf-8") + _TYPOGRAPHY_CSS_PATCH + + +def _list_themes() -> list[str]: + """List available theme names.""" + if not THEMES_DIR.exists(): + return [] + return sorted(f.stem for f in THEMES_DIR.glob("*.css")) + + +def _ensure_list_spacing(text: str) -> str: + """Ensure blank lines before list items for proper markdown parsing. + + Both Python markdown library and pandoc require a blank line before a list + when it follows a paragraph. Without it, list items render as plain text. + """ + lines = text.split("\n") + result = [] + list_re = re.compile(r"^(\s*)([-*+]|\d+\.)\s") + for i, line in enumerate(lines): + if i > 0 and list_re.match(line): + prev = lines[i - 1] + if prev.strip() and not list_re.match(prev): + result.append("") + result.append(line) + return "\n".join(result) + + +_CJK_RANGE = re.compile( + # Chinese: CJK Unified Ideographs + Extension A + Compatibility + Extensions B-F + r"[\u4e00-\u9fff\u3400-\u4dbf\uf900-\ufaff" + r"\U00020000-\U0002a6df\U0002a700-\U0002ebef" + # CJK Symbols and Punctuation + Halfwidth/Fullwidth Forms + r"\u3000-\u303f\uff00-\uffef" + # Japanese: Hiragana + Katakana + Katakana Phonetic Extensions + r"\u3040-\u309f\u30a0-\u30ff\u31f0-\u31ff" + # Korean: Hangul Syllables + Hangul Jamo + Hangul Compatibility Jamo + r"\uac00-\ud7af\u1100-\u11ff\u3130-\u318f]" +) + +# Match 
…
allowing attributes on both tags. +# Also handles pandoc's 

+# syntax highlighting wrapper, by matching the inner 
 structure.
+_PRE_CODE_RE = re.compile(
+    r"]*>\s*]*>(.+?)\s*
",
+    flags=re.DOTALL,
+)
+
+
+def _fix_cjk_code_blocks(html: str) -> str:
+    """Replace 
 blocks containing CJK with styled divs.
+
+    weasyprint renders 
 blocks using monospace fonts that lack CJK glyphs,
+    causing garbled output. This converts CJK-heavy code blocks to styled divs
+    that use the document's CJK font stack instead.
+
+    Pure-ASCII code blocks (including pandoc-highlighted ones with language
+    identifiers) are left untouched so syntax highlighting and monospace
+    rendering are preserved.
+    """
+
+    def _replace_if_cjk(match: re.Match) -> str:
+        content = match.group(1)
+        if _CJK_RANGE.search(content):
+            # Strip pandoc's  syntax-highlighting wrappers so the
+            # content renders as plain text in the inherited body font.
+            cleaned = re.sub(r"]*>", "", content)
+            cleaned = cleaned.replace("", "")
+            return f'
{cleaned}
'
+        return match.group(0)
+
+    return _PRE_CODE_RE.sub(_replace_if_cjk, html)
+
+
+def _md_to_html(md_file: str) -> str:
+    """Convert markdown to HTML using pandoc with list spacing preprocessing."""
+    if not shutil.which("pandoc"):
+        print(
+            "Error: pandoc not found. Install with: brew install pandoc",
+            file=sys.stderr,
+        )
+        sys.exit(1)
+
+    md_content = Path(md_file).read_text(encoding="utf-8")
+    md_content = _ensure_list_spacing(md_content)
+
+    result = subprocess.run(
+        ["pandoc", "-f", "markdown", "-t", "html"],
+        input=md_content,
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        print(f"Error: pandoc failed: {result.stderr}", file=sys.stderr)
+        sys.exit(1)
+
+    html = result.stdout
+    html = _fix_cjk_code_blocks(html)
+    return html
+
+
+def _build_full_html(html_content: str, css: str, title: str) -> str:
+    """Wrap HTML content in a full document with CSS."""
+    return f"""
+
+
+    
+    {title}
+    
+
+
+{html_content}
+
+"""
+
+
+def _render_weasyprint(full_html: str, pdf_file: str, css: str) -> None:
+    """Render PDF using weasyprint."""
+    from weasyprint import CSS, HTML
+
+    HTML(string=full_html).write_pdf(pdf_file, stylesheets=[CSS(string=css)])
+
+
+def _render_chrome(full_html: str, pdf_file: str) -> None:
+    """Render PDF using headless Chrome."""
+    chrome = _find_chrome()
+    if not chrome:
+        print("Error: Chrome not found.", file=sys.stderr)
+        sys.exit(1)
+
+    with tempfile.NamedTemporaryFile(
+        suffix=".html", mode="w", encoding="utf-8", delete=False
+    ) as f:
+        f.write(full_html)
+        html_path = f.name
+
+    try:
+        result = subprocess.run(
+            [
+                chrome,
+                "--headless",
+                "--disable-gpu",
+                "--no-pdf-header-footer",
+                f"--print-to-pdf={pdf_file}",
+                html_path,
+            ],
+            capture_output=True,
+            text=True,
+        )
+        if not Path(pdf_file).exists():
+            print(
+                f"Error: Chrome failed to generate PDF. stderr: {result.stderr}",
+                file=sys.stderr,
+            )
+            sys.exit(1)
+    finally:
+        Path(html_path).unlink(missing_ok=True)
+
+
+# ---------------- Visual self-check (post-render PNG previews) ----------------
+#
+# Why: "PDF generated successfully" ≠ "PDF renders correctly". Common silent
+# failures include paragraph collapsing (pseudo-list), table overflow, missing
+# CJK / emoji glyphs, code block garbling. The author/AI must visually inspect
+# the rendered output, not assume success from a clean exit code.
+#
+# Design: after PDF generation, automatically convert each page to PNG next to
+# the PDF (via pdftoppm), and print a structured self-check checklist to stdout.
+# This makes "Read each PNG and verify" the default contract of every PDF run,
+# not an optional step that's easy to skip.
+#
+# Reference: CLAUDE.md "Self-Verification Protocol" + "Visual Verification" rules.
+
+
+def _generate_pdf_previews(pdf_file: str, dpi: int = 130) -> list[Path]:
+    """Convert each PDF page to PNG for visual inspection.
+
+    Returns sorted list of PNG paths. Empty list if pdftoppm not available
+    or no pages produced. Old previews in target dir are cleaned first.
+    """
+    if not shutil.which("pdftoppm"):
+        return []
+
+    pdf_path = Path(pdf_file).resolve()
+    preview_dir = pdf_path.parent / f"{pdf_path.stem}-preview"
+    preview_dir.mkdir(exist_ok=True)
+    # Clean stale previews so old/extra pages don't linger after a shorter rerun
+    for old in preview_dir.glob("page-*.png"):
+        old.unlink()
+
+    subprocess.run(
+        [
+            "pdftoppm",
+            "-png",
+            "-r",
+            str(dpi),
+            str(pdf_path),
+            str(preview_dir / "page"),
+        ],
+        capture_output=True,
+    )
+
+    return sorted(preview_dir.glob("page-*.png"))
+
+
+def _lint_pdf_typography(pdf_file: str) -> list[dict]:
+    """Detect inappropriate Chinese line breaks in rendered PDF.
+
+    Uses `pdftotext -layout` to preserve visual line structure, then scans
+    each page for known typography anti-patterns per "中文文案排版指北":
+
+      1. Single CJK character alone on a line (cell too narrow → forced break
+         mid-word/mid-name)
+      2. Line ending with 全角左括号 「（」 followed by content on next line
+         (broken parenthesis pair: opener separated from content)
+      3. Line starting with 全角右括号 「）」 (same as #2 from receiving end)
+      4. Short line ending with mid-thought punctuation 「、，；：」 right
+         before next CJK content (suggests forced break in narrow cell)
+
+    Returns: list of dicts {page, line, kind, snippet, message}.
+    Empty list if pdftotext unavailable or PDF clean.
+
+    Note: this only catches obvious cases. Subtle typography issues (uneven
+    line spacing, awkward breaks at safe points) require visual inspection.
+    """
+    if not shutil.which("pdftotext"):
+        return []
+
+    findings: list[dict] = []
+
+    # Process each page separately to give accurate page numbers
+    page_count_result = subprocess.run(
+        ["pdfinfo", str(pdf_file)],
+        capture_output=True,
+        text=True,
+    )
+    page_count = 0
+    if page_count_result.returncode == 0:
+        for line in page_count_result.stdout.splitlines():
+            if line.startswith("Pages:"):
+                try:
+                    page_count = int(line.split(":", 1)[1].strip())
+                except ValueError:
+                    pass
+                break
+
+    if page_count == 0:
+        return []
+
+    cjk_re = re.compile(r"[一-鿿]")
+
+    for page_num in range(1, page_count + 1):
+        result = subprocess.run(
+            [
+                "pdftotext",
+                "-layout",
+                "-f",
+                str(page_num),
+                "-l",
+                str(page_num),
+                str(pdf_file),
+                "-",
+            ],
+            capture_output=True,
+            text=True,
+        )
+        if result.returncode != 0:
+            continue
+
+        lines = result.stdout.split("\n")
+        for i, line in enumerate(lines):
+            stripped = line.strip()
+            if not stripped:
+                continue
+
+            # Pattern 1: Single CJK character alone on a line
+            if len(stripped) == 1 and cjk_re.match(stripped):
+                findings.append({
+                    "page": page_num,
+                    "line": i + 1,
+                    "kind": "single-cjk-char",
+                    "snippet": stripped,
+                    "message": (
+                        f"Single CJK char 「{stripped}」 alone on a line — "
+                        "cell width forced mid-word break"
+                    ),
+                })
+                continue
+
+            # Pattern 2: Line ends with 全角左括号 followed by content next line
+            if stripped.endswith("（") and i + 1 < len(lines):
+                next_stripped = lines[i + 1].strip()
+                if next_stripped and not next_stripped.startswith("）"):
+                    findings.append({
+                        "page": page_num,
+                        "line": i + 1,
+                        "kind": "broken-bracket-open",
+                        "snippet": stripped[-15:],
+                        "message": (
+                            "Line ends with 全角左括号「（」 and content "
+                            "wraps to next line — broken bracket pair"
+                        ),
+                    })
+                    continue
+
+            # Pattern 3: Line starts with 全角右括号
+            if stripped.startswith("）"):
+                findings.append({
+                    "page": page_num,
+                    "line": i + 1,
+                    "kind": "broken-bracket-close",
+                    "snippet": stripped[:15],
+                    "message": (
+                        "Line starts with 全角右括号「）」 — "
+                        "broken from previous line's bracket pair"
+                    ),
+                })
+                continue
+
+            # Pattern 4: Line ends with 中文标点 (suggests forced break)
+            # This is informational — sometimes legitimate
+            if stripped.endswith(("、", "，", "；", "：")) and i + 1 < len(lines):
+                next_stripped = lines[i + 1].strip()
+                # Only warn if next line continues with CJK content (not a new section)
+                if next_stripped and cjk_re.match(next_stripped):
+                    # Skip if this looks like end of a list item or paragraph (heuristic)
+                    if len(stripped) < 30:  # short cell content suggests forced break
+                        findings.append({
+                            "page": page_num,
+                            "line": i + 1,
+                            "kind": "trailing-punctuation-break",
+                            "snippet": stripped[-15:],
+                            "message": (
+                                f"Short line ends with 「{stripped[-1]}」 mid-thought — "
+                                "may be forced break in narrow cell"
+                            ),
+                        })
+
+    return findings
+
+
+def _print_typography_findings(findings: list[dict]) -> None:
+    """Print typography lint findings to stderr."""
+    if not findings:
+        return
+    print("", file=sys.stderr)
+    print(
+        f"⚠️  Typography lint: {len(findings)} potential Chinese line-break issue(s)",
+        file=sys.stderr,
+    )
+    print(
+        "    Per 中文文案排版指北 — narrow cells / over-wide tables often force "
+        "inappropriate breaks:",
+        file=sys.stderr,
+    )
+    for f in findings[:20]:  # cap output
+        print(
+            f"  [page {f['page']} L{f['line']}] {f['kind']}: "
+            f"{f['message']}",
+            file=sys.stderr,
+        )
+        print(f"      snippet: 「{f['snippet']}」", file=sys.stderr)
+    if len(findings) > 20:
+        print(f"  ... ({len(findings) - 20} more)", file=sys.stderr)
+    print(
+        "    💡 Fix: reduce table column count, shorten cell content, or "
+        "restructure as separate paragraph below the table.",
+        file=sys.stderr,
+    )
+    print("", file=sys.stderr)
+
+
+def _print_self_check_hint(pages: list[Path]) -> None:
+    """Print a structured self-check checklist after PDF generation."""
+    if not pages:
+        print(
+            "ℹ️  Visual self-check skipped: pdftoppm not found "
+            "(install: brew install poppler).",
+            file=sys.stderr,
+        )
+        return
+
+    preview_dir = pages[0].parent
+    print("")
+    print(f"📋 Visual self-check required ({len(pages)} pages)")
+    print(f"   Previews: {preview_dir}/page-NN.png")
+    print("")
+    print("   ⚠️  PDF generated cleanly does NOT mean rendering matches intent.")
+    print("   Read each page PNG and verify against your markdown source:")
+    print("")
+    print("   [ ] Paragraphs render as separate blocks (NOT collapsed into one)")
+    print("       ↳ Common issue: ≥2 consecutive `**xxx**：text` lines without")
+    print("         blank lines collapse into one paragraph (CommonMark soft")
+    print("         break = space). Fix in markdown: use `- ` real list, or")
+    print("         insert blank lines between.")
+    print("   [ ] Tables fit within page margins (no right-side text cut off)")
+    print("   [ ] No inappropriate Chinese line breaks (per 中文文案排版指北)")
+    print("       ↳ Symptoms: single CJK char alone on a line; broken bracket")
+    print("         pairs (line ends with `（` while content wraps to next line,")
+    print("         or line starts with `）`); short cells with mid-thought breaks.")
+    print("       ↳ Cause: cell width too narrow / table has too many columns.")
+    print("       ↳ Fix: reduce column count, shorten cell content, or move")
+    print("         long content into a separate paragraph below the table.")
+    print("   [ ] Lists keep nested indentation (sub-items visually nested)")
+    print("   [ ] Emoji + CJK glyphs render correctly (no boxes / placeholders)")
+    print("   [ ] Code blocks readable (monospace + CJK both visible)")
+    print("   [ ] No content overflow / unexpected page breaks mid-table")
+    print("   [ ] Last page ends naturally (no orphan title at top)")
+    print("")
+
+
+def markdown_to_pdf(
+    md_file: str,
+    pdf_file: str | None = None,
+    theme: str = "default",
+    backend: str | None = None,
+    previews: bool = True,
+) -> str:
+    """
+    Convert markdown file to PDF.
+
+    Args:
+        md_file: Path to input markdown file
+        pdf_file: Path to output PDF (optional, defaults to same name as input)
+        theme: Theme name (from themes/ directory)
+        backend: 'weasyprint', 'chrome', or None (auto-detect)
+        previews: If True (default), auto-generate per-page PNG previews next
+                  to the PDF and print a visual self-check checklist. Disable
+                  with --no-preview for batch / non-interactive runs.
+
+    Returns:
+        Path to generated PDF file
+    """
+    md_path = Path(md_file)
+    if pdf_file is None:
+        pdf_file = str(md_path.with_suffix(".pdf"))
+
+    if backend is None:
+        backend = _detect_backend()
+
+    css = _load_theme(theme)
+    html_content = _md_to_html(md_file)
+    full_html = _build_full_html(html_content, css, md_path.stem)
+
+    if backend == "weasyprint":
+        _render_weasyprint(full_html, pdf_file, css)
+    elif backend == "chrome":
+        _render_chrome(full_html, pdf_file)
+    else:
+        print(f"Error: Unknown backend '{backend}'", file=sys.stderr)
+        sys.exit(1)
+
+    size_kb = Path(pdf_file).stat().st_size / 1024
+    print(f"Generated: {pdf_file} ({size_kb:.0f}KB, theme={theme}, backend={backend})")
+
+    if previews:
+        # Auto-run typography lint to catch obvious mid-word breaks in tables.
+        # Findings go to stderr; do NOT block (warnings, not errors).
+        typography_findings = _lint_pdf_typography(pdf_file)
+        _print_typography_findings(typography_findings)
+
+        pages = _generate_pdf_previews(pdf_file)
+        _print_self_check_hint(pages)
+
+    return pdf_file
+
+
+def main():
+    available_themes = _list_themes()
+
+    parser = argparse.ArgumentParser(
+        description="Markdown to PDF with Chinese font support and themes."
+    )
+    parser.add_argument("input", help="Input markdown file")
+    parser.add_argument("output", nargs="?", help="Output PDF file (optional)")
+    parser.add_argument(
+        "--theme",
+        default="default",
+        choices=available_themes or ["default"],
+        help=f"CSS theme (available: {', '.join(available_themes) or 'default'})",
+    )
+    parser.add_argument(
+        "--backend",
+        choices=["weasyprint", "chrome"],
+        default=None,
+        help="PDF rendering backend (default: auto-detect)",
+    )
+    parser.add_argument(
+        "--list-themes",
+        action="store_true",
+        help="List available themes and exit",
+    )
+    parser.add_argument(
+        "--no-preview",
+        action="store_true",
+        help="Skip per-page PNG preview generation and self-check hint",
+    )
+
+    args = parser.parse_args()
+
+    if args.list_themes:
+        for t in available_themes:
+            marker = " (default)" if t == "default" else ""
+            css_file = THEMES_DIR / f"{t}.css"
+            first_line = ""
+            for line in css_file.read_text().splitlines():
+                line = line.strip()
+                if line.startswith("*") and "—" in line:
+                    first_line = line.lstrip("* ").strip()
+                    break
+            print(f"  {t}{marker}: {first_line}")
+        sys.exit(0)
+
+    if not Path(args.input).exists():
+        print(f"Error: File not found: {args.input}", file=sys.stderr)
+        sys.exit(1)
+
+    markdown_to_pdf(
+        args.input,
+        args.output,
+        args.theme,
+        args.backend,
+        previews=not args.no_preview,
+    )
+
+
+if __name__ == "__main__":
+    main()
diff --git a/daymade-docs/pdf-creator/scripts/tests/test_cjk_code_blocks.py b/daymade-docs/pdf-creator/scripts/tests/test_cjk_code_blocks.py
new file mode 100644
index 00000000..9edbb1c7
--- /dev/null
+++ b/daymade-docs/pdf-creator/scripts/tests/test_cjk_code_blocks.py
@@ -0,0 +1,194 @@
+#!/usr/bin/env python3
+"""
+Regression test for CJK code block rendering.
+
+weasyprint renders 
 blocks with monospace fonts that lack CJK glyphs.
+md_to_pdf.py detects CJK characters inside 
 and converts those
+blocks to 
 which inherits the body font.
+
+This test verifies:
+1. CJK-containing code blocks get the .cjk-code-block class
+2. Pure-ASCII code blocks remain as 
 (monospace preserved)
+3. Mixed documents handle both cases in a single pass
+"""
+
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+TEST_MARKDOWN = """# CJK Code Block 测试
+
+## 场景1：含中文的代码块（应转为 div）
+
+```
+03/08 国金：GPT-5.4发布评测 ← 最早的报告
+03/10 华创：多Agent | 国海：Token与算力出海
+  ↓ [3/10 CNCERT发布安全预警] ← 重大事件
+```
+
+## 场景2：纯 ASCII 代码块（应保持 pre）
+
+```python
+def hello():
+    print("Hello, World")
+    return 42
+```
+
+## 场景3：含日文的代码块（应转为 div）
+
+```
+こんにちは
+さようなら
+```
+
+## 场景4：inline code（`中文` 和 `code` 都应保留）
+
+Use `uv run` to execute or reference `配置` file.
+"""
+
+
+def _extract_html(md_path: str) -> str:
+    """Invoke the internal _md_to_html helper to get the preprocessed HTML."""
+    script_dir = Path(__file__).parent.parent
+    result = subprocess.run(
+        [
+            "uv",
+            "run",
+            "--with",
+            "weasyprint",
+            "python",
+            "-c",
+            f"import sys; sys.path.insert(0, '{script_dir}'); "
+            f"from md_to_pdf import _md_to_html; "
+            f"print(_md_to_html('{md_path}'))",
+        ],
+        capture_output=True,
+        text=True,
+        cwd=script_dir.parent,
+    )
+    if result.returncode != 0:
+        raise RuntimeError(f"_md_to_html failed: {result.stderr}")
+    return result.stdout
+
+
+def run_test() -> bool:
+    with tempfile.NamedTemporaryFile(
+        mode="w", suffix=".md", delete=False, encoding="utf-8"
+    ) as md_file:
+        md_file.write(TEST_MARKDOWN)
+        md_path = md_file.name
+
+    pdf_path = md_path.replace(".md", ".pdf")
+
+    try:
+        # Step 1: verify HTML preprocessing
+        print("=== HTML 预处理验证 ===")
+        html = _extract_html(md_path)
+
+        tests_passed = 0
+        tests_total = 6
+
+        # Test 1: CJK code block converted to div
+        if 'class="cjk-code-block"' in html and "CNCERT发布安全预警" in html:
+            print("✅ 场景1: 中文 code block 已转为 .cjk-code-block div")
+            tests_passed += 1
+        else:
+            print("❌ 场景1: 中文 code block 未被转换")
+
+        # Test 2: Pure ASCII code block stays inside 
...
.
+        # pandoc wraps highlighted blocks as
+        # 
...
+        # so we only check for 
 last_cjk:
+                print("✅ 场景2: 纯 ASCII code block 保持 
 结构")
+                tests_passed += 1
+            else:
+                print("❌ 场景2: 纯 ASCII code block 被错误转换为 cjk div")
+        else:
+            print("❌ 场景2: 纯 ASCII code block 丢失")
+
+        # Test 3: Japanese code block also gets converted
+        if "こんにちは" in html:
+            jp_pos = html.find("こんにちは")
+            preceding = html[max(0, jp_pos - 200) : jp_pos]
+            if "cjk-code-block" in preceding:
+                print("✅ 场景3: 日文 code block 已转为 .cjk-code-block div")
+                tests_passed += 1
+            else:
+                print("❌ 场景3: 日文 code block 未被转换")
+        else:
+            print("❌ 场景3: 日文 code block 丢失")
+
+        # Test 4: Inline code preserved (both CJK and ASCII)
+        if "中文" in html and "code" in html:
+            print("✅ 场景4: inline code 保持  标签")
+            tests_passed += 1
+        else:
+            print("❌ 场景4: inline code 处理异常")
+
+        # Step 2: verify PDF can actually be generated end-to-end
+        print("\n=== PDF 生成验证 ===")
+        script_dir = Path(__file__).parent.parent
+        md_to_pdf = script_dir / "md_to_pdf.py"
+        result = subprocess.run(
+            ["uv", "run", "--with", "weasyprint", str(md_to_pdf), md_path, pdf_path],
+            capture_output=True,
+            text=True,
+            cwd=script_dir.parent,
+        )
+
+        if result.returncode == 0 and Path(pdf_path).exists():
+            print("✅ 场景5: PDF 生成成功")
+            tests_passed += 1
+        else:
+            print(f"❌ 场景5: PDF 生成失败: {result.stderr}")
+
+        # Step 3: verify PDF content contains the CJK text (not garbled)
+        txt_path = pdf_path.replace(".pdf", ".txt")
+        result = subprocess.run(
+            ["pdftotext", pdf_path, txt_path], capture_output=True, text=True
+        )
+        if result.returncode == 0 and Path(txt_path).exists():
+            with open(txt_path, "r", encoding="utf-8") as f:
+                pdf_text = f.read()
+            # Key test: the CJK text from the code block must be intact
+            if "CNCERT发布安全预警" in pdf_text and "国金" in pdf_text:
+                print("✅ 场景6: PDF 中 CJK 文本未乱码")
+                tests_passed += 1
+            else:
+                print("❌ 场景6: PDF 中 CJK 文本乱码或丢失")
+                print(f"   提取内容（前 500 字符）: {pdf_text[:500]}")
+        else:
+            print("⚠️  场景6: pdftotext 不可用，跳过 PDF 内容验证")
+            tests_total -= 1
+
+        print(f"\n=== 测试结果: {tests_passed}/{tests_total} 通过 ===")
+
+        if tests_passed == tests_total:
+            print("\n✅ 所有测试通过！")
+            return True
+        print(f"\n❌ {tests_total - tests_passed} 个测试失败")
+        return False
+
+    except Exception as exc:  # noqa: BLE001
+        print(f"❌ 测试异常: {exc}")
+        import traceback
+
+        traceback.print_exc()
+        return False
+
+    finally:
+        Path(md_path).unlink(missing_ok=True)
+        Path(pdf_path).unlink(missing_ok=True)
+        Path(pdf_path.replace(".pdf", ".txt")).unlink(missing_ok=True)
+
+
+if __name__ == "__main__":
+    sys.exit(0 if run_test() else 1)
diff --git a/daymade-docs/pdf-creator/scripts/tests/test_list_rendering.py b/daymade-docs/pdf-creator/scripts/tests/test_list_rendering.py
new file mode 100755
index 00000000..a7944c97
--- /dev/null
+++ b/daymade-docs/pdf-creator/scripts/tests/test_list_rendering.py
@@ -0,0 +1,166 @@
+#!/usr/bin/env python3
+"""
+Test list rendering in PDF generation.
+
+Verifies that markdown lists are correctly rendered in PDFs,
+even when they don't have blank lines before them.
+
+The original markdown files are NOT modified - preprocessing
+happens in memory during conversion.
+"""
+
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+# Test markdown content with various list scenarios
+TEST_MARKDOWN = """# 测试列表解析
+
+## 场景1：列表前有空行（正常）
+
+这是一段文字。
+
+- 列表项 1
+- 列表项 2
+- 列表项 3
+
+## 场景2：列表前没有空行（关键测试）
+
+这是一段文字。
+- 列表项 1
+- 列表项 2
+- 列表项 3
+
+## 场景3：有序列表前没有空行
+
+这是一段文字。
+1. 第一项
+2. 第二项
+3. 第三项
+
+## 场景4：有序列表前有空行（正常）
+
+这是一段文字。
+
+1. 第一项
+2. 第二项
+3. 第三项
+"""
+
+
+def run_test():
+    """Run the list rendering test."""
+    # Create temporary files
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.md', delete=False, encoding='utf-8') as md_file:
+        md_file.write(TEST_MARKDOWN)
+        md_path = md_file.name
+
+    pdf_path = md_path.replace('.md', '.pdf')
+    txt_path = md_path.replace('.md', '.txt')
+
+    try:
+        # Generate PDF
+        script_dir = Path(__file__).parent.parent
+        md_to_pdf = script_dir / 'md_to_pdf.py'
+
+        print(f"生成 PDF: {md_path} -> {pdf_path}")
+        result = subprocess.run(
+            ['uv', 'run', '--with', 'weasyprint', str(md_to_pdf), md_path, pdf_path],
+            capture_output=True, text=True, cwd=script_dir.parent
+        )
+
+        if result.returncode != 0:
+            print(f"❌ PDF 生成失败: {result.stderr}")
+            return False
+
+        print(f"✅ PDF 已生成")
+
+        # Extract text from PDF
+        result = subprocess.run(
+            ['pdftotext', pdf_path, txt_path],
+            capture_output=True, text=True
+        )
+
+        if result.returncode != 0:
+            print(f"❌ 文本提取失败: {result.stderr}")
+            return False
+
+        # Read extracted text
+        with open(txt_path, 'r', encoding='utf-8') as f:
+            pdf_text = f.read()
+
+        # Verify original file was not modified
+        with open(md_path, 'r', encoding='utf-8') as f:
+            original_content = f.read()
+
+        if original_content != TEST_MARKDOWN:
+            print("❌ 原始文件被修改了！")
+            return False
+
+        print("✅ 原始文件未被修改")
+
+        # Verify list rendering
+        print("\n=== 列表渲染验证 ===")
+
+        tests_passed = 0
+        tests_total = 4
+
+        # Test 1: List with blank line before it
+        if '• 列表项 1' in pdf_text:
+            print("✅ 场景1: 列表前有空行 - 正确渲染")
+            tests_passed += 1
+        else:
+            print("❌ 场景1: 列表前有空行 - 渲染失败")
+
+        # Test 2: Critical test - list without blank line before it
+        scene2_start = pdf_text.find('场景2')
+        scene2_section = pdf_text[scene2_start:scene2_start+200] if scene2_start != -1 else ""
+
+        if '• 列表项 1' in scene2_section and '- 列表项 1' not in scene2_section:
+            print("✅ 场景2: 列表前没有空行 - 正确渲染（关键测试）")
+            tests_passed += 1
+        else:
+            print("❌ 场景2: 列表前没有空行 - 渲染失败")
+            print(f"   实际内容: {scene2_section}")
+
+        # Test 3: Ordered list without blank line
+        scene3_start = pdf_text.find('场景3')
+        scene3_section = pdf_text[scene3_start:scene3_start+200] if scene3_start != -1 else ""
+
+        if '1. 第一项' in scene3_section and '2. 第二项' in scene3_section:
+            print("✅ 场景3: 有序列表前没有空行 - 正确渲染")
+            tests_passed += 1
+        else:
+            print("❌ 场景3: 有序列表前没有空行 - 渲染失败")
+
+        # Test 4: Ordered list with blank line
+        if '1. 第一项' in pdf_text and '2. 第二项' in pdf_text:
+            print("✅ 场景4: 有序列表前有空行 - 正确渲染")
+            tests_passed += 1
+        else:
+            print("❌ 场景4: 有序列表前有空行 - 渲染失败")
+
+        print(f"\n=== 测试结果: {tests_passed}/{tests_total} 通过 ===")
+
+        if tests_passed == tests_total:
+            print("\n✅ 所有测试通过！")
+            print(f"\n生成的文件:")
+            print(f"  Markdown: {md_path}")
+            print(f"  PDF:      {pdf_path}")
+            print(f"  Text:     {txt_path}")
+            return True
+        else:
+            print(f"\n❌ {tests_total - tests_passed} 个测试失败")
+            return False
+
+    except Exception as e:
+        print(f"❌ 测试失败: {e}")
+        import traceback
+        traceback.print_exc()
+        return False
+
+
+if __name__ == '__main__':
+    success = run_test()
+    sys.exit(0 if success else 1)
diff --git a/daymade-docs/pdf-creator/scripts/tests/test_self_check_preview.py b/daymade-docs/pdf-creator/scripts/tests/test_self_check_preview.py
new file mode 100644
index 00000000..e6583491
--- /dev/null
+++ b/daymade-docs/pdf-creator/scripts/tests/test_self_check_preview.py
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""
+Integration test for visual self-check helper.
+
+Verifies the contract:
+  1. Default PDF run auto-generates per-page PNG previews next to the PDF
+  2. Default run prints a self-check checklist to stdout (so AI/author
+     is automatically reminded to visually inspect the rendering)
+  3. --no-preview disables both PNG generation and checklist
+
+This test enforces "Visual Verification" rule from CLAUDE.md: PDF generation
+silently succeeding does NOT mean the rendering matches the markdown intent.
+The checklist makes "Read each page PNG and verify" the default contract,
+not an optional step that's easy to skip.
+"""
+
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+SAMPLE_MD = """# Test Document
+
+A short test for self-check preview generation.
+
+## Section A
+
+- Item 1
+- Item 2
+- Item 3
+
+## Section B
+
+正文段落（中文测试 CJK rendering）。
+"""
+
+
+def run_md_to_pdf(args: list[str], scripts_dir: Path) -> subprocess.CompletedProcess:
+    """Run md_to_pdf.py with the given CLI args.
+
+    scripts_dir: path to the pdf-creator/scripts/ directory. The script
+                 lives at scripts_dir/md_to_pdf.py; we run the subprocess
+                 with cwd=scripts_dir.parent (the pdf-creator/ root) so
+                 the script's relative themes/ lookup resolves correctly.
+    """
+    script = scripts_dir / "md_to_pdf.py"
+    return subprocess.run(
+        ["uv", "run", "--with", "weasyprint", str(script)] + args,
+        capture_output=True,
+        text=True,
+        cwd=scripts_dir.parent,
+    )
+
+
+def main() -> int:
+    passed = 0
+    total = 0
+    script_dir = Path(__file__).parent.parent
+
+    with tempfile.TemporaryDirectory() as tmpdir:
+        tmp = Path(tmpdir)
+        md_path = tmp / "test.md"
+        md_path.write_text(SAMPLE_MD, encoding="utf-8")
+
+        # ---- Test 1: Default run prints self-check checklist ----
+        total += 1
+        pdf_path = tmp / "test.pdf"
+        result = run_md_to_pdf([str(md_path), str(pdf_path)], script_dir)
+        if result.returncode != 0:
+            print(f"❌ PDF generation failed: {result.stderr}")
+            return 1
+
+        checklist_markers = [
+            "Visual self-check required",
+            "Paragraphs render as separate blocks",
+            "Tables fit within page margins",
+            "Emoji + CJK glyphs render",
+        ]
+        missing = [m for m in checklist_markers if m not in result.stdout]
+        if not missing:
+            print("✅ Default run prints self-check checklist with all key items")
+            passed += 1
+        else:
+            print(f"❌ Checklist missing items: {missing}")
+            print(f"   stdout: {result.stdout[:500]}")
+
+        # ---- Test 2: Preview directory + PNG files generated ----
+        total += 1
+        preview_dir = tmp / "test-preview"
+        if preview_dir.exists():
+            pages = sorted(preview_dir.glob("page-*.png"))
+            if pages and all(p.stat().st_size > 0 for p in pages):
+                print(f"✅ {len(pages)} non-empty preview PNGs generated in test-preview/")
+                passed += 1
+            else:
+                print(f"❌ Preview dir exists but PNGs missing/empty: {pages}")
+        else:
+            print(f"❌ Preview dir not created: {preview_dir}")
+
+        # ---- Test 3: --no-preview disables both PNG + checklist ----
+        total += 1
+        pdf_path2 = tmp / "test_disabled.pdf"
+        preview_dir2 = tmp / "test_disabled-preview"
+        result = run_md_to_pdf(
+            [str(md_path), str(pdf_path2), "--no-preview"], script_dir
+        )
+        no_checklist = "Visual self-check" not in result.stdout
+        no_preview_dir = not preview_dir2.exists()
+        if no_checklist and no_preview_dir:
+            print("✅ --no-preview correctly disables PNG + checklist")
+            passed += 1
+        else:
+            print(
+                f"❌ --no-preview failed: checklist_absent={no_checklist}, "
+                f"dir_absent={no_preview_dir}"
+            )
+
+        # ---- Test 4: Stale PNGs cleaned on rerun ----
+        total += 1
+        # Plant a stale page-99.png (simulating old preview from a longer doc)
+        preview_dir.mkdir(exist_ok=True)
+        stale = preview_dir / "page-99.png"
+        stale.write_bytes(b"stale")
+        result = run_md_to_pdf([str(md_path), str(pdf_path)], script_dir)
+        if not stale.exists():
+            print("✅ Stale preview PNGs cleaned on rerun")
+            passed += 1
+        else:
+            print("❌ Stale page-99.png not cleaned on rerun")
+
+    print(f"\n=== {passed}/{total} tests passed ===")
+    return 0 if passed == total else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/daymade-docs/pdf-creator/themes/default.css b/daymade-docs/pdf-creator/themes/default.css
new file mode 100644
index 00000000..f67588d8
--- /dev/null
+++ b/daymade-docs/pdf-creator/themes/default.css
@@ -0,0 +1,228 @@
+/*
+ * Default — PDF theme for formal documents
+ *
+ * Color palette: black/grey, no accent color
+ * Font: Songti SC (body) + Heiti SC (headings)
+ * Best for: legal documents, trademark filings, contracts, formal reports
+ *
+ * This is the original built-in theme from md_to_pdf.py, extracted for reference.
+ */
+
+/* Restrict 'Menlo' to Latin/ASCII range so CJK characters in inline code
+ * don't get marked as Menlo (which has no CJK glyphs). Without this, the
+ * generated PDF references Menlo for CJK chars, and strict PDF readers
+ * (macOS Preview, some print drivers) show blanks instead of falling back.
+ * Chrome falls back automatically; Preview does not. The unicode-range
+ * trick forces weasyprint to skip Menlo for CJK and use the next font in
+ * the chain (PingFang SC → Heiti SC → Songti SC) which has CJK glyphs. */
+@font-face {
+    font-family: 'Menlo';
+    src: local('Menlo');
+    unicode-range: U+0020-007F, U+00A0-00FF, U+2000-206F, U+2070-209F, U+20A0-20CF, U+2100-214F;
+}
+@font-face {
+    font-family: 'Menlo';
+    src: local('Menlo Bold');
+    font-weight: bold;
+    unicode-range: U+0020-007F, U+00A0-00FF, U+2000-206F, U+2070-209F, U+20A0-20CF, U+2100-214F;
+}
+
+@page {
+    size: A4;
+    margin: 2.5cm 2cm 2cm 2cm;
+    @bottom-center {
+        content: counter(page) " / " counter(pages);
+        font-family: 'Songti SC', 'SimSun', 'STSong', 'Noto Serif CJK SC', serif;
+        font-size: 9pt;
+        color: #666;
+    }
+}
+
+body {
+    font-family: 'Songti SC', 'SimSun', 'STSong', 'Noto Serif CJK SC', serif;
+    font-size: 11.5pt;
+    line-height: 1.6;
+    color: #000;
+    width: 100%;
+}
+
+h1 {
+    font-family: 'Heiti SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', sans-serif;
+    font-size: 17pt;
+    font-weight: bold;
+    text-align: center;
+    margin-top: 0;
+    margin-bottom: 1.5em;
+}
+
+/* Heading scale: each level visibly larger than body (11.5pt) AND visually
+ * distinct from adjacent levels. Font fallback chain widened to include
+ * PingFang SC and system-ui in case 'Heiti SC' is not registered with
+ * fontconfig (common on weasyprint installs).
+ *
+ * Size unity rule: all body-adjacent text (inline code, table, pre)
+ * stays within 0.5-1pt of body so nothing reads as "noticeably smaller".
+ */
+h2 {
+    font-family: 'Heiti SC', 'PingFang SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', system-ui, sans-serif;
+    font-size: 15pt;
+    font-weight: bold;
+    margin-top: 1.5em;
+    margin-bottom: 0.8em;
+    color: #000;
+}
+
+h3 {
+    font-family: 'Heiti SC', 'PingFang SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', system-ui, sans-serif;
+    font-size: 13pt;
+    font-weight: bold;
+    margin-top: 1.3em;
+    margin-bottom: 0.5em;
+    color: #000;
+}
+
+h4 {
+    font-family: 'Heiti SC', 'PingFang SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', system-ui, sans-serif;
+    font-size: 12.5pt;
+    font-weight: bold;
+    margin-top: 1.1em;
+    margin-bottom: 0.4em;
+    color: #1a1a1a;
+}
+
+/* h5: still distinct from body. Combine size bump (+0.5pt) with left border
+ * so the heading visually jumps out even when the size delta is subtle.
+ */
+h5 {
+    font-family: 'Heiti SC', 'PingFang SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', system-ui, sans-serif;
+    font-size: 12pt;
+    font-weight: bold;
+    margin-top: 1em;
+    margin-bottom: 0.35em;
+    padding-left: 0.5em;
+    border-left: 3px solid #888;
+    color: #1a1a1a;
+}
+
+h6 {
+    font-family: 'Heiti SC', 'PingFang SC', 'SimHei', 'STHeiti', 'Noto Sans CJK SC', system-ui, sans-serif;
+    font-size: 11.5pt;
+    font-weight: bold;
+    margin-top: 0.8em;
+    margin-bottom: 0.25em;
+    color: #444;
+    font-style: italic;
+}
+
+p {
+    margin: 0.8em 0;
+    text-align: justify;
+}
+
+ul, ol {
+    margin: 0.8em 0;
+    padding-left: 2em;
+}
+
+li {
+    margin: 0.4em 0;
+}
+
+table {
+    border-collapse: collapse;
+    width: 100%;
+    margin: 1em 0;
+    font-size: 11pt;
+    table-layout: fixed;
+}
+
+/* Keep table rows intact when paginating: prevents a single
from being + * split across page boundaries (cell content cut mid-line, no header repeat). + * Trade-off: short rows that don't fit at page bottom get pushed to next page, + * leaving some white space at the bottom — readability > compactness. */ +tr { + page-break-inside: avoid; + break-inside: avoid; +} + +/* Repeat on each page when a table spans multiple pages. */ +thead { + display: table-header-group; +} + +th, td { + border: 1px solid #666; + padding: 8px 6px; + text-align: left; + overflow-wrap: break-word; + word-break: normal; +} + +th { + background-color: #f0f0f0; + font-weight: bold; +} + +/* Neutralize pandoc's auto-emitted based on dash counts + * in the markdown separator row. Pandoc treats `| ----- | --- |` as a column- + * width hint and inlines `style="width: 17%"` etc on each . Inline styles + * beat external stylesheets at equal specificity, so without `!important` no + * `td:first-child { width: ... }` rule can recover. With this neutralizer + * weasyprint falls back to `table-layout: fixed` equal width allocation, + * which for typical 4-col tables gives 25% per column — enough for short + * CJK labels like `4/28（周二）下午` to render on one line. + * + * Authors who really want explicit widths can still write raw HTML + * `` directly in markdown — that overrides this rule when needed. */ +table colgroup col { + width: auto !important; +} + +hr { + border: none; + border-top: 1px solid #ccc; + margin: 1.5em 0; +} + +code { + font-family: 'Menlo', 'Songti SC', 'Heiti SC', 'SimSun', 'PingFang SC', 'Noto Sans CJK SC', monospace; + background: #f5f5f5; + padding: 1px 4px; + border-radius: 3px; + font-size: 11pt; +} + +pre { + background: #f5f5f5; + border: 1px solid #ddd; + border-radius: 4px; + padding: 12px 16px; + margin: 1em 0; + overflow-wrap: break-word; + white-space: pre-wrap; + word-break: break-all; +} + +pre code { + font-family: 'Menlo', 'Songti SC', 'Heiti SC', 'SimSun', 'PingFang SC', 'Noto Sans CJK SC', monospace; + background: none; + padding: 0; + border-radius: 0; + font-size: 10.5pt; + line-height: 1.5; +} + +/* CJK code blocks converted to styled divs by preprocessor. + Uses inherit to reuse body's CJK font (weasyprint may not find PingFang SC). */ +.cjk-code-block { + font-family: inherit; + background: #f5f5f5; + border: 1px solid #ddd; + border-radius: 4px; + padding: 12px 16px; + margin: 1em 0; + font-size: 11pt; + line-height: 1.6; + white-space: pre-wrap; + word-break: break-all; +} diff --git a/daymade-docs/pdf-creator/themes/warm-terra.css b/daymade-docs/pdf-creator/themes/warm-terra.css new file mode 100644 index 00000000..b1483917 --- /dev/null +++ b/daymade-docs/pdf-creator/themes/warm-terra.css @@ -0,0 +1,190 @@ +/* + * Warm Terra — PDF theme for workshop/training documents + * + * Color palette: terra cotta (#d97756) + warm neutrals + * Font: PingFang SC (macOS) / Microsoft YaHei (Windows) + * Best for: course outlines, training materials, workshop agendas + * + * Usage with md_to_pdf.py: + * python md_to_pdf.py input.md output.pdf --theme warm-terra + * + * Usage with pandoc + Chrome (fallback): + * pandoc input.md -o /tmp/out.html --standalone -H <(cat this-file.css wrapped in + + +

Eval Set Review: __SKILL_NAME_PLACEHOLDER__

+

Current description: __SKILL_DESCRIPTION_PLACEHOLDER__

+ +
+ + +
+ +
+ + + + + + + + +
QueryShould TriggerActions
+ +

+ + + + diff --git a/skill-creator/eval-viewer/generate_review.py b/skill-creator/eval-viewer/generate_review.py new file mode 100644 index 00000000..7fa59786 --- /dev/null +++ b/skill-creator/eval-viewer/generate_review.py @@ -0,0 +1,471 @@ +#!/usr/bin/env python3 +"""Generate and serve a review page for eval results. + +Reads the workspace directory, discovers runs (directories with outputs/), +embeds all output data into a self-contained HTML page, and serves it via +a tiny HTTP server. Feedback auto-saves to feedback.json in the workspace. + +Usage: + python generate_review.py [--port PORT] [--skill-name NAME] + python generate_review.py --previous-feedback /path/to/old/feedback.json + +No dependencies beyond the Python stdlib are required. +""" + +import argparse +import base64 +import json +import mimetypes +import os +import re +import signal +import subprocess +import sys +import time +import webbrowser +from functools import partial +from http.server import HTTPServer, BaseHTTPRequestHandler +from pathlib import Path + +# Files to exclude from output listings +METADATA_FILES = {"transcript.md", "user_notes.md", "metrics.json"} + +# Extensions we render as inline text +TEXT_EXTENSIONS = { + ".txt", ".md", ".json", ".csv", ".py", ".js", ".ts", ".tsx", ".jsx", + ".yaml", ".yml", ".xml", ".html", ".css", ".sh", ".rb", ".go", ".rs", + ".java", ".c", ".cpp", ".h", ".hpp", ".sql", ".r", ".toml", +} + +# Extensions we render as inline images +IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"} + +# MIME type overrides for common types +MIME_OVERRIDES = { + ".svg": "image/svg+xml", + ".xlsx": "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ".docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + ".pptx": "application/vnd.openxmlformats-officedocument.presentationml.presentation", +} + + +def get_mime_type(path: Path) -> str: + ext = path.suffix.lower() + if ext in MIME_OVERRIDES: + return MIME_OVERRIDES[ext] + mime, _ = mimetypes.guess_type(str(path)) + return mime or "application/octet-stream" + + +def find_runs(workspace: Path) -> list[dict]: + """Recursively find directories that contain an outputs/ subdirectory.""" + runs: list[dict] = [] + _find_runs_recursive(workspace, workspace, runs) + runs.sort(key=lambda r: (r.get("eval_id", float("inf")), r["id"])) + return runs + + +def _find_runs_recursive(root: Path, current: Path, runs: list[dict]) -> None: + if not current.is_dir(): + return + + outputs_dir = current / "outputs" + if outputs_dir.is_dir(): + run = build_run(root, current) + if run: + runs.append(run) + return + + skip = {"node_modules", ".git", "__pycache__", "skill", "inputs"} + for child in sorted(current.iterdir()): + if child.is_dir() and child.name not in skip: + _find_runs_recursive(root, child, runs) + + +def build_run(root: Path, run_dir: Path) -> dict | None: + """Build a run dict with prompt, outputs, and grading data.""" + prompt = "" + eval_id = None + + # Try eval_metadata.json + for candidate in [run_dir / "eval_metadata.json", run_dir.parent / "eval_metadata.json"]: + if candidate.exists(): + try: + metadata = json.loads(candidate.read_text()) + prompt = metadata.get("prompt", "") + eval_id = metadata.get("eval_id") + except (json.JSONDecodeError, OSError): + pass + if prompt: + break + + # Fall back to transcript.md + if not prompt: + for candidate in [run_dir / "transcript.md", run_dir / "outputs" / "transcript.md"]: + if candidate.exists(): + try: + text = candidate.read_text() + match = re.search(r"## Eval Prompt\n\n([\s\S]*?)(?=\n##|$)", text) + if match: + prompt = match.group(1).strip() + except OSError: + pass + if prompt: + break + + if not prompt: + prompt = "(No prompt found)" + + run_id = str(run_dir.relative_to(root)).replace("/", "-").replace("\\", "-") + + # Collect output files + outputs_dir = run_dir / "outputs" + output_files: list[dict] = [] + if outputs_dir.is_dir(): + for f in sorted(outputs_dir.iterdir()): + if f.is_file() and f.name not in METADATA_FILES: + output_files.append(embed_file(f)) + + # Load grading if present + grading = None + for candidate in [run_dir / "grading.json", run_dir.parent / "grading.json"]: + if candidate.exists(): + try: + grading = json.loads(candidate.read_text()) + except (json.JSONDecodeError, OSError): + pass + if grading: + break + + return { + "id": run_id, + "prompt": prompt, + "eval_id": eval_id, + "outputs": output_files, + "grading": grading, + } + + +def embed_file(path: Path) -> dict: + """Read a file and return an embedded representation.""" + ext = path.suffix.lower() + mime = get_mime_type(path) + + if ext in TEXT_EXTENSIONS: + try: + content = path.read_text(errors="replace") + except OSError: + content = "(Error reading file)" + return { + "name": path.name, + "type": "text", + "content": content, + } + elif ext in IMAGE_EXTENSIONS: + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "image", + "mime": mime, + "data_uri": f"data:{mime};base64,{b64}", + } + elif ext == ".pdf": + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "pdf", + "data_uri": f"data:{mime};base64,{b64}", + } + elif ext == ".xlsx": + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "xlsx", + "data_b64": b64, + } + else: + # Binary / unknown — base64 download link + try: + raw = path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + except OSError: + return {"name": path.name, "type": "error", "content": "(Error reading file)"} + return { + "name": path.name, + "type": "binary", + "mime": mime, + "data_uri": f"data:{mime};base64,{b64}", + } + + +def load_previous_iteration(workspace: Path) -> dict[str, dict]: + """Load previous iteration's feedback and outputs. + + Returns a map of run_id -> {"feedback": str, "outputs": list[dict]}. + """ + result: dict[str, dict] = {} + + # Load feedback + feedback_map: dict[str, str] = {} + feedback_path = workspace / "feedback.json" + if feedback_path.exists(): + try: + data = json.loads(feedback_path.read_text()) + feedback_map = { + r["run_id"]: r["feedback"] + for r in data.get("reviews", []) + if r.get("feedback", "").strip() + } + except (json.JSONDecodeError, OSError, KeyError): + pass + + # Load runs (to get outputs) + prev_runs = find_runs(workspace) + for run in prev_runs: + result[run["id"]] = { + "feedback": feedback_map.get(run["id"], ""), + "outputs": run.get("outputs", []), + } + + # Also add feedback for run_ids that had feedback but no matching run + for run_id, fb in feedback_map.items(): + if run_id not in result: + result[run_id] = {"feedback": fb, "outputs": []} + + return result + + +def generate_html( + runs: list[dict], + skill_name: str, + previous: dict[str, dict] | None = None, + benchmark: dict | None = None, +) -> str: + """Generate the complete standalone HTML page with embedded data.""" + template_path = Path(__file__).parent / "viewer.html" + template = template_path.read_text() + + # Build previous_feedback and previous_outputs maps for the template + previous_feedback: dict[str, str] = {} + previous_outputs: dict[str, list[dict]] = {} + if previous: + for run_id, data in previous.items(): + if data.get("feedback"): + previous_feedback[run_id] = data["feedback"] + if data.get("outputs"): + previous_outputs[run_id] = data["outputs"] + + embedded = { + "skill_name": skill_name, + "runs": runs, + "previous_feedback": previous_feedback, + "previous_outputs": previous_outputs, + } + if benchmark: + embedded["benchmark"] = benchmark + + data_json = json.dumps(embedded) + + return template.replace("/*__EMBEDDED_DATA__*/", f"const EMBEDDED_DATA = {data_json};") + + +# --------------------------------------------------------------------------- +# HTTP server (stdlib only, zero dependencies) +# --------------------------------------------------------------------------- + +def _kill_port(port: int) -> None: + """Kill any process listening on the given port.""" + try: + result = subprocess.run( + ["lsof", "-ti", f":{port}"], + capture_output=True, text=True, timeout=5, + ) + for pid_str in result.stdout.strip().split("\n"): + if pid_str.strip(): + try: + os.kill(int(pid_str.strip()), signal.SIGTERM) + except (ProcessLookupError, ValueError): + pass + if result.stdout.strip(): + time.sleep(0.5) + except subprocess.TimeoutExpired: + pass + except FileNotFoundError: + print("Note: lsof not found, cannot check if port is in use", file=sys.stderr) + +class ReviewHandler(BaseHTTPRequestHandler): + """Serves the review HTML and handles feedback saves. + + Regenerates the HTML on each page load so that refreshing the browser + picks up new eval outputs without restarting the server. + """ + + def __init__( + self, + workspace: Path, + skill_name: str, + feedback_path: Path, + previous: dict[str, dict], + benchmark_path: Path | None, + *args, + **kwargs, + ): + self.workspace = workspace + self.skill_name = skill_name + self.feedback_path = feedback_path + self.previous = previous + self.benchmark_path = benchmark_path + super().__init__(*args, **kwargs) + + def do_GET(self) -> None: + if self.path == "/" or self.path == "/index.html": + # Regenerate HTML on each request (re-scans workspace for new outputs) + runs = find_runs(self.workspace) + benchmark = None + if self.benchmark_path and self.benchmark_path.exists(): + try: + benchmark = json.loads(self.benchmark_path.read_text()) + except (json.JSONDecodeError, OSError): + pass + html = generate_html(runs, self.skill_name, self.previous, benchmark) + content = html.encode("utf-8") + self.send_response(200) + self.send_header("Content-Type", "text/html; charset=utf-8") + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content) + elif self.path == "/api/feedback": + data = b"{}" + if self.feedback_path.exists(): + data = self.feedback_path.read_bytes() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + else: + self.send_error(404) + + def do_POST(self) -> None: + if self.path == "/api/feedback": + length = int(self.headers.get("Content-Length", 0)) + body = self.rfile.read(length) + try: + data = json.loads(body) + if not isinstance(data, dict) or "reviews" not in data: + raise ValueError("Expected JSON object with 'reviews' key") + self.feedback_path.write_text(json.dumps(data, indent=2) + "\n") + resp = b'{"ok":true}' + self.send_response(200) + except (json.JSONDecodeError, OSError, ValueError) as e: + resp = json.dumps({"error": str(e)}).encode() + self.send_response(500) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(resp))) + self.end_headers() + self.wfile.write(resp) + else: + self.send_error(404) + + def log_message(self, format: str, *args: object) -> None: + # Suppress request logging to keep terminal clean + pass + + +def main() -> None: + parser = argparse.ArgumentParser(description="Generate and serve eval review") + parser.add_argument("workspace", type=Path, help="Path to workspace directory") + parser.add_argument("--port", "-p", type=int, default=3117, help="Server port (default: 3117)") + parser.add_argument("--skill-name", "-n", type=str, default=None, help="Skill name for header") + parser.add_argument( + "--previous-workspace", type=Path, default=None, + help="Path to previous iteration's workspace (shows old outputs and feedback as context)", + ) + parser.add_argument( + "--benchmark", type=Path, default=None, + help="Path to benchmark.json to show in the Benchmark tab", + ) + parser.add_argument( + "--static", "-s", type=Path, default=None, + help="Write standalone HTML to this path instead of starting a server", + ) + args = parser.parse_args() + + workspace = args.workspace.resolve() + if not workspace.is_dir(): + print(f"Error: {workspace} is not a directory", file=sys.stderr) + sys.exit(1) + + runs = find_runs(workspace) + if not runs: + print(f"No runs found in {workspace}", file=sys.stderr) + sys.exit(1) + + skill_name = args.skill_name or workspace.name.replace("-workspace", "") + feedback_path = workspace / "feedback.json" + + previous: dict[str, dict] = {} + if args.previous_workspace: + previous = load_previous_iteration(args.previous_workspace.resolve()) + + benchmark_path = args.benchmark.resolve() if args.benchmark else None + benchmark = None + if benchmark_path and benchmark_path.exists(): + try: + benchmark = json.loads(benchmark_path.read_text()) + except (json.JSONDecodeError, OSError): + pass + + if args.static: + html = generate_html(runs, skill_name, previous, benchmark) + args.static.parent.mkdir(parents=True, exist_ok=True) + args.static.write_text(html) + print(f"\n Static viewer written to: {args.static}\n") + sys.exit(0) + + # Kill any existing process on the target port + port = args.port + _kill_port(port) + handler = partial(ReviewHandler, workspace, skill_name, feedback_path, previous, benchmark_path) + try: + server = HTTPServer(("127.0.0.1", port), handler) + except OSError: + # Port still in use after kill attempt — find a free one + server = HTTPServer(("127.0.0.1", 0), handler) + port = server.server_address[1] + + url = f"http://localhost:{port}" + print(f"\n Eval Viewer") + print(f" ─────────────────────────────────") + print(f" URL: {url}") + print(f" Workspace: {workspace}") + print(f" Feedback: {feedback_path}") + if previous: + print(f" Previous: {args.previous_workspace} ({len(previous)} runs)") + if benchmark_path: + print(f" Benchmark: {benchmark_path}") + print(f"\n Press Ctrl+C to stop.\n") + + webbrowser.open(url) + + try: + server.serve_forever() + except KeyboardInterrupt: + print("\nStopped.") + server.server_close() + + +if __name__ == "__main__": + main() diff --git a/skill-creator/eval-viewer/viewer.html b/skill-creator/eval-viewer/viewer.html new file mode 100644 index 00000000..6d8e9634 --- /dev/null +++ b/skill-creator/eval-viewer/viewer.html @@ -0,0 +1,1325 @@ + + + + + + Eval Review + + + + + + + +
+
+
+

Eval Review:

+
Review each output and leave feedback below. Navigate with arrow keys or buttons. When done, copy feedback and paste into Claude Code.
+
+
+
+ + + + + +
+
+ +
+
Prompt
+
+
+
+
+ + +
+
Output
+
+
No output files found
+
+
+ + + + + + + + +
+
Your Feedback
+
+ + + +
+
+
+ + +
+ + +
+
+
No benchmark data available. Run a benchmark to see quantitative results here.
+
+
+
+ + +
+
+

Review Complete

+

Your feedback has been saved. Go back to your Claude Code session and tell Claude you're done reviewing.

+
+ +
+
+
+ + +
+ + + + diff --git a/skill-creator/references/prerequisites.md b/skill-creator/references/prerequisites.md new file mode 100644 index 00000000..ef9dbab0 --- /dev/null +++ b/skill-creator/references/prerequisites.md @@ -0,0 +1,109 @@ +# Skill Creator Prerequisites + +Auto-detect and install all dependencies before starting skill creation. This prevents failures mid-workflow (e.g., discovering gitleaks is missing only at the packaging step). + +## Quick Check Script + +Run all checks in one go: + +```bash +echo "=== Skill Creator Prerequisites ===" +echo -n "uv: "; uv --version 2>/dev/null || echo "MISSING" +echo -n "Python: "; uv run python --version 2>/dev/null || echo "MISSING" +echo -n "PyYAML: "; uv run --with PyYAML python -c "import yaml; print('OK')" 2>/dev/null || echo "MISSING" +echo -n "gitleaks: "; gitleaks version 2>/dev/null || echo "MISSING" +echo -n "claude CLI: "; which claude 2>/dev/null || echo "MISSING" +echo -n "anthropic SDK: "; uv run --with anthropic python -c "import anthropic; print('OK')" 2>/dev/null || echo "MISSING (optional)" +``` + +## Dependencies by Phase + +| Dependency | Required For | Phase | Severity | +|-----------|-------------|-------|----------| +| uv | Python runtime and dependency declaration | All Python phases | **Blocking** | +| Python 3.7+ | All scripts | All | **Blocking** | +| PyYAML | `quick_validate.py`, `package_skill.py` | Validation, Packaging | **Blocking** | +| gitleaks | `security_scan.py` | Security Review (Step 6) | **Blocking for packaging** | +| claude CLI | `run_eval.py`, `run_loop.py` | Testing, Description Optimization | **Blocking for evals** | +| anthropic SDK | `improve_description.py`, `run_loop.py` | Description Optimization | Optional (only for desc optimization) | +| webbrowser | `generate_review.py` (viewer) | Eval Review | Optional (can use `--static` fallback) | + +## Auto-Installation + +### PyYAML (required) + +```bash +# Preferred: declare it at the call site +uv run --with PyYAML python -c "import yaml; print(yaml.__version__)" + +# Validation +uv run --with PyYAML python -m scripts.quick_validate +``` + +### gitleaks (required for packaging) + +```bash +# macOS +brew install gitleaks + +# Linux +wget https://github.com/gitleaks/gitleaks/releases/download/v8.21.2/gitleaks_8.21.2_linux_x64.tar.gz +tar -xzf gitleaks_8.21.2_linux_x64.tar.gz && sudo mv gitleaks /usr/local/bin/ + +# Verify +gitleaks version +``` + +### anthropic SDK (optional, for description optimization) + +```bash +uv run --with anthropic python -c "import anthropic; print('OK')" +``` + +Also requires `ANTHROPIC_API_KEY` environment variable to be set. + +### claude CLI (required for evals) + +The `claude` CLI (Claude Code) must be installed and available in PATH. If the user is already running this skill inside Claude Code, this is already satisfied. + +```bash +# Verify +which claude && claude --version +``` + +If missing, the user needs to install Claude Code from https://claude.ai/claude-code. + +## Script Invocation + +Run scripts from the skill-creator root directory. Use `uv run --with ...` when a script has Python dependencies: + +```bash +# CORRECT — run from skill-creator directory +cd +uv run --with PyYAML python -m scripts.quick_validate +uv run --with PyYAML python -m scripts.package_skill +uv run python -m scripts.security_scan +uv run python -m scripts.aggregate_benchmark --skill-name + +# WRONG — bare Python depends on ambient site packages +python3 scripts/package_skill.py # Can fail: No module named 'yaml' +python3 -m scripts.quick_validate # Can fail: No module named 'yaml' +``` + +This avoids relying on machine-global Python packages and keeps validation/packaging reproducible. + +## Presenting Results to User + +After running all checks, present a summary table: + +``` +Skill Creator Prerequisites: + [x] Python 3.12.0 + [x] PyYAML 6.0.1 + [x] gitleaks 8.21.2 + [x] claude CLI (running inside Claude Code) + [ ] anthropic SDK — not installed (only needed for description optimization) + [x] uv 0.6.x +``` + +If any **blocking** dependency is missing and auto-install fails, clearly explain what the user needs to do and stop before proceeding to skill creation. diff --git a/skill-creator/references/sanitization_checklist.md b/skill-creator/references/sanitization_checklist.md index ddd9fce0..c79d54c1 100644 --- a/skill-creator/references/sanitization_checklist.md +++ b/skill-creator/references/sanitization_checklist.md @@ -66,7 +66,7 @@ grep -rniE "ultrathink|internal-only|confidential" skill-folder/ **What to find:** - Team-specific folders: `10-team-collaboration/Meeting Minutes` - Project-specific paths: `reviewer-portal-api-design` -- Environment-specific paths: `/Users/username/Projects/` +- Environment-specific paths: user home directory project paths **How to replace:** - Use generic paths: `project-docs/meeting-minutes` diff --git a/skill-creator/references/schemas.md b/skill-creator/references/schemas.md new file mode 100644 index 00000000..b6eeaa2d --- /dev/null +++ b/skill-creator/references/schemas.md @@ -0,0 +1,430 @@ +# JSON Schemas + +This document defines the JSON schemas used by skill-creator. + +--- + +## evals.json + +Defines the evals for a skill. Located at `evals/evals.json` within the skill directory. + +```json +{ + "skill_name": "example-skill", + "evals": [ + { + "id": 1, + "prompt": "User's example prompt", + "expected_output": "Description of expected result", + "files": ["evals/files/sample1.pdf"], + "expectations": [ + "The output includes X", + "The skill used script Y" + ] + } + ] +} +``` + +**Fields:** +- `skill_name`: Name matching the skill's frontmatter +- `evals[].id`: Unique integer identifier +- `evals[].prompt`: The task to execute +- `evals[].expected_output`: Human-readable description of success +- `evals[].files`: Optional list of input file paths (relative to skill root) +- `evals[].expectations`: List of verifiable statements + +--- + +## history.json + +Tracks version progression in Improve mode. Located at workspace root. + +```json +{ + "started_at": "2026-01-15T10:30:00Z", + "skill_name": "pdf", + "current_best": "v2", + "iterations": [ + { + "version": "v0", + "parent": null, + "expectation_pass_rate": 0.65, + "grading_result": "baseline", + "is_current_best": false + }, + { + "version": "v1", + "parent": "v0", + "expectation_pass_rate": 0.75, + "grading_result": "won", + "is_current_best": false + }, + { + "version": "v2", + "parent": "v1", + "expectation_pass_rate": 0.85, + "grading_result": "won", + "is_current_best": true + } + ] +} +``` + +**Fields:** +- `started_at`: ISO timestamp of when improvement started +- `skill_name`: Name of the skill being improved +- `current_best`: Version identifier of the best performer +- `iterations[].version`: Version identifier (v0, v1, ...) +- `iterations[].parent`: Parent version this was derived from +- `iterations[].expectation_pass_rate`: Pass rate from grading +- `iterations[].grading_result`: "baseline", "won", "lost", or "tie" +- `iterations[].is_current_best`: Whether this is the current best version + +--- + +## grading.json + +Output from the grader agent. Located at `/grading.json`. + +```json +{ + "expectations": [ + { + "text": "The output includes the name 'John Smith'", + "passed": true, + "evidence": "Found in transcript Step 3: 'Extracted names: John Smith, Sarah Johnson'" + }, + { + "text": "The spreadsheet has a SUM formula in cell B10", + "passed": false, + "evidence": "No spreadsheet was created. The output was a text file." + } + ], + "summary": { + "passed": 2, + "failed": 1, + "total": 3, + "pass_rate": 0.67 + }, + "execution_metrics": { + "tool_calls": { + "Read": 5, + "Write": 2, + "Bash": 8 + }, + "total_tool_calls": 15, + "total_steps": 6, + "errors_encountered": 0, + "output_chars": 12450, + "transcript_chars": 3200 + }, + "timing": { + "executor_duration_seconds": 165.0, + "grader_duration_seconds": 26.0, + "total_duration_seconds": 191.0 + }, + "claims": [ + { + "claim": "The form has 12 fillable fields", + "type": "factual", + "verified": true, + "evidence": "Counted 12 fields in field_info.json" + } + ], + "user_notes_summary": { + "uncertainties": ["Used 2023 data, may be stale"], + "needs_review": [], + "workarounds": ["Fell back to text overlay for non-fillable fields"] + }, + "eval_feedback": { + "suggestions": [ + { + "assertion": "The output includes the name 'John Smith'", + "reason": "A hallucinated document that mentions the name would also pass" + } + ], + "overall": "Assertions check presence but not correctness." + } +} +``` + +**Fields:** +- `expectations[]`: Graded expectations with evidence +- `summary`: Aggregate pass/fail counts +- `execution_metrics`: Tool usage and output size (from executor's metrics.json) +- `timing`: Wall clock timing (from timing.json) +- `claims`: Extracted and verified claims from the output +- `user_notes_summary`: Issues flagged by the executor +- `eval_feedback`: (optional) Improvement suggestions for the evals, only present when the grader identifies issues worth raising + +--- + +## metrics.json + +Output from the executor agent. Located at `/outputs/metrics.json`. + +```json +{ + "tool_calls": { + "Read": 5, + "Write": 2, + "Bash": 8, + "Edit": 1, + "Glob": 2, + "Grep": 0 + }, + "total_tool_calls": 18, + "total_steps": 6, + "files_created": ["filled_form.pdf", "field_values.json"], + "errors_encountered": 0, + "output_chars": 12450, + "transcript_chars": 3200 +} +``` + +**Fields:** +- `tool_calls`: Count per tool type +- `total_tool_calls`: Sum of all tool calls +- `total_steps`: Number of major execution steps +- `files_created`: List of output files created +- `errors_encountered`: Number of errors during execution +- `output_chars`: Total character count of output files +- `transcript_chars`: Character count of transcript + +--- + +## timing.json + +Wall clock timing for a run. Located at `/timing.json`. + +**How to capture:** When a subagent task completes, the task notification includes `total_tokens` and `duration_ms`. Save these immediately — they are not persisted anywhere else and cannot be recovered after the fact. + +```json +{ + "total_tokens": 84852, + "duration_ms": 23332, + "total_duration_seconds": 23.3, + "executor_start": "2026-01-15T10:30:00Z", + "executor_end": "2026-01-15T10:32:45Z", + "executor_duration_seconds": 165.0, + "grader_start": "2026-01-15T10:32:46Z", + "grader_end": "2026-01-15T10:33:12Z", + "grader_duration_seconds": 26.0 +} +``` + +--- + +## benchmark.json + +Output from Benchmark mode. Located at `benchmarks//benchmark.json`. + +```json +{ + "metadata": { + "skill_name": "pdf", + "skill_path": "/path/to/pdf", + "executor_model": "claude-sonnet-4-20250514", + "analyzer_model": "most-capable-model", + "timestamp": "2026-01-15T10:30:00Z", + "evals_run": [1, 2, 3], + "runs_per_configuration": 3 + }, + + "runs": [ + { + "eval_id": 1, + "eval_name": "Ocean", + "configuration": "with_skill", + "run_number": 1, + "result": { + "pass_rate": 0.85, + "passed": 6, + "failed": 1, + "total": 7, + "time_seconds": 42.5, + "tokens": 3800, + "tool_calls": 18, + "errors": 0 + }, + "expectations": [ + {"text": "...", "passed": true, "evidence": "..."} + ], + "notes": [ + "Used 2023 data, may be stale", + "Fell back to text overlay for non-fillable fields" + ] + } + ], + + "run_summary": { + "with_skill": { + "pass_rate": {"mean": 0.85, "stddev": 0.05, "min": 0.80, "max": 0.90}, + "time_seconds": {"mean": 45.0, "stddev": 12.0, "min": 32.0, "max": 58.0}, + "tokens": {"mean": 3800, "stddev": 400, "min": 3200, "max": 4100} + }, + "without_skill": { + "pass_rate": {"mean": 0.35, "stddev": 0.08, "min": 0.28, "max": 0.45}, + "time_seconds": {"mean": 32.0, "stddev": 8.0, "min": 24.0, "max": 42.0}, + "tokens": {"mean": 2100, "stddev": 300, "min": 1800, "max": 2500} + }, + "delta": { + "pass_rate": "+0.50", + "time_seconds": "+13.0", + "tokens": "+1700" + } + }, + + "notes": [ + "Assertion 'Output is a PDF file' passes 100% in both configurations - may not differentiate skill value", + "Eval 3 shows high variance (50% ± 40%) - may be flaky or model-dependent", + "Without-skill runs consistently fail on table extraction expectations", + "Skill adds 13s average execution time but improves pass rate by 50%" + ] +} +``` + +**Fields:** +- `metadata`: Information about the benchmark run + - `skill_name`: Name of the skill + - `timestamp`: When the benchmark was run + - `evals_run`: List of eval names or IDs + - `runs_per_configuration`: Number of runs per config (e.g. 3) +- `runs[]`: Individual run results + - `eval_id`: Numeric eval identifier + - `eval_name`: Human-readable eval name (used as section header in the viewer) + - `configuration`: Must be `"with_skill"` or `"without_skill"` (the viewer uses this exact string for grouping and color coding) + - `run_number`: Integer run number (1, 2, 3...) + - `result`: Nested object with `pass_rate`, `passed`, `total`, `time_seconds`, `tokens`, `errors` +- `run_summary`: Statistical aggregates per configuration + - `with_skill` / `without_skill`: Each contains `pass_rate`, `time_seconds`, `tokens` objects with `mean` and `stddev` fields + - `delta`: Difference strings like `"+0.50"`, `"+13.0"`, `"+1700"` +- `notes`: Freeform observations from the analyzer + +**Important:** The viewer reads these field names exactly. Using `config` instead of `configuration`, or putting `pass_rate` at the top level of a run instead of nested under `result`, will cause the viewer to show empty/zero values. Always reference this schema when generating benchmark.json manually. + +--- + +## comparison.json + +Output from blind comparator. Located at `/comparison-N.json`. + +```json +{ + "winner": "A", + "reasoning": "Output A provides a complete solution with proper formatting and all required fields. Output B is missing the date field and has formatting inconsistencies.", + "rubric": { + "A": { + "content": { + "correctness": 5, + "completeness": 5, + "accuracy": 4 + }, + "structure": { + "organization": 4, + "formatting": 5, + "usability": 4 + }, + "content_score": 4.7, + "structure_score": 4.3, + "overall_score": 9.0 + }, + "B": { + "content": { + "correctness": 3, + "completeness": 2, + "accuracy": 3 + }, + "structure": { + "organization": 3, + "formatting": 2, + "usability": 3 + }, + "content_score": 2.7, + "structure_score": 2.7, + "overall_score": 5.4 + } + }, + "output_quality": { + "A": { + "score": 9, + "strengths": ["Complete solution", "Well-formatted", "All fields present"], + "weaknesses": ["Minor style inconsistency in header"] + }, + "B": { + "score": 5, + "strengths": ["Readable output", "Correct basic structure"], + "weaknesses": ["Missing date field", "Formatting inconsistencies", "Partial data extraction"] + } + }, + "expectation_results": { + "A": { + "passed": 4, + "total": 5, + "pass_rate": 0.80, + "details": [ + {"text": "Output includes name", "passed": true} + ] + }, + "B": { + "passed": 3, + "total": 5, + "pass_rate": 0.60, + "details": [ + {"text": "Output includes name", "passed": true} + ] + } + } +} +``` + +--- + +## analysis.json + +Output from post-hoc analyzer. Located at `/analysis.json`. + +```json +{ + "comparison_summary": { + "winner": "A", + "winner_skill": "path/to/winner/skill", + "loser_skill": "path/to/loser/skill", + "comparator_reasoning": "Brief summary of why comparator chose winner" + }, + "winner_strengths": [ + "Clear step-by-step instructions for handling multi-page documents", + "Included validation script that caught formatting errors" + ], + "loser_weaknesses": [ + "Vague instruction 'process the document appropriately' led to inconsistent behavior", + "No script for validation, agent had to improvise" + ], + "instruction_following": { + "winner": { + "score": 9, + "issues": ["Minor: skipped optional logging step"] + }, + "loser": { + "score": 6, + "issues": [ + "Did not use the skill's formatting template", + "Invented own approach instead of following step 3" + ] + } + }, + "improvement_suggestions": [ + { + "priority": "high", + "category": "instructions", + "suggestion": "Replace 'process the document appropriately' with explicit steps", + "expected_impact": "Would eliminate ambiguity that caused inconsistent behavior" + } + ], + "transcript_insights": { + "winner_execution_pattern": "Read skill -> Followed 5-step process -> Used validation script", + "loser_execution_pattern": "Read skill -> Unclear on approach -> Tried 3 different methods" + } +} +``` diff --git a/skill-creator/references/skill-development-methodology.md b/skill-creator/references/skill-development-methodology.md new file mode 100644 index 00000000..0d93b075 --- /dev/null +++ b/skill-creator/references/skill-development-methodology.md @@ -0,0 +1,149 @@ +# Skill Development Methodology + +综合 Anthropic 官方最佳实践、skill-creator 工作流、社区经验和实战教训的完整方法论。 + +本文档只包含 SKILL.md 中**没有覆盖**的内容。SKILL.md 已经详细描述的流程（Prior Art 8 渠道表、决策矩阵、Inline vs Fork、测试用例格式、描述优化循环等）不在此重复——请直接参考 SKILL.md 对应章节。 + +## Phase 1: 先手动解决问题，不要上来就建 skill + +SKILL.md 的 "Capture Intent" 章节覆盖了意图收集的 4 个问题和 skill 类型分类。本节补充一个被忽略的前置步骤： + +**不要一开始就写 skill。** 先用 Claude Code 正常解决用户的问题，在过程中积累经验——哪些方案有效、哪些失败、最终的 working solution 是什么。如果你没有亲自失败过，你写不出能防止别人失败的 skill。 + +很多 skill 都是从"把我们刚做的变成一个 skill"中诞生的。先从对话历史中提取已验证的模式（SKILL.md "Capture Intent" 第三段已提及），然后才开始规划 skill 结构。 + +## Phase 2: 用 Agent Team 做并行调研 + +SKILL.md 的 "Prior Art Research" 章节覆盖了 8 个搜索渠道、clone-and-verify 检查清单、和 Adopt/Extend/Build 决策矩阵。本节补充 SKILL.md 未提及的**并行调研模式**： + +遇到不确定的技术方案时，不要串行尝试（太慢），也不要凭经验猜（太危险）。同时启动 3+ 个研究 agent，每个负责一个调研方向： + +| Agent | 职责 | 搜索范围 | +|-------|------|---------| +| 工具调研 | 找已有成熟工具 | GitHub stars、npm/PyPI、社区 skill 注册表 | +| API 调研 | 找可用 API 端点 | 官方文档、逆向工程、移动端 API | +| 约束调研 | 理解技术限制 | 反爬机制、认证要求、平台限制 | + +每个 agent 必须独立验证（读源码、确认 API 可达、检查最近提交日期），不能只看 README。 + +**案例**：开发一个数据导出 skill 时，3 个 agent 并行跑了 5-20 分钟，分别发现：一个关键工具当前版本 broken（605 stars 但 PR 待合并）、一个未公开的移动端 API（唯一可行方案）、目标平台升级了 PoW 反爬（所有 HTTP 抓取失效）。没有并行研究，这些信息需要串行试错 3+ 小时才能获得。 + +## Phase 3: 用真实数据验证原型 + +SKILL.md 的 Evaluation-Driven Development 流程覆盖了"先跑 baseline → 建 eval → 迭代"的过程。本节补充两个 SKILL.md 未强调的验证原则： + +### 3.1 数据完整性验证 + +"it runs without errors" ≠ "it exported all items correctly"。必须： +- 对比 API 报告的 total 和实际导出行数 +- 检查字段格式（评分、日期、编码是否符合预期） +- 用不同规模的数据测试（0 条、100 条、1000+ 条） + +**常见静默 bug**： +- 分页逻辑：某些页面返回的数据量少于请求值（如请求 50 条返回 48 条），被误判为最后一页导致提前终止。修复：检查 `total` 而非 `page_size` +- 数据转换：API 返回 `{value: 2, max: 5}` 表示 2/5 星，但代码按 `max: 10` 处理后变成 1 星。修复：检查 `max` 字段确定 scale + +### 3.2 记录失败 + +详细记录每个失败方案的方法、失败模式、根因。这些将成为 skill 中 "Do NOT attempt" 部分的内容——这是 skill 最独特的价值，防止未来的 agent 重走弯路。 + +失败记录的结构： + +| 方案 | 结果 | 根因 | +|------|------|------| +| 方案名称 | 具体失败表现（HTTP 状态码、错误信息） | 架构层面的原因分析 | + +## Phase 4: Skill 写作补充原则 + +SKILL.md 的 "Skill Writing Guide" 已覆盖 frontmatter、progressive disclosure、bundled resources、命名规范等。本节补充 SKILL.md 未提及的内容层面原则： + +### 4.1 写清楚 skill 不能做什么 + +防止 agent 尝试不可能的操作。例如： +- "Cannot export reviews (长评) — different API endpoint, not implemented" +- "Cannot filter by single category — exports all 4 types together" + +### 4.2 写清楚失败过什么 + +在 SKILL.md 或 references 中保留失败方案的摘要（详见 Phase 3.2），加上明确的"Do NOT attempt"警告。这比正面指令更有效——agent 看到 7 种方案的失败记录后，不会尝试第 8 种类似方案。 + +### 4.3 安全说明 + +如果脚本包含 API key、HMAC 密钥或其他凭据，必须解释来源和安全性。例如："These are the app's public credentials extracted from the APK, shared by all users. No personal credentials are used." + +### 4.4 Console output 示例 + +展示一次成功运行的完整控制台输出。让 agent 知道"正确运行"长什么样，方便验证（SKILL.md Phase 5 的 self-verification）。 + +### 4.5 脚本健壮性 + +SKILL.md 的 "Solve, don't punt" 覆盖了基本错误处理。补充实战中发现的常见遗漏： +- 只捕获 HTTPError，遗漏 URLError / socket.timeout / JSONDecodeError +- 无限分页循环（API 异常时）——需要 max-page 安全阀 +- CSV 中的换行符/回车符——`csvEscape` 必须处理 `\r` +- 用户输入是完整 URL 而非 ID——脚本应自动提取 + +## Phase 5: 测试迭代补充 + +SKILL.md 的测试流程非常详细（A/B 测试、断言、评分、viewer）。本节补充两个 SKILL.md 未覆盖的实操教训： + +### 5.1 删除竞争的旧 skill + +如果系统中存在旧版 skill（关键词冲突），eval agent 会被旧 skill 截胡，导致测试结果完全无效。必须在测试前删除旧 skill。 + +**信号**：eval agent 使用了不同于预期的脚本或方法 → 检查是否有同名/同领域的旧 skill 被加载。 + +### 5.2 量化迭代对比 + +SKILL.md 提到 timing.json 和 benchmark，但未给出具体应跟踪哪些指标。推荐： + +| 指标 | 为什么重要 | +|------|-----------| +| 数据完整性（实际/预期） | 核心正确性 | +| 执行时间 | 用户体验 | +| Token 消耗 | 成本 | +| 工具调用次数 | skill 引导效率——次数越少说明 skill 的指令越清晰 | +| 错误数 | 必须为 0 | + +**案例对比**：某 skill 迭代后，工具调用从 31 次降到 8 次（74% 减少）、Token 从 72K 降到 41K（43% 减少），说明 skill 的指令让 agent 不再需要自己摸索。 + +## Phase 6: Counter Review — 用 Agent Team 做对抗性审查 + +这是 SKILL.md 未覆盖的独立环节。SKILL.md 的 "Improving the skill" 章节关注用户反馈驱动的迭代，但没有系统化的多视角审查流程。 + +### 6.1 第一轮：3 个视角并行 + +用 Task 工具同时启动 3 个 review agent： + +| Reviewer | 视角 | 关注点 | +|----------|------|--------| +| Skill 质量 | 对标 Anthropic 最佳实践 | 描述质量、简洁性、progressive disclosure、可操作性、错误预防、示例、术语一致性 | +| 代码健壮性 | 高级工程师找 bug | 错误处理、安全性、跨平台、边界情况、依赖、幂等性 | +| 用户视角 | 首次使用者体验 | 首次成功率、输入容错、输出预期、隐私顾虑、失败恢复 | + +### 6.2 修复后 Final Gate + +修复所有 Critical 和 HIGH 问题后，再启动 final gate reviewers 验证修复正确性。评分 >= 8 才放行。 + +### 6.3 常见发现模式 + +根据实战经验，reviewer 经常发现的问题类型： +- **SKILL.md 和 references 内容重复**（每次都会犯，包括本文档自己） +- **异常类型遗漏**（只捕获 HTTPError，漏掉 URLError/socket.timeout） +- **substring 误匹配**（`content.includes(url)` 导致 `/1234/` 匹配 `/12345/`） +- **docstring 与实际行为不一致**（写了 "4.5 → 5" 但实际行为是 "4.5 → 4"） +- **误导性注释**（注释说"每个分类写入后立即保存"但代码在最后才写入） +- **时间敏感数据**（特定日期的测试结果、版本号——下周就过时了） + +## Phase 7 & 8: Description Optimization + Packaging + +SKILL.md 已完整覆盖描述优化循环（20 个 eval query、60/40 train/test split、5 轮迭代）和打包流程（prerequisites、security scan、marketplace.json）。无补充。 + +## 来源 + +| 来源 | 本文档引用的独有贡献 | +|------|-------------------| +| Anthropic Official | Evaluation-driven development、conciseness imperative（已由 SKILL.md 覆盖，本文不重复） | +| skill-creator SKILL.md | 完整工作流和工具链（本文引用但不复制，请直接参考 SKILL.md） | +| 社区经验 | 激活率数据（20%→90%）、Encoded Preference > Capability Uplift | +| 实战教训 | 并行研究 agent、失败记录的价值、竞争 skill 删除、量化迭代对比、Counter Review 流程 | diff --git a/skill-creator/scripts/__init__.py b/skill-creator/scripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/skill-creator/scripts/aggregate_benchmark.py b/skill-creator/scripts/aggregate_benchmark.py new file mode 100755 index 00000000..3e66e8c1 --- /dev/null +++ b/skill-creator/scripts/aggregate_benchmark.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +""" +Aggregate individual run results into benchmark summary statistics. + +Reads grading.json files from run directories and produces: +- run_summary with mean, stddev, min, max for each metric +- delta between with_skill and without_skill configurations + +Usage: + python aggregate_benchmark.py + +Example: + python aggregate_benchmark.py benchmarks/2026-01-15T10-30-00/ + +The script supports two directory layouts: + + Workspace layout (from skill-creator iterations): + / + └── eval-N/ + ├── with_skill/ + │ ├── run-1/grading.json + │ └── run-2/grading.json + └── without_skill/ + ├── run-1/grading.json + └── run-2/grading.json + + Legacy layout (with runs/ subdirectory): + / + └── runs/ + └── eval-N/ + ├── with_skill/ + │ └── run-1/grading.json + └── without_skill/ + └── run-1/grading.json +""" + +import argparse +import json +import math +import sys +from datetime import datetime, timezone +from pathlib import Path + + +def calculate_stats(values: list[float]) -> dict: + """Calculate mean, stddev, min, max for a list of values.""" + if not values: + return {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0} + + n = len(values) + mean = sum(values) / n + + if n > 1: + variance = sum((x - mean) ** 2 for x in values) / (n - 1) + stddev = math.sqrt(variance) + else: + stddev = 0.0 + + return { + "mean": round(mean, 4), + "stddev": round(stddev, 4), + "min": round(min(values), 4), + "max": round(max(values), 4) + } + + +def load_run_results(benchmark_dir: Path) -> dict: + """ + Load all run results from a benchmark directory. + + Returns dict keyed by config name (e.g. "with_skill"/"without_skill", + or "new_skill"/"old_skill"), each containing a list of run results. + """ + # Support both layouts: eval dirs directly under benchmark_dir, or under runs/ + runs_dir = benchmark_dir / "runs" + if runs_dir.exists(): + search_dir = runs_dir + elif list(benchmark_dir.glob("eval-*")): + search_dir = benchmark_dir + else: + print(f"No eval directories found in {benchmark_dir} or {benchmark_dir / 'runs'}") + return {} + + results: dict[str, list] = {} + + for eval_idx, eval_dir in enumerate(sorted(search_dir.glob("eval-*"))): + metadata_path = eval_dir / "eval_metadata.json" + if metadata_path.exists(): + try: + with open(metadata_path) as mf: + eval_id = json.load(mf).get("eval_id", eval_idx) + except (json.JSONDecodeError, OSError): + eval_id = eval_idx + else: + try: + eval_id = int(eval_dir.name.split("-")[1]) + except ValueError: + eval_id = eval_idx + + # Discover config directories dynamically rather than hardcoding names + for config_dir in sorted(eval_dir.iterdir()): + if not config_dir.is_dir(): + continue + # Skip non-config directories (inputs, outputs, etc.) + if not list(config_dir.glob("run-*")): + continue + config = config_dir.name + if config not in results: + results[config] = [] + + for run_dir in sorted(config_dir.glob("run-*")): + run_number = int(run_dir.name.split("-")[1]) + grading_file = run_dir / "grading.json" + + if not grading_file.exists(): + print(f"Warning: grading.json not found in {run_dir}") + continue + + try: + with open(grading_file) as f: + grading = json.load(f) + except json.JSONDecodeError as e: + print(f"Warning: Invalid JSON in {grading_file}: {e}") + continue + + # Extract metrics + result = { + "eval_id": eval_id, + "run_number": run_number, + "pass_rate": grading.get("summary", {}).get("pass_rate", 0.0), + "passed": grading.get("summary", {}).get("passed", 0), + "failed": grading.get("summary", {}).get("failed", 0), + "total": grading.get("summary", {}).get("total", 0), + } + + # Extract timing — check grading.json first, then sibling timing.json + timing = grading.get("timing", {}) + result["time_seconds"] = timing.get("total_duration_seconds", 0.0) + timing_file = run_dir / "timing.json" + if result["time_seconds"] == 0.0 and timing_file.exists(): + try: + with open(timing_file) as tf: + timing_data = json.load(tf) + result["time_seconds"] = timing_data.get("total_duration_seconds", 0.0) + result["tokens"] = timing_data.get("total_tokens", 0) + except json.JSONDecodeError: + pass + + # Extract metrics if available + metrics = grading.get("execution_metrics", {}) + result["tool_calls"] = metrics.get("total_tool_calls", 0) + if not result.get("tokens"): + result["tokens"] = metrics.get("output_chars", 0) + result["errors"] = metrics.get("errors_encountered", 0) + + # Extract expectations — viewer requires fields: text, passed, evidence + raw_expectations = grading.get("expectations", []) + for exp in raw_expectations: + if "text" not in exp or "passed" not in exp: + print(f"Warning: expectation in {grading_file} missing required fields (text, passed, evidence): {exp}") + result["expectations"] = raw_expectations + + # Extract notes from user_notes_summary + notes_summary = grading.get("user_notes_summary", {}) + notes = [] + notes.extend(notes_summary.get("uncertainties", [])) + notes.extend(notes_summary.get("needs_review", [])) + notes.extend(notes_summary.get("workarounds", [])) + result["notes"] = notes + + results[config].append(result) + + return results + + +def aggregate_results(results: dict) -> dict: + """ + Aggregate run results into summary statistics. + + Returns run_summary with stats for each configuration and delta. + """ + run_summary = {} + configs = list(results.keys()) + + for config in configs: + runs = results.get(config, []) + + if not runs: + run_summary[config] = { + "pass_rate": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}, + "time_seconds": {"mean": 0.0, "stddev": 0.0, "min": 0.0, "max": 0.0}, + "tokens": {"mean": 0, "stddev": 0, "min": 0, "max": 0} + } + continue + + pass_rates = [r["pass_rate"] for r in runs] + times = [r["time_seconds"] for r in runs] + tokens = [r.get("tokens", 0) for r in runs] + + run_summary[config] = { + "pass_rate": calculate_stats(pass_rates), + "time_seconds": calculate_stats(times), + "tokens": calculate_stats(tokens) + } + + # Calculate delta between the first two configs (if two exist) + if len(configs) >= 2: + primary = run_summary.get(configs[0], {}) + baseline = run_summary.get(configs[1], {}) + else: + primary = run_summary.get(configs[0], {}) if configs else {} + baseline = {} + + delta_pass_rate = primary.get("pass_rate", {}).get("mean", 0) - baseline.get("pass_rate", {}).get("mean", 0) + delta_time = primary.get("time_seconds", {}).get("mean", 0) - baseline.get("time_seconds", {}).get("mean", 0) + delta_tokens = primary.get("tokens", {}).get("mean", 0) - baseline.get("tokens", {}).get("mean", 0) + + run_summary["delta"] = { + "pass_rate": f"{delta_pass_rate:+.2f}", + "time_seconds": f"{delta_time:+.1f}", + "tokens": f"{delta_tokens:+.0f}" + } + + return run_summary + + +def generate_benchmark(benchmark_dir: Path, skill_name: str = "", skill_path: str = "") -> dict: + """ + Generate complete benchmark.json from run results. + """ + results = load_run_results(benchmark_dir) + run_summary = aggregate_results(results) + + # Build runs array for benchmark.json + runs = [] + for config in results: + for result in results[config]: + runs.append({ + "eval_id": result["eval_id"], + "configuration": config, + "run_number": result["run_number"], + "result": { + "pass_rate": result["pass_rate"], + "passed": result["passed"], + "failed": result["failed"], + "total": result["total"], + "time_seconds": result["time_seconds"], + "tokens": result.get("tokens", 0), + "tool_calls": result.get("tool_calls", 0), + "errors": result.get("errors", 0) + }, + "expectations": result["expectations"], + "notes": result["notes"] + }) + + # Determine eval IDs from results + eval_ids = sorted(set( + r["eval_id"] + for config in results.values() + for r in config + )) + + benchmark = { + "metadata": { + "skill_name": skill_name or "", + "skill_path": skill_path or "", + "executor_model": "", + "analyzer_model": "", + "timestamp": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "evals_run": eval_ids, + "runs_per_configuration": 3 + }, + "runs": runs, + "run_summary": run_summary, + "notes": [] # To be filled by analyzer + } + + return benchmark + + +def generate_markdown(benchmark: dict) -> str: + """Generate human-readable benchmark.md from benchmark data.""" + metadata = benchmark["metadata"] + run_summary = benchmark["run_summary"] + + # Determine config names (excluding "delta") + configs = [k for k in run_summary if k != "delta"] + config_a = configs[0] if len(configs) >= 1 else "config_a" + config_b = configs[1] if len(configs) >= 2 else "config_b" + label_a = config_a.replace("_", " ").title() + label_b = config_b.replace("_", " ").title() + + lines = [ + f"# Skill Benchmark: {metadata['skill_name']}", + "", + f"**Model**: {metadata['executor_model']}", + f"**Date**: {metadata['timestamp']}", + f"**Evals**: {', '.join(map(str, metadata['evals_run']))} ({metadata['runs_per_configuration']} runs each per configuration)", + "", + "## Summary", + "", + f"| Metric | {label_a} | {label_b} | Delta |", + "|--------|------------|---------------|-------|", + ] + + a_summary = run_summary.get(config_a, {}) + b_summary = run_summary.get(config_b, {}) + delta = run_summary.get("delta", {}) + + # Format pass rate + a_pr = a_summary.get("pass_rate", {}) + b_pr = b_summary.get("pass_rate", {}) + lines.append(f"| Pass Rate | {a_pr.get('mean', 0)*100:.0f}% ± {a_pr.get('stddev', 0)*100:.0f}% | {b_pr.get('mean', 0)*100:.0f}% ± {b_pr.get('stddev', 0)*100:.0f}% | {delta.get('pass_rate', '—')} |") + + # Format time + a_time = a_summary.get("time_seconds", {}) + b_time = b_summary.get("time_seconds", {}) + lines.append(f"| Time | {a_time.get('mean', 0):.1f}s ± {a_time.get('stddev', 0):.1f}s | {b_time.get('mean', 0):.1f}s ± {b_time.get('stddev', 0):.1f}s | {delta.get('time_seconds', '—')}s |") + + # Format tokens + a_tokens = a_summary.get("tokens", {}) + b_tokens = b_summary.get("tokens", {}) + lines.append(f"| Tokens | {a_tokens.get('mean', 0):.0f} ± {a_tokens.get('stddev', 0):.0f} | {b_tokens.get('mean', 0):.0f} ± {b_tokens.get('stddev', 0):.0f} | {delta.get('tokens', '—')} |") + + # Notes section + if benchmark.get("notes"): + lines.extend([ + "", + "## Notes", + "" + ]) + for note in benchmark["notes"]: + lines.append(f"- {note}") + + return "\n".join(lines) + + +def main(): + parser = argparse.ArgumentParser( + description="Aggregate benchmark run results into summary statistics" + ) + parser.add_argument( + "benchmark_dir", + type=Path, + help="Path to the benchmark directory" + ) + parser.add_argument( + "--skill-name", + default="", + help="Name of the skill being benchmarked" + ) + parser.add_argument( + "--skill-path", + default="", + help="Path to the skill being benchmarked" + ) + parser.add_argument( + "--output", "-o", + type=Path, + help="Output path for benchmark.json (default: /benchmark.json)" + ) + + args = parser.parse_args() + + if not args.benchmark_dir.exists(): + print(f"Directory not found: {args.benchmark_dir}") + sys.exit(1) + + # Generate benchmark + benchmark = generate_benchmark(args.benchmark_dir, args.skill_name, args.skill_path) + + # Determine output paths + output_json = args.output or (args.benchmark_dir / "benchmark.json") + output_md = output_json.with_suffix(".md") + + # Write benchmark.json + with open(output_json, "w") as f: + json.dump(benchmark, f, indent=2) + print(f"Generated: {output_json}") + + # Write benchmark.md + markdown = generate_markdown(benchmark) + with open(output_md, "w") as f: + f.write(markdown) + print(f"Generated: {output_md}") + + # Print summary + run_summary = benchmark["run_summary"] + configs = [k for k in run_summary if k != "delta"] + delta = run_summary.get("delta", {}) + + print(f"\nSummary:") + for config in configs: + pr = run_summary[config]["pass_rate"]["mean"] + label = config.replace("_", " ").title() + print(f" {label}: {pr*100:.1f}% pass rate") + print(f" Delta: {delta.get('pass_rate', '—')}") + + +if __name__ == "__main__": + main() diff --git a/skill-creator/scripts/generate_report.py b/skill-creator/scripts/generate_report.py new file mode 100755 index 00000000..959e30a0 --- /dev/null +++ b/skill-creator/scripts/generate_report.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Generate an HTML report from run_loop.py output. + +Takes the JSON output from run_loop.py and generates a visual HTML report +showing each description attempt with check/x for each test case. +Distinguishes between train and test queries. +""" + +import argparse +import html +import json +import sys +from pathlib import Path + + +def generate_html(data: dict, auto_refresh: bool = False, skill_name: str = "") -> str: + """Generate HTML report from loop output data. If auto_refresh is True, adds a meta refresh tag.""" + history = data.get("history", []) + holdout = data.get("holdout", 0) + title_prefix = html.escape(skill_name + " \u2014 ") if skill_name else "" + + # Get all unique queries from train and test sets, with should_trigger info + train_queries: list[dict] = [] + test_queries: list[dict] = [] + if history: + for r in history[0].get("train_results", history[0].get("results", [])): + train_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)}) + if history[0].get("test_results"): + for r in history[0].get("test_results", []): + test_queries.append({"query": r["query"], "should_trigger": r.get("should_trigger", True)}) + + refresh_tag = ' \n' if auto_refresh else "" + + html_parts = [""" + + + +""" + refresh_tag + """ """ + title_prefix + """Skill Description Optimization + + + + + + +

""" + title_prefix + """Skill Description Optimization

+
+ Optimizing your skill's description. This page updates automatically as Claude tests different versions of your skill's description. Each row is an iteration — a new description attempt. The columns show test queries: green checkmarks mean the skill triggered correctly (or correctly didn't trigger), red crosses mean it got it wrong. The "Train" score shows performance on queries used to improve the description; the "Test" score shows performance on held-out queries the optimizer hasn't seen. When it's done, Claude will apply the best-performing description to your skill. +
+"""] + + # Summary section + best_test_score = data.get('best_test_score') + best_train_score = data.get('best_train_score') + html_parts.append(f""" +
+

Original: {html.escape(data.get('original_description', 'N/A'))}

+

Best: {html.escape(data.get('best_description', 'N/A'))}

+

Best Score: {data.get('best_score', 'N/A')} {'(test)' if best_test_score else '(train)'}

+

Iterations: {data.get('iterations_run', 0)} | Train: {data.get('train_size', '?')} | Test: {data.get('test_size', '?')}

+
+""") + + # Legend + html_parts.append(""" +
+ Query columns: + Should trigger + Should NOT trigger + Train + Test +
+""") + + # Table header + html_parts.append(""" +
+ + + + + + + +""") + + # Add column headers for train queries + for qinfo in train_queries: + polarity = "positive-col" if qinfo["should_trigger"] else "negative-col" + html_parts.append(f' \n') + + # Add column headers for test queries (different color) + for qinfo in test_queries: + polarity = "positive-col" if qinfo["should_trigger"] else "negative-col" + html_parts.append(f' \n') + + html_parts.append(""" + + +""") + + # Find best iteration for highlighting + if test_queries: + best_iter = max(history, key=lambda h: h.get("test_passed") or 0).get("iteration") + else: + best_iter = max(history, key=lambda h: h.get("train_passed", h.get("passed", 0))).get("iteration") + + # Add rows for each iteration + for h in history: + iteration = h.get("iteration", "?") + train_passed = h.get("train_passed", h.get("passed", 0)) + train_total = h.get("train_total", h.get("total", 0)) + test_passed = h.get("test_passed") + test_total = h.get("test_total") + description = h.get("description", "") + train_results = h.get("train_results", h.get("results", [])) + test_results = h.get("test_results", []) + + # Create lookups for results by query + train_by_query = {r["query"]: r for r in train_results} + test_by_query = {r["query"]: r for r in test_results} if test_results else {} + + # Compute aggregate correct/total runs across all retries + def aggregate_runs(results: list[dict]) -> tuple[int, int]: + correct = 0 + total = 0 + for r in results: + runs = r.get("runs", 0) + triggers = r.get("triggers", 0) + total += runs + if r.get("should_trigger", True): + correct += triggers + else: + correct += runs - triggers + return correct, total + + train_correct, train_runs = aggregate_runs(train_results) + test_correct, test_runs = aggregate_runs(test_results) + + # Determine score classes + def score_class(correct: int, total: int) -> str: + if total > 0: + ratio = correct / total + if ratio >= 0.8: + return "score-good" + elif ratio >= 0.5: + return "score-ok" + return "score-bad" + + train_class = score_class(train_correct, train_runs) + test_class = score_class(test_correct, test_runs) + + row_class = "best-row" if iteration == best_iter else "" + + html_parts.append(f""" + + + + +""") + + # Add result for each train query + for qinfo in train_queries: + r = train_by_query.get(qinfo["query"], {}) + did_pass = r.get("pass", False) + triggers = r.get("triggers", 0) + runs = r.get("runs", 0) + + icon = "✓" if did_pass else "✗" + css_class = "pass" if did_pass else "fail" + + html_parts.append(f' \n') + + # Add result for each test query (with different background) + for qinfo in test_queries: + r = test_by_query.get(qinfo["query"], {}) + did_pass = r.get("pass", False) + triggers = r.get("triggers", 0) + runs = r.get("runs", 0) + + icon = "✓" if did_pass else "✗" + css_class = "pass" if did_pass else "fail" + + html_parts.append(f' \n') + + html_parts.append(" \n") + + html_parts.append(""" +
IterTrainTestDescription{html.escape(qinfo["query"])}{html.escape(qinfo["query"])}
{iteration}{train_correct}/{train_runs}{test_correct}/{test_runs}{html.escape(description)}{icon}{triggers}/{runs}{icon}{triggers}/{runs}
+
+""") + + html_parts.append(""" + + +""") + + return "".join(html_parts) + + +def main(): + parser = argparse.ArgumentParser(description="Generate HTML report from run_loop output") + parser.add_argument("input", help="Path to JSON output from run_loop.py (or - for stdin)") + parser.add_argument("-o", "--output", default=None, help="Output HTML file (default: stdout)") + parser.add_argument("--skill-name", default="", help="Skill name to include in the report title") + args = parser.parse_args() + + if args.input == "-": + data = json.load(sys.stdin) + else: + data = json.loads(Path(args.input).read_text()) + + html_output = generate_html(data, skill_name=args.skill_name) + + if args.output: + Path(args.output).write_text(html_output) + print(f"Report written to {args.output}", file=sys.stderr) + else: + print(html_output) + + +if __name__ == "__main__": + main() diff --git a/skill-creator/scripts/improve_description.py b/skill-creator/scripts/improve_description.py new file mode 100755 index 00000000..06bcec76 --- /dev/null +++ b/skill-creator/scripts/improve_description.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +"""Improve a skill description based on eval results. + +Takes eval results (from run_eval.py) and generates an improved description +by calling `claude -p` as a subprocess (same auth pattern as run_eval.py — +uses the session's Claude Code auth, no separate ANTHROPIC_API_KEY needed). +""" + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path + +from scripts.utils import parse_skill_md + + +def _call_claude(prompt: str, model: str | None, timeout: int = 300) -> str: + """Run `claude -p` with the prompt on stdin and return the text response. + + Prompt goes over stdin (not argv) because it embeds the full SKILL.md + body and can easily exceed comfortable argv length. + """ + cmd = ["claude", "-p", "--output-format", "text"] + if model: + cmd.extend(["--model", model]) + + # Remove CLAUDECODE env var to allow nesting claude -p inside a + # Claude Code session. The guard is for interactive terminal conflicts; + # programmatic subprocess usage is safe. Same pattern as run_eval.py. + env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"} + + result = subprocess.run( + cmd, + input=prompt, + capture_output=True, + text=True, + env=env, + timeout=timeout, + ) + if result.returncode != 0: + raise RuntimeError( + f"claude -p exited {result.returncode}\nstderr: {result.stderr}" + ) + return result.stdout + + +def improve_description( + skill_name: str, + skill_content: str, + current_description: str, + eval_results: dict, + history: list[dict], + model: str, + test_results: dict | None = None, + log_dir: Path | None = None, + iteration: int | None = None, +) -> str: + """Call Claude to improve the description based on eval results.""" + failed_triggers = [ + r for r in eval_results["results"] + if r["should_trigger"] and not r["pass"] + ] + false_triggers = [ + r for r in eval_results["results"] + if not r["should_trigger"] and not r["pass"] + ] + + # Build scores summary + train_score = f"{eval_results['summary']['passed']}/{eval_results['summary']['total']}" + if test_results: + test_score = f"{test_results['summary']['passed']}/{test_results['summary']['total']}" + scores_summary = f"Train: {train_score}, Test: {test_score}" + else: + scores_summary = f"Train: {train_score}" + + prompt = f"""You are optimizing a skill description for a Claude Code skill called "{skill_name}". A "skill" is sort of like a prompt, but with progressive disclosure -- there's a title and description that Claude sees when deciding whether to use the skill, and then if it does use the skill, it reads the .md file which has lots more details and potentially links to other resources in the skill folder like helper files and scripts and additional documentation or examples. + +The description appears in Claude's "available_skills" list. When a user sends a query, Claude decides whether to invoke the skill based solely on the title and on this description. Your goal is to write a description that triggers for relevant queries, and doesn't trigger for irrelevant ones. + +Here's the current description: + +"{current_description}" + + +Current scores ({scores_summary}): + +""" + if failed_triggers: + prompt += "FAILED TO TRIGGER (should have triggered but didn't):\n" + for r in failed_triggers: + prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n' + prompt += "\n" + + if false_triggers: + prompt += "FALSE TRIGGERS (triggered but shouldn't have):\n" + for r in false_triggers: + prompt += f' - "{r["query"]}" (triggered {r["triggers"]}/{r["runs"]} times)\n' + prompt += "\n" + + if history: + prompt += "PREVIOUS ATTEMPTS (do NOT repeat these — try something structurally different):\n\n" + for h in history: + train_s = f"{h.get('train_passed', h.get('passed', 0))}/{h.get('train_total', h.get('total', 0))}" + test_s = f"{h.get('test_passed', '?')}/{h.get('test_total', '?')}" if h.get('test_passed') is not None else None + score_str = f"train={train_s}" + (f", test={test_s}" if test_s else "") + prompt += f'\n' + prompt += f'Description: "{h["description"]}"\n' + if "results" in h: + prompt += "Train results:\n" + for r in h["results"]: + status = "PASS" if r["pass"] else "FAIL" + prompt += f' [{status}] "{r["query"][:80]}" (triggered {r["triggers"]}/{r["runs"]})\n' + if h.get("note"): + prompt += f'Note: {h["note"]}\n' + prompt += "\n\n" + + prompt += f""" + +Skill content (for context on what the skill does): + +{skill_content} + + +Based on the failures, write a new and improved description that is more likely to trigger correctly. When I say "based on the failures", it's a bit of a tricky line to walk because we don't want to overfit to the specific cases you're seeing. So what I DON'T want you to do is produce an ever-expanding list of specific queries that this skill should or shouldn't trigger for. Instead, try to generalize from the failures to broader categories of user intent and situations where this skill would be useful or not useful. The reason for this is twofold: + +1. Avoid overfitting +2. The list might get loooong and it's injected into ALL queries and there might be a lot of skills, so we don't want to blow too much space on any given description. + +Concretely, your description should not be more than about 100-200 words, even if that comes at the cost of accuracy. There is a hard limit of 1024 characters — descriptions over that will be truncated, so stay comfortably under it. + +Here are some tips that we've found to work well in writing these descriptions: +- The skill should be phrased in the imperative -- "Use this skill for" rather than "this skill does" +- The skill description should focus on the user's intent, what they are trying to achieve, vs. the implementation details of how the skill works. +- The description competes with other skills for Claude's attention — make it distinctive and immediately recognizable. +- If you're getting lots of failures after repeated attempts, change things up. Try different sentence structures or wordings. + +I'd encourage you to be creative and mix up the style in different iterations since you'll have multiple opportunities to try different approaches and we'll just grab the highest-scoring one at the end. + +Please respond with only the new description text in tags, nothing else.""" + + text = _call_claude(prompt, model) + + match = re.search(r"(.*?)", text, re.DOTALL) + description = match.group(1).strip().strip('"') if match else text.strip().strip('"') + + transcript: dict = { + "iteration": iteration, + "prompt": prompt, + "response": text, + "parsed_description": description, + "char_count": len(description), + "over_limit": len(description) > 1024, + } + + # Safety net: the prompt already states the 1024-char hard limit, but if + # the model blew past it anyway, make one fresh single-turn call that + # quotes the too-long version and asks for a shorter rewrite. (The old + # SDK path did this as a true multi-turn; `claude -p` is one-shot, so we + # inline the prior output into the new prompt instead.) + if len(description) > 1024: + shorten_prompt = ( + f"{prompt}\n\n" + f"---\n\n" + f"A previous attempt produced this description, which at " + f"{len(description)} characters is over the 1024-character hard limit:\n\n" + f'"{description}"\n\n' + f"Rewrite it to be under 1024 characters while keeping the most " + f"important trigger words and intent coverage. Respond with only " + f"the new description in tags." + ) + shorten_text = _call_claude(shorten_prompt, model) + match = re.search(r"(.*?)", shorten_text, re.DOTALL) + shortened = match.group(1).strip().strip('"') if match else shorten_text.strip().strip('"') + + transcript["rewrite_prompt"] = shorten_prompt + transcript["rewrite_response"] = shorten_text + transcript["rewrite_description"] = shortened + transcript["rewrite_char_count"] = len(shortened) + description = shortened + + transcript["final_description"] = description + + if log_dir: + log_dir.mkdir(parents=True, exist_ok=True) + log_file = log_dir / f"improve_iter_{iteration or 'unknown'}.json" + log_file.write_text(json.dumps(transcript, indent=2)) + + return description + + +def main(): + parser = argparse.ArgumentParser(description="Improve a skill description based on eval results") + parser.add_argument("--eval-results", required=True, help="Path to eval results JSON (from run_eval.py)") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--history", default=None, help="Path to history JSON (previous attempts)") + parser.add_argument("--model", required=True, help="Model for improvement") + parser.add_argument("--verbose", action="store_true", help="Print thinking to stderr") + args = parser.parse_args() + + skill_path = Path(args.skill_path) + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + eval_results = json.loads(Path(args.eval_results).read_text()) + history = [] + if args.history: + history = json.loads(Path(args.history).read_text()) + + name, _, content = parse_skill_md(skill_path) + current_description = eval_results["description"] + + if args.verbose: + print(f"Current: {current_description}", file=sys.stderr) + print(f"Score: {eval_results['summary']['passed']}/{eval_results['summary']['total']}", file=sys.stderr) + + new_description = improve_description( + skill_name=name, + skill_content=content, + current_description=current_description, + eval_results=eval_results, + history=history, + model=args.model, + ) + + if args.verbose: + print(f"Improved: {new_description}", file=sys.stderr) + + # Output as JSON with both the new description and updated history + output = { + "description": new_description, + "history": history + [{ + "description": current_description, + "passed": eval_results["summary"]["passed"], + "failed": eval_results["summary"]["failed"], + "total": eval_results["summary"]["total"], + "results": eval_results["results"], + }], + } + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/skill-creator/scripts/package_skill.py b/skill-creator/scripts/package_skill.py index eb665363..506b8f63 100755 --- a/skill-creator/scripts/package_skill.py +++ b/skill-creator/scripts/package_skill.py @@ -1,24 +1,54 @@ #!/usr/bin/env python3 """ -Skill Packager - Creates a distributable zip file of a skill folder +Skill Packager - Creates a distributable .skill file of a skill folder Usage: - python utils/package_skill.py [output-directory] + uv run --with PyYAML python -m scripts.package_skill [output-directory] Example: - python utils/package_skill.py skills/public/my-skill - python utils/package_skill.py skills/public/my-skill ./dist + uv run --with PyYAML python -m scripts.package_skill skills/public/my-skill + uv run --with PyYAML python -m scripts.package_skill skills/public/my-skill ./dist """ +import fnmatch +import re import sys import zipfile -import re from pathlib import Path -from quick_validate import validate_skill -from security_scan import calculate_skill_hash - - -def validate_security_marker(skill_path: Path) -> tuple[bool, str]: +from typing import Optional, Tuple + +SCRIPT_DIR = Path(__file__).resolve().parent +PACKAGE_ROOT = SCRIPT_DIR.parent +if str(PACKAGE_ROOT) not in sys.path: + sys.path.insert(0, str(PACKAGE_ROOT)) + +from scripts.quick_validate import validate_skill +from scripts.security_scan import calculate_skill_hash + +# Patterns to exclude when packaging skills. +EXCLUDE_DIRS = {"__pycache__", "node_modules"} +EXCLUDE_GLOBS = {"*.pyc"} +EXCLUDE_FILES = {".DS_Store"} +# Directories excluded only at the skill root (not when nested deeper). +ROOT_EXCLUDE_DIRS = {"evals"} + + +def should_exclude(rel_path: Path) -> bool: + """Check if a path should be excluded from packaging.""" + parts = rel_path.parts + if any(part in EXCLUDE_DIRS for part in parts): + return True + # rel_path is relative to skill_path.parent, so parts[0] is the skill + # folder name and parts[1] (if present) is the first subdir. + if len(parts) > 1 and parts[1] in ROOT_EXCLUDE_DIRS: + return True + name = rel_path.name + if name in EXCLUDE_FILES: + return True + return any(fnmatch.fnmatch(name, pat) for pat in EXCLUDE_GLOBS) + + +def validate_security_marker(skill_path: Path) -> Tuple[bool, str]: """ Validate security marker file exists and hash matches current content @@ -58,54 +88,54 @@ def validate_security_marker(skill_path: Path) -> tuple[bool, str]: def package_skill(skill_path, output_dir=None): """ - Package a skill folder into a zip file. + Package a skill folder into a .skill file. Args: skill_path: Path to the skill folder - output_dir: Optional output directory for the zip file (defaults to current directory) + output_dir: Optional output directory for the .skill file (defaults to current directory) Returns: - Path to the created zip file, or None if error + Path to the created .skill file, or None if error """ skill_path = Path(skill_path).resolve() # Validate skill folder exists if not skill_path.exists(): - print(f"❌ Error: Skill folder not found: {skill_path}") + print(f"Error: Skill folder not found: {skill_path}") return None if not skill_path.is_dir(): - print(f"❌ Error: Path is not a directory: {skill_path}") + print(f"Error: Path is not a directory: {skill_path}") return None # Validate SKILL.md exists skill_md = skill_path / "SKILL.md" if not skill_md.exists(): - print(f"❌ Error: SKILL.md not found in {skill_path}") + print(f"Error: SKILL.md not found in {skill_path}") return None # Step 1: Validate skill structure and metadata - print("🔍 Step 1: Validating skill structure...") + print("Step 1: Validating skill structure...") valid, message = validate_skill(skill_path) if not valid: - print(f"❌ FAILED: {message}") + print(f"FAILED: {message}") print(" Fix validation errors before packaging.") return None - print(f"✅ PASSED: {message}\n") + print(f"PASSED: {message}\n") # Step 2: Validate security scan (HARD REQUIREMENT) - print("🔍 Step 2: Validating security scan...") + print("Step 2: Validating security scan...") is_valid, message = validate_security_marker(skill_path) if not is_valid: - print(f"❌ BLOCKED: {message}") + print(f"BLOCKED: {message}") print(f" You MUST run: python scripts/security_scan.py {skill_path.name}") print(" Security review is MANDATORY before packaging.") return None - print(f"✅ PASSED: {message}\n") + print(f"PASSED: {message}\n") # Step 3: Package the skill - print("📦 Step 3: Creating package...") + print("Step 3: Creating package...") # Determine output location skill_name = skill_path.name @@ -115,39 +145,42 @@ def package_skill(skill_path, output_dir=None): else: output_path = Path.cwd() - zip_filename = output_path / f"{skill_name}.zip" + skill_filename = output_path / f"{skill_name}.skill" - # Create the zip file + # Create the .skill file (zip format) try: - with zipfile.ZipFile(zip_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: - # Walk through the skill directory + with zipfile.ZipFile(skill_filename, 'w', zipfile.ZIP_DEFLATED) as zipf: + # Walk through the skill directory, excluding build artifacts for file_path in skill_path.rglob('*'): - if file_path.is_file(): - # Calculate the relative path within the zip - arcname = file_path.relative_to(skill_path.parent) - zipf.write(file_path, arcname) - print(f" Added: {arcname}") - - print(f"\n✅ Successfully packaged skill to: {zip_filename}") - return zip_filename + if not file_path.is_file(): + continue + arcname = file_path.relative_to(skill_path.parent) + if should_exclude(arcname): + print(f" Skipped: {arcname}") + continue + zipf.write(file_path, arcname) + print(f" Added: {arcname}") + + print(f"\nSuccessfully packaged skill to: {skill_filename}") + return skill_filename except Exception as e: - print(f"❌ Error creating zip file: {e}") + print(f"Error creating .skill file: {e}") return None def main(): if len(sys.argv) < 2: - print("Usage: python utils/package_skill.py [output-directory]") + print("Usage: uv run --with PyYAML python -m scripts.package_skill [output-directory]") print("\nExample:") - print(" python utils/package_skill.py skills/public/my-skill") - print(" python utils/package_skill.py skills/public/my-skill ./dist") + print(" uv run --with PyYAML python -m scripts.package_skill skills/public/my-skill") + print(" uv run --with PyYAML python -m scripts.package_skill skills/public/my-skill ./dist") sys.exit(1) skill_path = sys.argv[1] output_dir = sys.argv[2] if len(sys.argv) > 2 else None - print(f"📦 Packaging skill: {skill_path}") + print(f"Packaging skill: {skill_path}") if output_dir: print(f" Output directory: {output_dir}") print() diff --git a/skill-creator/scripts/quick_validate.py b/skill-creator/scripts/quick_validate.py index 2d022000..2a24e18d 100755 --- a/skill-creator/scripts/quick_validate.py +++ b/skill-creator/scripts/quick_validate.py @@ -8,6 +8,21 @@ import re from pathlib import Path +try: + import yaml +except ModuleNotFoundError: + print( + "Missing dependency: PyYAML.\n" + "Run validation with an explicit dependency declaration:\n" + " uv run --with PyYAML python skill-creator/scripts/quick_validate.py \n" + "Or from the skill-creator directory:\n" + " uv run --with PyYAML python -m scripts.quick_validate \n" + "For packaging from the skill-creator directory:\n" + " uv run --with PyYAML python -m scripts.package_skill ", + file=sys.stderr, + ) + sys.exit(2) + def find_invalid_frontmatter_indentation(frontmatter: str) -> list[tuple[int, str]]: """ @@ -57,7 +72,7 @@ def find_path_references(content: str) -> list[str]: if any(x in line_lower for x in [ 'example:', 'examples:', 'e.g.', 'for example', '- **example', '- example:', 'such as', - 'pattern:', 'usage:', '❌', '✅', + 'pattern:', 'usage:', '\u274c', '\u2705', '- **allowed', '- **best practice', 'would be helpful', 'like `scripts/', 'like `references/', 'like `assets/', ]): @@ -92,6 +107,14 @@ def validate_path_references(skill_path: Path, content: str) -> tuple[bool, list return len(missing) == 0, missing +# Define allowed properties (union of official and our extensions) +ALLOWED_PROPERTIES = { + 'name', 'description', 'license', 'allowed-tools', 'metadata', + 'compatibility', 'context', 'agent', 'disable-model-invocation', + 'user-invocable', 'model', 'argument-hint', 'hooks', +} + + def validate_skill(skill_path): """Basic validation of a skill""" skill_path = Path(skill_path) @@ -111,10 +134,10 @@ def validate_skill(skill_path): if not match: return False, "Invalid frontmatter format" - frontmatter = match.group(1) + frontmatter_text = match.group(1) # Check for invalid indentation characters in frontmatter - invalid_indent = find_invalid_frontmatter_indentation(frontmatter) + invalid_indent = find_invalid_frontmatter_indentation(frontmatter_text) if invalid_indent: samples = ", ".join( f"line {line_no} ({describe_whitespace(ch)})" @@ -126,29 +149,62 @@ def validate_skill(skill_path): f"Found: {samples}{more}" ) + # Parse YAML frontmatter + try: + frontmatter = yaml.safe_load(frontmatter_text) + if not isinstance(frontmatter, dict): + return False, "Frontmatter must be a YAML dictionary" + except yaml.YAMLError as e: + return False, f"Invalid YAML in frontmatter: {e}" + + # Check for unexpected properties + unexpected_keys = set(frontmatter.keys()) - ALLOWED_PROPERTIES + if unexpected_keys: + return False, ( + f"Unexpected key(s) in SKILL.md frontmatter: {', '.join(sorted(unexpected_keys))}. " + f"Allowed properties are: {', '.join(sorted(ALLOWED_PROPERTIES))}" + ) + # Check required fields - if 'name:' not in frontmatter: - return False, "Missing 'name' in frontmatter" - if 'description:' not in frontmatter: + if 'description' not in frontmatter: return False, "Missing 'description' in frontmatter" - # Extract name for validation - name_match = re.search(r'name:\s*(.+)', frontmatter) - if name_match: - name = name_match.group(1).strip() - # Check naming convention (hyphen-case: lowercase with hyphens) - if not re.match(r'^[a-z0-9-]+$', name): - return False, f"Name '{name}' should be hyphen-case (lowercase letters, digits, and hyphens only)" - if name.startswith('-') or name.endswith('-') or '--' in name: - return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens" + # Extract name for validation (optional per official spec, but validate if present) + name = frontmatter.get('name', '') + if isinstance(name, str): + name = name.strip() + if name: + # Check naming convention (kebab-case: lowercase with hyphens) + if not re.match(r'^[a-z0-9-]+$', name): + return False, f"Name '{name}' should be kebab-case (lowercase letters, digits, and hyphens only)" + if name.startswith('-') or name.endswith('-') or '--' in name: + return False, f"Name '{name}' cannot start/end with hyphen or contain consecutive hyphens" + # Check name length (max 64 characters per spec) + if len(name) > 64: + return False, f"Name is too long ({len(name)} characters). Maximum is 64 characters." + elif name is not None: + return False, f"Name must be a string, got {type(name).__name__}" # Extract and validate description - desc_match = re.search(r'description:\s*(.+)', frontmatter) - if desc_match: - description = desc_match.group(1).strip() + description = frontmatter.get('description', '') + if not isinstance(description, str): + return False, f"Description must be a string, got {type(description).__name__}" + description = description.strip() + if description: # Check for angle brackets if '<' in description or '>' in description: return False, "Description cannot contain angle brackets (< or >)" + # Check description length (max 1024 characters per spec) + if len(description) > 1024: + return False, f"Description is too long ({len(description)} characters). Maximum is 1024 characters." + + # Validate compatibility field if present (optional) + compatibility = frontmatter.get('compatibility', '') + if compatibility: + if not isinstance(compatibility, str): + return False, f"Compatibility must be a string, got {type(compatibility).__name__}" + if len(compatibility) > 500: + return False, f"Compatibility is too long ({len(compatibility)} characters). Maximum is 500 characters." # Validate path references exist paths_valid, missing_paths = validate_path_references(skill_path, content) @@ -161,7 +217,7 @@ def validate_skill(skill_path): if len(sys.argv) != 2: print("Usage: python quick_validate.py ") sys.exit(1) - + valid, message = validate_skill(sys.argv[1]) print(message) sys.exit(0 if valid else 1) diff --git a/skill-creator/scripts/run_eval.py b/skill-creator/scripts/run_eval.py new file mode 100755 index 00000000..e58c70be --- /dev/null +++ b/skill-creator/scripts/run_eval.py @@ -0,0 +1,310 @@ +#!/usr/bin/env python3 +"""Run trigger evaluation for a skill description. + +Tests whether a skill's description causes Claude to trigger (read the skill) +for a set of queries. Outputs results as JSON. +""" + +import argparse +import json +import os +import select +import subprocess +import sys +import time +import uuid +from concurrent.futures import ProcessPoolExecutor, as_completed +from pathlib import Path + +from scripts.utils import parse_skill_md + + +def find_project_root() -> Path: + """Find the project root by walking up from cwd looking for .claude/. + + Mimics how Claude Code discovers its project root, so the command file + we create ends up where claude -p will look for it. + """ + current = Path.cwd() + for parent in [current, *current.parents]: + if (parent / ".claude").is_dir(): + return parent + return current + + +def run_single_query( + query: str, + skill_name: str, + skill_description: str, + timeout: int, + project_root: str, + model: str | None = None, +) -> bool: + """Run a single query and return whether the skill was triggered. + + Creates a command file in .claude/commands/ so it appears in Claude's + available_skills list, then runs `claude -p` with the raw query. + Uses --include-partial-messages to detect triggering early from + stream events (content_block_start) rather than waiting for the + full assistant message, which only arrives after tool execution. + """ + unique_id = uuid.uuid4().hex[:8] + clean_name = f"{skill_name}-skill-{unique_id}" + project_commands_dir = Path(project_root) / ".claude" / "commands" + command_file = project_commands_dir / f"{clean_name}.md" + + try: + project_commands_dir.mkdir(parents=True, exist_ok=True) + # Use YAML block scalar to avoid breaking on quotes in description + indented_desc = "\n ".join(skill_description.split("\n")) + command_content = ( + f"---\n" + f"description: |\n" + f" {indented_desc}\n" + f"---\n\n" + f"# {skill_name}\n\n" + f"This skill handles: {skill_description}\n" + ) + command_file.write_text(command_content) + + cmd = [ + "claude", + "-p", query, + "--output-format", "stream-json", + "--verbose", + "--include-partial-messages", + ] + if model: + cmd.extend(["--model", model]) + + # Remove CLAUDECODE env var to allow nesting claude -p inside a + # Claude Code session. The guard is for interactive terminal conflicts; + # programmatic subprocess usage is safe. + env = {k: v for k, v in os.environ.items() if k != "CLAUDECODE"} + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + cwd=project_root, + env=env, + ) + + triggered = False + start_time = time.time() + buffer = "" + # Track state for stream event detection + pending_tool_name = None + accumulated_json = "" + + try: + while time.time() - start_time < timeout: + if process.poll() is not None: + remaining = process.stdout.read() + if remaining: + buffer += remaining.decode("utf-8", errors="replace") + break + + ready, _, _ = select.select([process.stdout], [], [], 1.0) + if not ready: + continue + + chunk = os.read(process.stdout.fileno(), 8192) + if not chunk: + break + buffer += chunk.decode("utf-8", errors="replace") + + while "\n" in buffer: + line, buffer = buffer.split("\n", 1) + line = line.strip() + if not line: + continue + + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + + # Early detection via stream events + if event.get("type") == "stream_event": + se = event.get("event", {}) + se_type = se.get("type", "") + + if se_type == "content_block_start": + cb = se.get("content_block", {}) + if cb.get("type") == "tool_use": + tool_name = cb.get("name", "") + if tool_name in ("Skill", "Read"): + pending_tool_name = tool_name + accumulated_json = "" + else: + return False + + elif se_type == "content_block_delta" and pending_tool_name: + delta = se.get("delta", {}) + if delta.get("type") == "input_json_delta": + accumulated_json += delta.get("partial_json", "") + if clean_name in accumulated_json: + return True + + elif se_type in ("content_block_stop", "message_stop"): + if pending_tool_name: + return clean_name in accumulated_json + if se_type == "message_stop": + return False + + # Fallback: full assistant message + elif event.get("type") == "assistant": + message = event.get("message", {}) + for content_item in message.get("content", []): + if content_item.get("type") != "tool_use": + continue + tool_name = content_item.get("name", "") + tool_input = content_item.get("input", {}) + if tool_name == "Skill" and clean_name in tool_input.get("skill", ""): + triggered = True + elif tool_name == "Read" and clean_name in tool_input.get("file_path", ""): + triggered = True + return triggered + + elif event.get("type") == "result": + return triggered + finally: + # Clean up process on any exit path (return, exception, timeout) + if process.poll() is None: + process.kill() + process.wait() + + return triggered + finally: + if command_file.exists(): + command_file.unlink() + + +def run_eval( + eval_set: list[dict], + skill_name: str, + description: str, + num_workers: int, + timeout: int, + project_root: Path, + runs_per_query: int = 1, + trigger_threshold: float = 0.5, + model: str | None = None, +) -> dict: + """Run the full eval set and return results.""" + results = [] + + with ProcessPoolExecutor(max_workers=num_workers) as executor: + future_to_info = {} + for item in eval_set: + for run_idx in range(runs_per_query): + future = executor.submit( + run_single_query, + item["query"], + skill_name, + description, + timeout, + str(project_root), + model, + ) + future_to_info[future] = (item, run_idx) + + query_triggers: dict[str, list[bool]] = {} + query_items: dict[str, dict] = {} + for future in as_completed(future_to_info): + item, _ = future_to_info[future] + query = item["query"] + query_items[query] = item + if query not in query_triggers: + query_triggers[query] = [] + try: + query_triggers[query].append(future.result()) + except Exception as e: + print(f"Warning: query failed: {e}", file=sys.stderr) + query_triggers[query].append(False) + + for query, triggers in query_triggers.items(): + item = query_items[query] + trigger_rate = sum(triggers) / len(triggers) + should_trigger = item["should_trigger"] + if should_trigger: + did_pass = trigger_rate >= trigger_threshold + else: + did_pass = trigger_rate < trigger_threshold + results.append({ + "query": query, + "should_trigger": should_trigger, + "trigger_rate": trigger_rate, + "triggers": sum(triggers), + "runs": len(triggers), + "pass": did_pass, + }) + + passed = sum(1 for r in results if r["pass"]) + total = len(results) + + return { + "skill_name": skill_name, + "description": description, + "results": results, + "summary": { + "total": total, + "passed": passed, + "failed": total - passed, + }, + } + + +def main(): + parser = argparse.ArgumentParser(description="Run trigger evaluation for a skill description") + parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--description", default=None, help="Override description to test") + parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers") + parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds") + parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query") + parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold") + parser.add_argument("--model", default=None, help="Model to use for claude -p (default: user's configured model)") + parser.add_argument("--verbose", action="store_true", help="Print progress to stderr") + args = parser.parse_args() + + eval_set = json.loads(Path(args.eval_set).read_text()) + skill_path = Path(args.skill_path) + + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + name, original_description, content = parse_skill_md(skill_path) + description = args.description or original_description + project_root = find_project_root() + + if args.verbose: + print(f"Evaluating: {description}", file=sys.stderr) + + output = run_eval( + eval_set=eval_set, + skill_name=name, + description=description, + num_workers=args.num_workers, + timeout=args.timeout, + project_root=project_root, + runs_per_query=args.runs_per_query, + trigger_threshold=args.trigger_threshold, + model=args.model, + ) + + if args.verbose: + summary = output["summary"] + print(f"Results: {summary['passed']}/{summary['total']} passed", file=sys.stderr) + for r in output["results"]: + status = "PASS" if r["pass"] else "FAIL" + rate_str = f"{r['triggers']}/{r['runs']}" + print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:70]}", file=sys.stderr) + + print(json.dumps(output, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/skill-creator/scripts/run_loop.py b/skill-creator/scripts/run_loop.py new file mode 100755 index 00000000..30a263d6 --- /dev/null +++ b/skill-creator/scripts/run_loop.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Run the eval + improve loop until all pass or max iterations reached. + +Combines run_eval.py and improve_description.py in a loop, tracking history +and returning the best description found. Supports train/test split to prevent +overfitting. +""" + +import argparse +import json +import random +import sys +import tempfile +import time +import webbrowser +from pathlib import Path + +from scripts.generate_report import generate_html +from scripts.improve_description import improve_description +from scripts.run_eval import find_project_root, run_eval +from scripts.utils import parse_skill_md + + +def split_eval_set(eval_set: list[dict], holdout: float, seed: int = 42) -> tuple[list[dict], list[dict]]: + """Split eval set into train and test sets, stratified by should_trigger.""" + random.seed(seed) + + # Separate by should_trigger + trigger = [e for e in eval_set if e["should_trigger"]] + no_trigger = [e for e in eval_set if not e["should_trigger"]] + + # Shuffle each group + random.shuffle(trigger) + random.shuffle(no_trigger) + + # Calculate split points + n_trigger_test = max(1, int(len(trigger) * holdout)) + n_no_trigger_test = max(1, int(len(no_trigger) * holdout)) + + # Split + test_set = trigger[:n_trigger_test] + no_trigger[:n_no_trigger_test] + train_set = trigger[n_trigger_test:] + no_trigger[n_no_trigger_test:] + + return train_set, test_set + + +def run_loop( + eval_set: list[dict], + skill_path: Path, + description_override: str | None, + num_workers: int, + timeout: int, + max_iterations: int, + runs_per_query: int, + trigger_threshold: float, + holdout: float, + model: str, + verbose: bool, + live_report_path: Path | None = None, + log_dir: Path | None = None, +) -> dict: + """Run the eval + improvement loop.""" + project_root = find_project_root() + name, original_description, content = parse_skill_md(skill_path) + current_description = description_override or original_description + + # Split into train/test if holdout > 0 + if holdout > 0: + train_set, test_set = split_eval_set(eval_set, holdout) + if verbose: + print(f"Split: {len(train_set)} train, {len(test_set)} test (holdout={holdout})", file=sys.stderr) + else: + train_set = eval_set + test_set = [] + + history = [] + exit_reason = "unknown" + + for iteration in range(1, max_iterations + 1): + if verbose: + print(f"\n{'='*60}", file=sys.stderr) + print(f"Iteration {iteration}/{max_iterations}", file=sys.stderr) + print(f"Description: {current_description}", file=sys.stderr) + print(f"{'='*60}", file=sys.stderr) + + # Evaluate train + test together in one batch for parallelism + all_queries = train_set + test_set + t0 = time.time() + all_results = run_eval( + eval_set=all_queries, + skill_name=name, + description=current_description, + num_workers=num_workers, + timeout=timeout, + project_root=project_root, + runs_per_query=runs_per_query, + trigger_threshold=trigger_threshold, + model=model, + ) + eval_elapsed = time.time() - t0 + + # Split results back into train/test by matching queries + train_queries_set = {q["query"] for q in train_set} + train_result_list = [r for r in all_results["results"] if r["query"] in train_queries_set] + test_result_list = [r for r in all_results["results"] if r["query"] not in train_queries_set] + + train_passed = sum(1 for r in train_result_list if r["pass"]) + train_total = len(train_result_list) + train_summary = {"passed": train_passed, "failed": train_total - train_passed, "total": train_total} + train_results = {"results": train_result_list, "summary": train_summary} + + if test_set: + test_passed = sum(1 for r in test_result_list if r["pass"]) + test_total = len(test_result_list) + test_summary = {"passed": test_passed, "failed": test_total - test_passed, "total": test_total} + test_results = {"results": test_result_list, "summary": test_summary} + else: + test_results = None + test_summary = None + + history.append({ + "iteration": iteration, + "description": current_description, + "train_passed": train_summary["passed"], + "train_failed": train_summary["failed"], + "train_total": train_summary["total"], + "train_results": train_results["results"], + "test_passed": test_summary["passed"] if test_summary else None, + "test_failed": test_summary["failed"] if test_summary else None, + "test_total": test_summary["total"] if test_summary else None, + "test_results": test_results["results"] if test_results else None, + # For backward compat with report generator + "passed": train_summary["passed"], + "failed": train_summary["failed"], + "total": train_summary["total"], + "results": train_results["results"], + }) + + # Write live report if path provided + if live_report_path: + partial_output = { + "original_description": original_description, + "best_description": current_description, + "best_score": "in progress", + "iterations_run": len(history), + "holdout": holdout, + "train_size": len(train_set), + "test_size": len(test_set), + "history": history, + } + live_report_path.write_text(generate_html(partial_output, auto_refresh=True, skill_name=name)) + + if verbose: + def print_eval_stats(label, results, elapsed): + pos = [r for r in results if r["should_trigger"]] + neg = [r for r in results if not r["should_trigger"]] + tp = sum(r["triggers"] for r in pos) + pos_runs = sum(r["runs"] for r in pos) + fn = pos_runs - tp + fp = sum(r["triggers"] for r in neg) + neg_runs = sum(r["runs"] for r in neg) + tn = neg_runs - fp + total = tp + tn + fp + fn + precision = tp / (tp + fp) if (tp + fp) > 0 else 1.0 + recall = tp / (tp + fn) if (tp + fn) > 0 else 1.0 + accuracy = (tp + tn) / total if total > 0 else 0.0 + print(f"{label}: {tp+tn}/{total} correct, precision={precision:.0%} recall={recall:.0%} accuracy={accuracy:.0%} ({elapsed:.1f}s)", file=sys.stderr) + for r in results: + status = "PASS" if r["pass"] else "FAIL" + rate_str = f"{r['triggers']}/{r['runs']}" + print(f" [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:60]}", file=sys.stderr) + + print_eval_stats("Train", train_results["results"], eval_elapsed) + if test_summary: + print_eval_stats("Test ", test_results["results"], 0) + + if train_summary["failed"] == 0: + exit_reason = f"all_passed (iteration {iteration})" + if verbose: + print(f"\nAll train queries passed on iteration {iteration}!", file=sys.stderr) + break + + if iteration == max_iterations: + exit_reason = f"max_iterations ({max_iterations})" + if verbose: + print(f"\nMax iterations reached ({max_iterations}).", file=sys.stderr) + break + + # Improve the description based on train results + if verbose: + print(f"\nImproving description...", file=sys.stderr) + + t0 = time.time() + # Strip test scores from history so improvement model can't see them + blinded_history = [ + {k: v for k, v in h.items() if not k.startswith("test_")} + for h in history + ] + new_description = improve_description( + skill_name=name, + skill_content=content, + current_description=current_description, + eval_results=train_results, + history=blinded_history, + model=model, + log_dir=log_dir, + iteration=iteration, + ) + improve_elapsed = time.time() - t0 + + if verbose: + print(f"Proposed ({improve_elapsed:.1f}s): {new_description}", file=sys.stderr) + + current_description = new_description + + # Find the best iteration by TEST score (or train if no test set) + if test_set: + best = max(history, key=lambda h: h["test_passed"] or 0) + best_score = f"{best['test_passed']}/{best['test_total']}" + else: + best = max(history, key=lambda h: h["train_passed"]) + best_score = f"{best['train_passed']}/{best['train_total']}" + + if verbose: + print(f"\nExit reason: {exit_reason}", file=sys.stderr) + print(f"Best score: {best_score} (iteration {best['iteration']})", file=sys.stderr) + + return { + "exit_reason": exit_reason, + "original_description": original_description, + "best_description": best["description"], + "best_score": best_score, + "best_train_score": f"{best['train_passed']}/{best['train_total']}", + "best_test_score": f"{best['test_passed']}/{best['test_total']}" if test_set else None, + "final_description": current_description, + "iterations_run": len(history), + "holdout": holdout, + "train_size": len(train_set), + "test_size": len(test_set), + "history": history, + } + + +def main(): + parser = argparse.ArgumentParser(description="Run eval + improve loop") + parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file") + parser.add_argument("--skill-path", required=True, help="Path to skill directory") + parser.add_argument("--description", default=None, help="Override starting description") + parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers") + parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds") + parser.add_argument("--max-iterations", type=int, default=5, help="Max improvement iterations") + parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query") + parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold") + parser.add_argument("--holdout", type=float, default=0.4, help="Fraction of eval set to hold out for testing (0 to disable)") + parser.add_argument("--model", required=True, help="Model for improvement") + parser.add_argument("--verbose", action="store_true", help="Print progress to stderr") + parser.add_argument("--report", default="auto", help="Generate HTML report at this path (default: 'auto' for temp file, 'none' to disable)") + parser.add_argument("--results-dir", default=None, help="Save all outputs (results.json, report.html, log.txt) to a timestamped subdirectory here") + args = parser.parse_args() + + eval_set = json.loads(Path(args.eval_set).read_text()) + skill_path = Path(args.skill_path) + + if not (skill_path / "SKILL.md").exists(): + print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr) + sys.exit(1) + + name, _, _ = parse_skill_md(skill_path) + + # Set up live report path + if args.report != "none": + if args.report == "auto": + timestamp = time.strftime("%Y%m%d_%H%M%S") + live_report_path = Path(tempfile.gettempdir()) / f"skill_description_report_{skill_path.name}_{timestamp}.html" + else: + live_report_path = Path(args.report) + # Open the report immediately so the user can watch + live_report_path.write_text("

Starting optimization loop...

") + webbrowser.open(str(live_report_path)) + else: + live_report_path = None + + # Determine output directory (create before run_loop so logs can be written) + if args.results_dir: + timestamp = time.strftime("%Y-%m-%d_%H%M%S") + results_dir = Path(args.results_dir) / timestamp + results_dir.mkdir(parents=True, exist_ok=True) + else: + results_dir = None + + log_dir = results_dir / "logs" if results_dir else None + + output = run_loop( + eval_set=eval_set, + skill_path=skill_path, + description_override=args.description, + num_workers=args.num_workers, + timeout=args.timeout, + max_iterations=args.max_iterations, + runs_per_query=args.runs_per_query, + trigger_threshold=args.trigger_threshold, + holdout=args.holdout, + model=args.model, + verbose=args.verbose, + live_report_path=live_report_path, + log_dir=log_dir, + ) + + # Save JSON output + json_output = json.dumps(output, indent=2) + print(json_output) + if results_dir: + (results_dir / "results.json").write_text(json_output) + + # Write final HTML report (without auto-refresh) + if live_report_path: + live_report_path.write_text(generate_html(output, auto_refresh=False, skill_name=name)) + print(f"\nReport: {live_report_path}", file=sys.stderr) + + if results_dir and live_report_path: + (results_dir / "report.html").write_text(generate_html(output, auto_refresh=False, skill_name=name)) + + if results_dir: + print(f"Results saved to: {results_dir}", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/skill-creator/scripts/security_scan.py b/skill-creator/scripts/security_scan.py index b91f6180..75fa3f64 100755 --- a/skill-creator/scripts/security_scan.py +++ b/skill-creator/scripts/security_scan.py @@ -89,7 +89,7 @@ def get_pattern_rules() -> List[Dict]: "id": "insecure_http", "category": "urls", "name": "Insecure HTTP URLs", - "patterns": [r'http://(?!localhost|127\.0\.0\.1|0\.0\.0\.0|example\.com)'], + "patterns": [r'http' r'://(?!localhost|127\.0\.0\.1|0\.0\.0\.0|example\.com)'], "severity": "MEDIUM", "message": "HTTP (insecure) URL detected", "recommendation": "Use HTTPS for external resources", diff --git a/skill-creator/scripts/utils.py b/skill-creator/scripts/utils.py new file mode 100644 index 00000000..51b6a07d --- /dev/null +++ b/skill-creator/scripts/utils.py @@ -0,0 +1,47 @@ +"""Shared utilities for skill-creator scripts.""" + +from pathlib import Path + + + +def parse_skill_md(skill_path: Path) -> tuple[str, str, str]: + """Parse a SKILL.md file, returning (name, description, full_content).""" + content = (skill_path / "SKILL.md").read_text() + lines = content.split("\n") + + if lines[0].strip() != "---": + raise ValueError("SKILL.md missing frontmatter (no opening ---)") + + end_idx = None + for i, line in enumerate(lines[1:], start=1): + if line.strip() == "---": + end_idx = i + break + + if end_idx is None: + raise ValueError("SKILL.md missing frontmatter (no closing ---)") + + name = "" + description = "" + frontmatter_lines = lines[1:end_idx] + i = 0 + while i < len(frontmatter_lines): + line = frontmatter_lines[i] + if line.startswith("name:"): + name = line[len("name:"):].strip().strip('"').strip("'") + elif line.startswith("description:"): + value = line[len("description:"):].strip() + # Handle YAML multiline indicators (>, |, >-, |-) + if value in (">", "|", ">-", "|-"): + continuation_lines: list[str] = [] + i += 1 + while i < len(frontmatter_lines) and (frontmatter_lines[i].startswith(" ") or frontmatter_lines[i].startswith("\t")): + continuation_lines.append(frontmatter_lines[i].strip()) + i += 1 + description = " ".join(continuation_lines) + continue + else: + description = value.strip('"').strip("'") + i += 1 + + return name, description, content diff --git a/skill-creator/workflows/wrapper-skill/architecture_contract.md b/skill-creator/workflows/wrapper-skill/architecture_contract.md new file mode 100644 index 00000000..c10eeb95 --- /dev/null +++ b/skill-creator/workflows/wrapper-skill/architecture_contract.md @@ -0,0 +1,94 @@ +# Wrapper Skill Architecture Contract + +The non-negotiable principles that every wrapper skill generated by this workflow must follow. Think of this as the bill of rights for users of wrapper skills — if any of these principles is violated, the wrapper loses its core value and will eventually produce the kind of confusion and git conflicts that "helpful fork" skills always do. + +Read this file once before filling in `patterns.md` and the generated `SKILL.md`. The templates in `patterns.md` are the enforcement mechanism, but templates only work if you understand why they're shaped the way they are. + +## Contract + +### 1. Never vendor upstream files + +The wrapper skill directory contains **zero** copies, excerpts, or edits of the upstream tool's own files. No `SKILL.md` fragments copied from the upstream package. No API docs pulled from upstream's README. No patched versions of upstream scripts. + +**Why**: Vendoring creates two versions of the same truth — ours and upstream's. Within weeks they drift. Users who update the upstream tool get a broken install because our vendored copy disagrees with what upstream now ships. Users who look at our repo get stale information that contradicts upstream's documentation. Both failure modes destroy trust in the wrapper. + +**Instead**: The wrapper describes *how to interact with* the upstream tool (download from URL, install via package manager, read credentials from path). The actual upstream content is fetched at install time and never committed. + +**Concrete test**: `git ls-files /` should never show a file whose content is a copy of an upstream file. A reviewer opening any file in the wrapper should be able to explain what it's *for* without reference to any upstream file. + +### 2. Repair at runtime, not at ship time + +When the upstream tool has a bug that the wrapper knows how to fix, the fix lives as **instructions** in `references/known_issues.md` — a text document the agent reads and executes with user consent — not as pre-patched files shipped in the wrapper. + +**Why**: Pre-patched files become stale the instant upstream releases a new version. The user pulls the new upstream, the old patched files no longer match, merge conflicts appear, and the user hates the wrapper. Runtime instructions re-apply cleanly against whatever upstream version the user happens to have, because the instructions match on symptoms (file content, error messages) rather than file hashes. + +**Concrete test**: Generate the wrapper, apply a fix via the wrapper, then simulate the user running the upstream's own installer again. The wrapper's fix should be safely re-appliable with zero conflict. If any step fails or requires manual reconciliation, the contract is violated. + +### 3. Every modification to upstream files requires explicit user consent + +The wrapper's diagnose script is **read-only**. The install script is **additive** (it places new files from the upstream archive into agent directories). The only code that modifies existing upstream files lives in `references/known_issues.md` as documentation, and is only executed by the agent after an explicit `AskUserQuestion` round where the user chooses a specific repair strategy. + +**Why**: Users keep workarounds they aren't told about. When upstream ships a fix and the user's local install silently diverges, the user spends days debugging a problem that ended up being "the wrapper helpfully patched this last month". Explicit consent means the user knows what's been changed and can reproduce or undo it. + +**Concrete test**: Search the generated scripts for any file mutation that doesn't have a prior `AskUserQuestion` or CLI prompt. `install_.sh` may write new files into empty agent directories (allowed). It must not modify files that already existed before the install. `diagnose.sh` must not write anything anywhere. + +### 4. Every repair command is idempotent, reversible, and alias-safe + +- **Idempotent**: Running the command a second time is a harmless no-op. No duplicate backups, no doubled-up edits, no "already applied" errors. +- **Reversible**: Before modifying any file, the command copies the original to `/tmp/-backups//`. The backup location is printed to stdout so the user knows where to look. A rollback command using that backup appears in the same document. +- **Alias-safe**: Every `cp`, `mv`, `rm` is prefixed with `command` to bypass user-defined shell aliases like `alias mv='mv -i'` that would make the script hang on an interactive prompt. + +**Why**: Wrappers run in other people's shells. Other people have other people's aliases. Other people come back a week later and rerun the wrapper without remembering what it already did. All three of these fail modes are common and all three are prevented by following this principle strictly. + +**Concrete test**: Run the repair command, then run it again immediately. The second run should print "already applied" or similar and exit cleanly with zero side effects. Inspect the rollback procedure — can you recover the original state with one command? Search the command for a bare `mv` or `cp` — none should appear. + +### 5. The skill teaches agents, not humans + +Wrapper skills are read by Claude (or similar agents) at the time a user asks for help with the tool. The audience of every file in the wrapper is "a future LLM agent that has never seen this tool before, talking to a user who has never installed it before." + +This implies: + +- **Instructions are imperative and agent-friendly**: "When the user reports X, run `diagnose.sh` and parse the output." Not "You should check the diagnostic." +- **Decision points are explicit**: If the agent has to choose between two repair strategies, the wrapper tells it to use `AskUserQuestion` with specific option labels, not "use your judgment." +- **Examples are concrete**: The `known_issues.md` file quotes literal error messages, not generic descriptions. When the agent sees the literal message in user output, it can match it directly. +- **Trigger signals are verbose**: The `description` field lists concrete symptoms, tool names, file paths, and error fragments. Claude's skill-triggering selector is pattern matching; give it the patterns. + +**Why**: Human documentation is written to be scannable and engaging. Agent documentation is written to be deterministic and unambiguous. The two styles optimize for different things. A wrapper skill written for humans leaves the agent guessing; a wrapper written for agents lets humans skim the headers and drill into the code when curious. + +### 6. Independent evolution from upstream + +The wrapper skill's version, release cadence, bug list, and documentation are **decoupled** from the upstream tool's. If upstream ships v1.2.0, the wrapper does not automatically bump to match. If the wrapper fixes a packaging bug on its side, the upstream has no reason to care. + +**Why**: Coupling versions creates a coordination problem between two projects that have no contractual relationship. Upstream doesn't know the wrapper exists; the wrapper has no authority to tell upstream to release. Trying to keep the two in lockstep wastes effort and creates fragility. + +**Concrete test**: The wrapper's `marketplace.json` version, `SKILL.md` description, and `CHANGELOG.md` entries never mention a specific upstream version number. The `install_.sh` script may default to a specific upstream version but must accept an override flag for any other version. When upstream ships a new release, bumping the wrapper's default is a one-line change that does not require changing any other file. + +### 7. The user's private preferences stay on the user's machine + +If the wrapper supports per-user configuration (priority lists, skip lists, rendering preferences), the configuration file lives in the user's XDG config directory (typically `~/.config//`). The wrapper repo contains only a **template** file with illustrative placeholder values, never real values. + +**Why**: Committing real user preferences into a shared wrapper is both a privacy leak and a reproducibility problem. Other users copy the preferences unintentionally, lose their own, and get surprised when the wrapper behaves differently for them than for its author. + +**Concrete test**: `grep -r` the wrapper directory for any string that looks like a user's real preference, knowledge base name, email, username, or private identifier. None should exist. + +## How to use this file + +When filling in `patterns.md` templates, every section of every template must be justifiable by one of the seven principles above. If a template section exists without a clear contract-justification, it is decoration and should be deleted. + +When distilling Step 2's session mining into the generated files, check each generated section against this list. If any section violates any principle, fix it before committing. The contract is not aspirational — it is enforced by the generated code's shape. + +## Reference implementation + +Every principle above is concretely enforced in `ima-copilot/`. Cross-reference: + +| Principle | Where it's visible in ima-copilot | +|---|---| +| 1. Never vendor | Look at `ima-copilot/`. No file in it is a copy of anything in the upstream ima-skills-1.1.2.zip. | +| 2. Runtime repair | `references/known_issues.md` has repair commands, not patched files. Re-running `install_ima_skill.sh` after a repair restores the broken state, and re-running the repair fixes it again. | +| 3. Explicit consent | `scripts/diagnose.sh` is read-only. Repairs live in `known_issues.md` behind `AskUserQuestion` routing instructions in `SKILL.md`. | +| 4. Idempotent / reversible / alias-safe | Every command in `known_issues.md` uses `command cp` / `command mv`, copies to `/tmp/ima-copilot-backups//` first, and is safe to rerun. | +| 5. Teaches agents | `SKILL.md` reads as imperative routing. The description lists literal error strings. Decision points use `AskUserQuestion`. | +| 6. Independent evolution | `ima-copilot` version is `1.0.0`, completely independent of upstream `ima-skills-1.1.2.zip`. The install script's `IMA_VERSION` default is `1.1.2` but overridable. | +| 7. Private preferences | `config-template/copilot.json.example` has placeholder KB names like `your-curated-kb-name`. Nothing in the repo references any real knowledge base. | + +If in doubt about how to interpret a principle for your specific wrapper, open the corresponding part of `ima-copilot` and imitate it. diff --git a/skill-creator/workflows/wrapper-skill/patterns.md b/skill-creator/workflows/wrapper-skill/patterns.md new file mode 100644 index 00000000..49b8e9b1 --- /dev/null +++ b/skill-creator/workflows/wrapper-skill/patterns.md @@ -0,0 +1,740 @@ +# Wrapper Skill Code Patterns + +Copy-pasteable templates for every file a generated wrapper skill ships. Each template has placeholders that need to be filled from the Step 2 mining output. Every template is accompanied by an explanation of why it's shaped that way, and a reference to the concrete version in `ima-copilot/` for comparison. + +Rule of thumb: when adapting these templates, remove anything the original session didn't actually need. Don't leave placeholder sections that apply to other wrappers but not yours. + +## File: SKILL.md + +```markdown +--- +name: +description: +--- + +# + + + +## Overview + +<2-4 sentences describing the upstream tool, the specific friction this wrapper removes, and the scope of its coverage.> + +## Architectural principles (do not violate) + +This skill is a **wrapper layer** around . The wrapper contract is non-negotiable: + +- **Never vendor upstream files.** This skill directory does not contain any copy, fork, or excerpt of 's own content. +- **Repairs happen at runtime, not at ship time.** If an upstream bug needs patching, this skill carries the *instructions* for how to patch, not the patched files. Running a repair is idempotent. +- **Always ask before touching upstream files.** Modifying installed files requires explicit user consent via AskUserQuestion. +- **Teach rather than hide.** Every fix shows the user exactly what changed and where the backup was saved. + +## What this skill does + +| Capability | Entry point | Detail | +|---|---|---| +| Install upstream | `scripts/install_.sh` | See `references/installation_flow.md` | +| Configure credentials | Inline workflow below | See `references/credentials_setup.md` | +| Diagnose and fix known issues | `scripts/diagnose.sh` + workflow below | See `references/known_issues.md` | +| | `scripts/` | See `references/.md` | + +## Routing + + + +When in doubt, default to Capability 3 (diagnose). It is the only read-only entry point and it surfaces exactly which capabilities are currently blocked and in what order, which is almost always the correct first step when a new user arrives with a vague question. Put a one-line "when in doubt → diagnose" note at the bottom of the routing table so the agent has an unambiguous default. + +## Capability 1: Install upstream + +<2-3 paragraph explanation of what the installer does, with a code block showing the one-line invocation. Reference `references/installation_flow.md` for details.> + +## Capability 2: Configure credentials + + + +## Capability 3: Diagnose and fix known issues + + + +## Capability 4: + + + +## What this skill refuses to do + +- Vendor, fork, or mirror upstream files into this directory +- Pin an upstream version in SKILL.md (installer uses overridable defaults, SKILL.md stays version-agnostic) +- Silently patch upstream files — every modification path requires explicit consent +- Hardcode user-specific values + +## File layout + +``` +/ +├── SKILL.md # This file +├── scripts/ +│ ├── install_.sh # Download → stage → distribute +│ └── diagnose.sh # Read-only health report +├── references/ +│ ├── installation_flow.md +│ ├── credentials_setup.md +│ ├── known_issues.md # Issue registry — source of truth +│ └── best_practices.md +└── config-template/ # Optional; omit if no per-user config + └── .json.example +``` +``` + +**Concrete version**: `ima-copilot/SKILL.md`. + +**Why the description is so long**: Claude's skill selector is pattern matching on the description field. A 3-sentence description gets triggered 30% of the time it should; an 8-sentence description with literal error strings gets triggered 95% of the time. The cost of false positives (skill fires when it isn't needed) is much lower than the cost of false negatives (user hits an error this skill could have fixed but the skill didn't fire). Err on the verbose side. Note: there is a hard 1024-character cap on the description field, enforced by `skill-creator/scripts/quick_validate.py`. Run validation before commit to catch overlong descriptions early. + +**What to pack into the description** (checklist): + +- **Literal error strings from the session** — if the upstream tool emits `Skipped loading skill(s) due to invalid SKILL.md`, that exact phrase goes in the description so a future user hitting the same error triggers this skill automatically. Paraphrases do not match, literal strings do. +- **Tool name in every language the session used**. If the user spoke to you in Chinese, put the Chinese name (`腾讯 IMA`, `知识库搜索`, `笔记搜索`) alongside the English (`Tencent IMA`, `knowledge base search`). Claude's selector is language-agnostic but a monolingual description only triggers on monolingual queries. +- **A self-disambiguation clause** naming the upstream package. The wrapper and the upstream often fight for the same triggers — a user asking "install ima-skill" could route to either this wrapper or to the upstream skill package if both are installed. Put a clause like "This is a wrapper layer around — it installs and orchestrates rather than replacing it" so the selector has a distinguishing signal to prefer the wrapper when the user's intent is installation, and defer to the upstream when the user's intent is direct operation. +- **The symptoms that triggered the original session**. If the user came to you because something was broken, put that symptom in the description so a future user with the same symptom gets pushed here. + +## File: scripts/install_.sh + +```bash +#!/usr/bin/env bash +# +# install_.sh — Install upstream to +# +# Flow: +# 1. Download the official from +# 2. Stage it in a temp directory +# 3. Detect which of the target agents are installed locally +# 4. Delegate to `npx skills add ` (vercel-labs/skills) in its +# default symlink mode so that the agents share one canonical copy +# 5. Clean up the staging dir on exit +# +# Re-run safely — every step is idempotent. + +set -euo pipefail + +_VERSION="${_VERSION:-}" +BASE_URL="" +STAGING_ROOT="/tmp/-staging" +STAGING_DIR="${STAGING_ROOT}/$(date +%s)-$$" + +cleanup() { + if [ -n "${STAGING_DIR:-}" ] && [ -d "$STAGING_DIR" ]; then + rm -rf "$STAGING_DIR" + fi +} +trap cleanup EXIT + +usage() { + cat <<'EOF' +Usage: install_.sh [--version ] + + +EOF +} + +while [ $# -gt 0 ]; do + case "$1" in + --version) _VERSION="$2"; shift 2 ;; + --version=*) _VERSION="${1#*=}"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown argument: $1" >&2; usage >&2; exit 1 ;; + esac +done + +# Require basic tools — fail fast with a specific missing-tool message. +for tool in curl unzip npx; do + if ! command -v "$tool" >/dev/null 2>&1; then + echo "✗ Required tool not found on PATH: $tool" >&2 + exit 1 + fi +done + +# Require Node.js >= 18 — `npx skills add` from vercel-labs/skills needs a +# modern Node runtime. The error on old Node is otherwise opaque and blames +# the wrong layer (npm cache, package resolution) rather than the version. +if command -v node >/dev/null 2>&1; then + node_major=$(node --version 2>/dev/null | sed -E 's/^v([0-9]+).*/\1/') + if [ -n "$node_major" ] && [ "$node_major" -lt 18 ] 2>/dev/null; then + echo "✗ Node.js 18+ required for 'npx skills add' — found: $(node --version)" >&2 + echo " Upgrade via your package manager (brew/apt/nvm) and retry." >&2 + exit 1 + fi +fi + +echo "▶ Staging upstream v${_VERSION}" +mkdir -p "$STAGING_DIR" + +ZIP_URL="${BASE_URL}/_VERSION}.zip>" +ZIP_PATH="${STAGING_DIR}/upstream.zip" + +# Download with `--fail` so HTTP errors surface as non-zero exit codes, +# and capture the HTTP code for the error-branch message. +echo " Downloading ${ZIP_URL}" +http_code=$(curl -sS -L --fail -o "$ZIP_PATH" -w "%{http_code}" "$ZIP_URL" || echo "000") +if [ "$http_code" != "200" ]; then + echo "" >&2 + echo "✗ Download failed (HTTP ${http_code})" >&2 + echo "" >&2 + echo "If has released a newer version, pass it explicitly:" >&2 + echo " _VERSION=x.y.z bash $0" >&2 + echo "" >&2 + echo "or find the latest version at " >&2 + exit 1 +fi + +# Size sanity check — a redirect to an HTML error page or a yanked package +# often returns a "success" status with a tiny non-archive body. Reject +# anything below an absolute floor to fail fast before extraction corrupts +# the staging dir. +actual_size=$(wc -c < "$ZIP_PATH" | tr -d ' ') +echo " Downloaded ${actual_size} bytes" +if [ "$actual_size" -lt 1000 ]; then + echo "✗ Downloaded file is suspiciously small — aborting before extraction" >&2 + exit 1 +fi + +echo " Extracting…" +unzip -q -o "$ZIP_PATH" -d "$STAGING_DIR" + +# Locate the root directory inside the extracted archive. Prefer well-known +# layout first, fall back to a recursive scan that picks the shallowest +# SKILL.md — which is the root by construction of every legal SKILL.md tree. +SKILL_SRC="" +if [ -f "$STAGING_DIR//SKILL.md" ]; then + SKILL_SRC="$STAGING_DIR/" +else + shallowest_depth=999 + while IFS= read -r candidate; do + rel="${candidate#$STAGING_DIR/}" + depth=$(awk -F/ '{print NF}' <<< "$rel") + if [ "$depth" -lt "$shallowest_depth" ]; then + shallowest_depth="$depth" + SKILL_SRC=$(dirname "$candidate") + fi + done < <(find "$STAGING_DIR" -maxdepth 4 -type f -name SKILL.md -print) +fi + +if [ -z "$SKILL_SRC" ]; then + echo "✗ Could not locate SKILL.md in extracted archive" >&2 + exit 1 +fi + +# Detect which target agents are installed +AGENTS=() +[ -d "$HOME/.claude" ] && AGENTS+=("claude-code") +[ -d "$HOME/.agents" ] && AGENTS+=("codex") +if [ -d "$HOME/.openclaw" ] || command -v openclaw >/dev/null 2>&1; then + AGENTS+=("openclaw") +fi + +if [ ${#AGENTS[@]} -eq 0 ]; then + # Zero-agents-detected fallback. Three options considered during the + # ima-copilot session, the selected one documented here: + # (a) abort with a "nothing to install into" error — too strict for a + # user who just installed claude-code and forgot to restart their + # shell between the install and our skill's install. + # (b) silently install nothing — most surprising and hardest to debug. + # (c) print a warning naming the paths we looked at and default to + # claude-code, which is the most common case. ← chosen + echo "" >&2 + echo "⚠ No supported agent detected." >&2 + echo " Looked for: ~/.claude (Claude Code), ~/.agents (Codex), openclaw on PATH." >&2 + echo " Defaulting to claude-code as the most common target." >&2 + echo "" >&2 + AGENTS=("claude-code") +fi + +AGENT_FLAGS=() +for a in "${AGENTS[@]}"; do + AGENT_FLAGS+=("-a" "$a") +done + +# Distribute via vercel-labs/skills in default symlink mode — repairs applied +# to any agent propagate to all of them. +if ! npx -y skills add "$SKILL_SRC" -g -y "${AGENT_FLAGS[@]}"; then + echo "✗ npx skills add failed" >&2 + exit 1 +fi + +echo "" +echo "✓ Upstream v${_VERSION} installed" +``` + +**Concrete version**: `ima-copilot/scripts/install_ima_skill.sh`. + +**Lessons baked into this template**: + +- **Prerequisite check discipline**: every external tool the script depends on is verified up front. `curl`, `unzip`, `npx` are checked by the `command -v` loop. Node.js is checked separately with a *numeric major-version parse* because `command -v node` only verifies presence and says nothing about version — and `npx skills add` from vercel-labs/skills is known to fail opaquely on Node 16. If any prerequisite is missing or too old, the script fails fast with a specific actionable message, not after half the download. +- **Download integrity defense in depth**: `curl --fail` catches HTTP errors as non-zero exits; `-w "%{http_code}"` captures the code for a specific error message; an explicit `!= "200"` branch gives the user an override hint (`_VERSION=x.y.z bash $0`); and a `wc -c` size check rejects absurdly small downloads *before extraction*. The size check is the one that catches the worst real-world failure mode: an upstream CDN redirects a yanked-version URL to an HTML error page that returns 200, and a naive script then passes the HTML to `unzip` and produces confusing downstream errors. Reject anything below an absolute floor (1 KB works for most archives) and the cause is obvious. +- **Root SKILL.md detection prefers a known layout first, then falls back to the shallowest match**. This is a real bug discovered during ima-copilot dogfood: a naive `find` returned `ima-skill/notes/SKILL.md` as "first match" and the installer then tried to install from the `notes/` subdirectory, which failed because that file has no frontmatter. The fix is to bias the search toward known layouts. +- **Agent-detection philosophy: "only install where the user has opted in"**. The `AGENTS=()` block walks a fixed set of home directories and only installs to the ones that already exist. A missing agent path is treated as "the user did not opt into this agent" — not as a precondition failure. This avoids silently installing into directories that aren't part of the user's setup, which matters when the same machine has been used to experiment with multiple agent products. +- **Zero-agents fallback**: when no target agent is detected, the script prints an explicit "looked for: …" list before defaulting to claude-code. This is the single most-debated branch in the ima-copilot session because all three options (abort / silent-skip / default-to-claude-code) are defensible. The documented choice is to default-to-claude-code because that is the most common case when detection legitimately fails (e.g., user just installed the agent and hasn't restarted their shell). Abort would be hostile; silent-skip would be mystifying. +- **`-g -y` no `--copy`**: vercel's default symlink mode is strictly better for wrapper skills because a repair applied to any agent's install propagates via symlink to all agents. If your upstream tool has a different natural distribution story, reconsider — but the symlink default is correct in the vast majority of cases. +- **`trap cleanup EXIT`**: the staging directory is always removed, even if the script fails midway. No leftover clutter in `/tmp/`. + +## File: scripts/diagnose.sh + +```bash +#!/usr/bin/env bash +# +# diagnose.sh — Read-only health check for upstream installs. +# +# Prints one status line per check, then a summary. +# +# Exit codes: +# 0 — all checks passed +# 1 — one or more issues need user action +# 2 — diagnostic itself failed (network error, missing tooling) +# +# This script is strictly read-only. + +set -uo pipefail + +PASS=0; WARN=0; FAIL=0 + +status_ok() { echo "✅ $1"; PASS=$((PASS + 1)); } +status_warn() { echo "⚠️ $1"; WARN=$((WARN + 1)); } +status_fail() { echo "❌ $1"; FAIL=$((FAIL + 1)); } + +echo "=== diagnostic report ===" +echo + +# Agent target path resolution +find_install() { + local agent="$1"; shift + local path + for path in "$@"; do + if [ -f "$path/SKILL.md" ]; then + echo "$path" + return 0 + fi + done + return 1 +} + +# Resolve symlinks to detect shared canonical installs +canonical() { + python3 -c "import os,sys; print(os.path.realpath(sys.argv[1]))" "$1" 2>/dev/null || echo "$1" +} + +# Per-agent install presence +echo "--- Upstream installs ---" +CLAUDE_PATH=""; CODEX_PATH=""; OPENCLAW_PATH="" + +if CLAUDE_PATH=$(find_install claude-code "$HOME/.claude/skills/"); then + status_ok " installed (claude-code) at $CLAUDE_PATH" +else + status_warn " NOT installed (claude-code) — run install_.sh" +fi + +if CODEX_PATH=$(find_install codex "$HOME/.agents/skills/" "$HOME/.codex/skills/"); then + status_ok " installed (codex) at $CODEX_PATH" +else + status_warn " NOT installed (codex) — run install_.sh" +fi + +if OPENCLAW_PATH=$(find_install openclaw \ + "$HOME/.openclaw/skills/" \ + "$HOME/.config/openclaw/skills/" \ + "$HOME/.local/share/openclaw/skills/"); then + status_ok " installed (openclaw) at $OPENCLAW_PATH" +else + status_warn " NOT installed (openclaw) — run install_.sh" +fi + +# Detect shared canonical via symlink +CLAUDE_REAL=$(canonical "${CLAUDE_PATH:-}") +CODEX_REAL=$(canonical "${CODEX_PATH:-}") +OPENCLAW_REAL=$(canonical "${OPENCLAW_PATH:-}") +if [ -n "$CLAUDE_REAL" ] && [ -n "$CODEX_REAL" ] && [ "$CLAUDE_REAL" = "$CODEX_REAL" ]; then + echo "ℹ️ claude-code and codex share the same install via symlink" +fi + +# ... similar for other agent pairs ... + +echo + +# Credentials +echo "--- Credentials ---" + +echo + +# Known issues — one scan function per issue, called for each unique canonical dir +echo "--- Known issues ---" + +SCANNED_REALS="" +scan_agent() { + local agent="$1" + local path="$2" + local real="$3" + [ -z "$path" ] && return + case " $SCANNED_REALS " in + *" $real "*) return ;; # already scanned via another agent + esac + SCANNED_REALS="$SCANNED_REALS $real" + scan_issue_001 "$agent" "$path" + # scan_issue_002, etc. +} + +scan_issue_001() { + local agent="$1" + local base="$2" + +} + +scan_agent "claude-code" "$CLAUDE_PATH" "$CLAUDE_REAL" +scan_agent "codex" "$CODEX_PATH" "$CODEX_REAL" +scan_agent "openclaw" "$OPENCLAW_PATH" "$OPENCLAW_REAL" + +echo + +# Summary +echo "--- Summary ---" +echo " ✅ ${PASS} pass ⚠️ ${WARN} warn ❌ ${FAIL} fail" +echo + +if [ "$FAIL" -gt 0 ] || [ "$WARN" -gt 0 ]; then + echo "Next step: open references/known_issues.md and walk the agent through" + echo "the warnings above. Each issue ID maps to a concrete repair procedure." + exit 1 +fi + +exit 0 +``` + +**Concrete version**: `ima-copilot/scripts/diagnose.sh`. + +**Lessons baked into this template**: + +- **`canonical()` via Python realpath**: detecting symlink-shared installs is essential to avoid reporting the same issue multiple times. Real discovery from ima-copilot dogfood. +- **`SCANNED_REALS` dedup**: only scan each underlying canonical directory once per issue, even if multiple agents point at it. +- **`find_install` takes a *variadic list* of candidate paths**: for each target agent, pass a short ordered list of known install paths and return the first that exists, rather than hardcoding one path. This matters most for agents whose home-directory layout has not stabilized — e.g., OpenClaw in ima-copilot was probed against `~/.openclaw/skills/...`, `~/.config/openclaw/skills/...`, and `~/.local/share/openclaw/skills/...` because the standard wasn't settled. For agents with a firmly-established layout (Claude Code's `~/.claude/skills/`), a one-entry list is fine. Designing the helper as variadic from day one avoids a painful refactor when a second candidate path becomes necessary. +- **One `scan_issue_NNN` function per known issue**: keeps the main loop clean and lets you add new issues by adding one function and one line in `scan_agent`. +- **`set -uo pipefail` (not `-e`)**: the diagnostic itself should not exit on the first command failure — it should continue and report all issues. `-u` and `-o pipefail` still catch real bugs in the script. + +### Detection function return-code contract + +The single hardest lesson from the ima-copilot session was that a detection function cannot be binary (broken / not-broken). It has to recognize **every post-repair state** the wrapper can produce, because users rerun the repair, restore partial backups, and switch between strategies mid-session. A function that only knows "original broken state" vs "Strategy A applied" will silently misreport anything else. + +The contract: **one return code per healthy state, one code per broken state, and one code for the conflicted dual-state that arises when two fix strategies have partially collided**. Spelled out: + +``` + 0 — OK: original untouched and already valid + (upstream shipped a fixed version, or the bug never applied to this + install, or the tool is now at a release where the issue is gone) + + 1 — BROKEN: original untouched and still needs repair + + 2 — NOT APPLICABLE: the target file doesn't exist at all, because upstream + changed the layout or the tool moved the affected file elsewhere. + This is legitimately different from BROKEN (the repair is not the + right fix because there is nothing to repair) and deserves its own + status line in the output. + + 3 — STRATEGY A APPLIED: the file is in the state that Strategy A's fix + produces (e.g., SKILL.md renamed to MODULE.md, root references + patched). This is healthy — do not report as BROKEN. + + 3+ — STRATEGY B, C, ... APPLIED: one additional healthy code per strategy + the known_issues.md file documents. Each strategy that touches a + different set of files gets its own code. + + 4 — DUAL-STATE CONFLICTED: files from more than one strategy exist + simultaneously (e.g., both SKILL.md and MODULE.md present, or a + backup restored on top of an in-progress fix). This state is the + single most important thing a detection function must recognize, + because reporting it as healthy hides a latent footgun and reporting + it as BROKEN triggers a fresh repair that will make the conflict + worse. The correct response is always CONFLICTED with a message + that names the conflicting files and points the user at the + rollback block. +``` + +Add a new healthy code whenever you add a new strategy to `known_issues.md`; add the dual-state code whenever more than one strategy can be applied to the same install. `ima-copilot/scripts/diagnose.sh` `check_submodule` function is the reference implementation — it returns 0/1/2/3/4 and the scan function's `case` statement handles each code distinctly. + +**Why this matters**: the dual-state code is the single place a careless author will skip. The symptom is always "my repair worked, but `diagnose.sh` says everything is clean, and now my install is subtly broken in a way neither strategy's repair command will fix". The fix — as simple as adding one `[ -f "$A" ] && [ -f "$B" ]` branch at the top of the check function — prevents that class of failure entirely. + +## File: references/known_issues.md + +```markdown +# Known Issues in Upstream + +This file is the **source of truth** for every upstream bug that can detect and help repair. + +## How the agent should use this file + +When `scripts/diagnose.sh` reports a `⚠️` line mentioning `ISSUE-`: + +1. Explain to the user in plain language what's broken and why it matters. +2. If the issue has more than one repair strategy, use **AskUserQuestion** to present the choices. +3. After the user picks, execute the exact commands under that strategy. Every command backs up originals to `/tmp/-backups//` first. +4. Re-run `diagnose.sh` and show the before/after. +5. Remind the user that upstream upgrades replace these files, so reruns after an upgrade are expected — and safe. + +## Issue registry + +### ISSUE- — + +**Status**: +**Symptom**: +**Root cause**: +**Impact**: + +**Why upstream probably hasn't fixed it**: , which tolerates the missing field; the bug is invisible from the upstream maintainer's primary testing platform."> + +**How to explain it to the user** (plain language): +> <1-2 sentence, jargon-free> + +**Repair strategies**: + +#### Strategy A — + +<1-paragraph explanation of what this strategy does and why it's labeled as it is> + +**What this strategy changes**: +- +- + +**Commands** (agent executes after user consent; replace `` with the specific agent path from `diagnose.sh`): + +```bash +# Use `command cp` / `command mv` / `command rm` / `command sed` to bypass +# any user-defined shell aliases. Interactive-mode aliases like `alias mv='mv -i'` +# will otherwise hang the script on an "overwrite?" prompt, and `alias rm='rm -i'` +# will stall cleanup steps. + +# 1. Back up originals. Every cp is `[ -f ... ] &&` guarded so that a rerun +# after partial application (where the source file has already been renamed +# or deleted by a previous fix run) doesn't print "file not found" errors. +# The guard is what makes the backup step idempotent across reruns. +BACKUP="/tmp/-backups/$(date +%Y%m%d-%H%M%S)" +mkdir -p "$BACKUP" +[ -f "/" ] && \ + command cp "/" "$BACKUP/" +# ... more guarded backup lines ... +echo "backup saved to: $BACKUP" + +# 2. Apply the fix (idempotent). All sed/rm/cp/mv calls go through `command`. +# sed -i.bak is the portable form that works on both BSD sed (macOS) and +# GNU sed (Linux) — a bare `sed -i` fails on BSD and `sed -i ''` fails on +# GNU. Always `command rm -f *.bak` after the sed to clean up the backup +# files sed creates, since leaving them around clutters the install dir. + +``` + +**Rollback**: + +```bash +command cp "$BACKUP/" "/" +# ... more rollback lines ... +command rm -f "/" +``` + +**Pros**: +**Cons**: + +#### Strategy B — + +... + +#### Strategy skip — Leave the file alone + +Every issue should document a "do nothing" branch explicitly, with the conditions under which it is actually valid. Users who only run the tool on a tolerant platform (e.g., Claude Code's lenient loader for ISSUE-001 in ima-copilot) may legitimately not want the repair. Naming the skip path as a first-class strategy makes it clear that "no action" was considered and the user is choosing it, rather than forgetting. When a strategy-skip branch is valid, the `AskUserQuestion` prompt in the agent's repair flow should list it as option (3) alongside Strategy A and Strategy B. + +Shape of the entry: + +``` +#### Strategy skip — Leave the file alone + +Valid when . Not recommended if . +``` + +## Adding new issues to this file + +When you discover a new upstream bug worth capturing: + +1. Assign the next sequential `ISSUE-` number. +2. Fill in the same template: symptom, root cause, impact, plain-language explanation, at least one strategy with idempotent + reversible commands. +3. Update `scripts/diagnose.sh` to detect it (still read-only) and print a line with the same issue ID. +4. **Do not** add the fix commands into any shipped script — keep them in this file so the agent reads and executes them at runtime under user consent. +``` + +**Concrete version**: `ima-copilot/references/known_issues.md`. + +**Why every command needs `command` prefix (including `sed` and `rm`, not just `cp`/`mv`)**: a user's shell may alias any of these to its `-i` variant. `alias mv='mv -i'` is common and was discovered during ima-copilot dogfood when it caused the repair to hang on a TTY prompt. `alias rm='rm -i'` is equally common — it affects the post-sed `.bak` cleanup and the rollback commands. `alias sed='sed -i'` is rarer but exists in some corporate dotfiles. The safe rule: **every cp, mv, rm, and sed in a repair block goes through `command` prefix, no exceptions**. + +**Why the `[ -f ... ] &&` guard wraps every backup cp**: without the guard, a rerun of the repair after a partial first run (where some source files have already been renamed or consumed by a previous run) prints "file not found" errors during the backup step. Those errors are cosmetically ugly, but more importantly they break the user's mental model of "the repair completed cleanly". The guard makes the backup step a no-op on files that no longer exist at the expected location, which is the correct behavior across reruns. + +**Why every fix backs up before modifying**: trust. A user running a wrapper skill for the first time wants to know "what did this skill change, and how do I undo it?". The backup path printed to stdout answers both questions without requiring the user to read the wrapper's source. + +**Backup directory naming convention**: use `/tmp/-backups/$(date +%Y%m%d-%H%M%S)`. The `%Y%m%d-%H%M%S` format sorts correctly when a user has multiple backup directories from different runs, and it is human-readable when the user is trying to find the most recent one. If reruns within the same second are possible (rapid test loops, CI), append `$$` (the shell's PID) for sub-second uniqueness: `/tmp/-backups/$(date +%Y%m%d-%H%M%S)-$$`. + +**Why `sed -i.bak` specifically (and why the `.bak` cleanup)**: `sed -i` has an incompatible argument between BSD sed (macOS default) and GNU sed (Linux default). BSD sed requires `-i ''` (empty string argument naming the backup suffix); GNU sed requires `-i` with no argument. The portable form is `sed -i.bak ...` — both sed variants accept it, and both leave behind a `.bak` backup copy that you then clean up with `command rm -f ".bak"`. Do not try to write a conditional that branches on OS — just use `.bak` unconditionally and clean up after. + +**Why idempotency is mandatory**: users re-run the wrapper after upstream upgrades, after system migrations, after their coworker broke something. The repair must tolerate being rerun in any state the user hands it. + +## File: config-template/.json.example + +```json +{ + "_comment_": "", + "": ["", ""], + + "_comment_": "", + "": [] +} +``` + +**Concrete version**: `ima-copilot/config-template/copilot.json.example`. + +**Why `_comment_*` pseudo-fields**: JSON doesn't support comments, and many wrapper skill config files are read by shells or Python without JSON5 support. The `_comment_*` prefix puts the documentation in the same file as the field it documents without breaking any JSON parser — the loader ignores unknown fields. + +**Why placeholder values, not real ones**: a committed template is a shared artifact. Real values leak information about the wrapper's author (what knowledge bases they read, what API endpoints they hit, which projects they work on). Placeholder values protect privacy and keep the template genuinely reusable. + +## Credential setup patterns (file content varies) + +For `references/credentials_setup.md`, the pattern is: + +1. **XDG-style paths** (`~/.config//{client_id, api_key}`) with mode `600`. +2. **Env var fallback** (`_OPENAPI_CLIENTID` / `_OPENAPI_APIKEY`) documented as "env vars win over files when both are set". +3. **Scoped liveness check** — see the next section. The liveness call must probe the lowest-privilege operation the skill actually performs, not the easiest API call to make. +4. **Liveness verification by response-body shape, not HTTP status**. Many third-party APIs return HTTP 200 with a JSON body containing an error code (`{"code": 401, "msg": "..."}` style). A liveness check that only looks at `curl --fail` or HTTP 2xx will pass for a credential that will fail the very first real operation. The correct shape check parses the response body and matches on a success-indicator field — for IMA-style APIs, that's `"code"\s*:\s*0`; for OAuth APIs, it's often `"access_token"` present; for REST APIs, it's the presence of an expected data field. Whatever the indicator is, the diagnose step should verify the *body shape*, not just the HTTP layer. +5. **Rotation procedure** showing `printf '%s' "" > ~/.config//client_id` followed by a re-run of the liveness check. + +Do not make the template literal — credential setup varies a lot by tool. Use the pattern as a checklist when writing `credentials_setup.md` for your specific wrapper, not as a copy-paste target. + +**Concrete version**: `ima-copilot/references/api_key_setup.md`. + +## Runtime-logic patterns shared across wrappers + +The install / diagnose / known_issues templates above are what make a wrapper *installable*. They are necessary but not sufficient. The three patterns in this section are what make a wrapper *correct at runtime* when it fans out operations across a third-party API, and they are frequently the most transferable insights a wrapper discovers — more transferable than any specific bug fix, because they are structural properties of the class of API the wrapper is talking to. + +Every one of these patterns was discovered during the ima-copilot session, lived inside `search_fanout.py`, and applies to a far wider class of tools than IMA. Consider whether your tool has the same failure mode before claiming "this only applies to the reference implementation". + +### Capability partitioning — enumerate vs operate + +**The problem**: many third-party APIs have a permission model where the set of entities the credential can *list* is strictly larger than the set it can *act on*. A wrapper that fans out an operation across "all listable entities" will hit authorization errors on a large fraction of them, and if those errors are mixed into the primary result, they will drown out the actual successes. + +**Examples by tool**: + +- **IMA**: `search_knowledge_base` enumerates every KB the user can read, including subscribed public KBs. `search_knowledge` on a subscribed KB returns `code: 220030, msg: 没有权限` because search permission requires ownership. A 12-KB account may have only 2 searchable KBs. +- **GitHub**: `GET /user/repos` lists every repo you can see, including private repos you're a collaborator on. Admin actions (`DELETE /repos/{owner}/{repo}`, `PATCH /repos/.../archive`) require repo-owner privilege and return 403 on collaborators-only entries. +- **Slack**: `conversations.list` returns every channel you're in. `chat.postMessage` can be rejected on channels where the bot lacks the `chat:write` scope or the channel has posting-locked. +- **Aliyun RAM**: RAM users can list resources in an account (`ECS DescribeInstances`) but can't operate on resources outside their policy scope — you see the inventory, you can't touch most of it. +- **Linear**: `workspaces` are viewable; mutation API (issue create/edit) is gated per-workspace on your role. + +**The pattern**: partition the fan-out result into four buckets, not one: + +``` +succeeded — operation returned real output you can render +denied — entity enumerated fine, but the operation was rejected with + a permission/scope/role error (NOT a tool bug — an entitlement + gap in the credential). Collect these for an informational + footer, do not render them alongside successes. +errored — a transient/unexpected failure (timeout, 5xx, malformed + response). These are bugs or service incidents and deserve a + retry or a loud warning, not silent inclusion in the footer. +empty — the operation succeeded but returned no output for this + entity. Silence entirely unless the user asked "why no results". +``` + +The core idea: **enumerate-ability is not operate-ability**, and the wrapper must surface the gap as a distinct result category so the user can understand why "I have 12 knowledge bases but only 2 searches landed." + +**Implementation template** (adapt to your API's error codes): + +```python +PERMISSION_DENIED_MARKERS = ["220030", "no_permission", "Forbidden", "403"] + +def is_permission_denied(result): + if result.get("error") is None: + return False + err = str(result["error"]) + return any(m in err for m in PERMISSION_DENIED_MARKERS) + +def partition(results): + succeeded, denied, errored, empty = [], [], [], [] + for r in results: + if is_permission_denied(r): + denied.append(r) + elif r["error"]: + errored.append(r) + elif r["output"]: + succeeded.append(r) + else: + empty.append(r) + return succeeded, denied, errored, empty +``` + +**Render rule**: show successes first, then any `errored` entries with a ⚠️ prefix (user should care), then `denied` entries in a collapsible `ℹ️ N entities returned 'no permission'` footer. Do not show `empty` entries unless the user asked for the full list. + +**Concrete reference**: `ima-copilot/scripts/search_fanout.py` lines around the `rank_groups` / `is_permission_denied` functions, and `ima-copilot/references/search_best_practices.md` "Permission model" section. + +### Undocumented limit detection + +**The problem**: many third-party APIs have undocumented hard limits — a request that says "return all results" actually returns a truncated subset, and the response contains no `is_end`, `next_cursor`, `has_more`, or equivalent signal to tell you the truncation happened. A naive wrapper will silently show the first N results as if they were the complete set, and the user will make decisions based on a lie. + +**Examples by tool**: + +- **IMA**: `search_knowledge` returns exactly 100 hits per KB on high-frequency queries with no pagination token in the response body. The 100-hit cap is not documented anywhere; the only way to know is to send a query you know matches more than 100 items and observe the exact-100 count. +- **GitHub Search**: `/search/code` caps total results at 1000 but the `total_count` field may report 12000. Without the cap awareness, a wrapper shows "page 10 of 120" and blows up when page 11 returns empty. +- **Notion**: databases with > 100 pages return `has_more: true` correctly for up to ~1000 iterations but silently stop returning new pages around item 2000 on some plans. +- **Google Drive**: `files.list` with a broad query caps at 1000 results per page regardless of `pageSize` parameter, and the `nextPageToken` is omitted — you have to detect the hit-at-1000 as the signal. +- **Confluence / Jira**: `/search` endpoints have per-tenant "result ceiling" configurations that aren't exposed anywhere in the API — you discover them by hitting the wall. + +**The pattern**: detect truncation heuristically and surface it as a prominent warning. The detection rule is usually "result count equals a round number like 50, 100, 500, or 1000, AND no pagination signal in the response" — because legit result sets do not coincidentally round to powers of ten. + +**Implementation template**: + +```python +SUSPICIOUS_ROUND_CAPS = {50, 100, 500, 1000, 10000} + +def looks_truncated(response, results): + """Return True if this response smells like a silent truncation.""" + n = len(results) + # Did we hit a suspicious round cap? + if n not in SUSPICIOUS_ROUND_CAPS: + return False + # Is there any pagination signal? If yes, we can page through, not a silent cap. + for key in ("is_end", "next_cursor", "has_more", "nextPageToken", "next"): + if response.get(key) not in (None, False, "", 0): + return False + return True +``` + +**Render rule**: when `looks_truncated` fires on any branch of the fan-out, append a `⚠️ N entity/entities may have been silently truncated at K results; try a narrower query to see more` block after the results. Do not swallow the signal — the entire point of this pattern is to tell the user about the lie. + +**Concrete reference**: `ima-copilot/scripts/search_fanout.py` `HARD_HIT_CAP` constant and the `truncated` flag propagation, and `ima-copilot/references/search_best_practices.md` "Silent 100-result truncation" section. + +### Scoped liveness checks + +**The problem**: a wrapper's credential-liveness probe usually tests "can I make any authenticated call at all?" — but the credential may have scopes that pass the easy probe and fail the actual operation the skill performs. The user then gets a false ✅ on `diagnose.sh` and a confusing failure later when they try to use the skill's main capability. + +**The ima-copilot case**: `diagnose.sh` probes `search_knowledge_base` with empty query, which needs only `list` scope on any one KB. This passes. But `search_fanout.py` actually needs `search` scope, which is a different tier of permission. A user with `list`-only credentials would see diagnose report everything as healthy and then hit 220030 on every single KB when they tried to run a search. (This is latent in the shipped version and is its own small bug.) + +**The rule**: the liveness check must probe the **lowest-privilege operation the skill actually performs**, not the first API the credential can hit. If the skill has multiple capabilities with different permission tiers, the check must probe the most restrictive tier. If probing the lowest tier would have side effects (e.g., the lowest-privilege operation in the tool is "create a resource"), use the narrowest read equivalent you can find that still requires the target scope. + +**Rule of thumb**: + +``` +For each capability the skill exposes: + identify the minimum scope needed for that capability's main API call + pick the union of all required scopes + design a liveness probe that requires all scopes in that union + if no single call requires all scopes, make multiple probes and require them all to pass +``` + +**Render rule**: `diagnose.sh` should name the scope each probe checks in its output so the user can see which capability is verified. For example: + +``` +✅ Credentials present +✅ Liveness (scope: list) — can enumerate KBs +✅ Liveness (scope: search) — can search the smallest KB +⚠️ Liveness (scope: write) — tried to add a test note and failed; Capability 4 (note creation) will not work until you regenerate credentials with write scope +``` + +**Concrete reference**: the corrected scoped-liveness behavior is a future fix for `ima-copilot/scripts/diagnose.sh` (currently only probes `list` scope, known limitation filed as a follow-up). diff --git a/skill-creator/workflows/wrapper-skill/scripts/init_wrapper_skill.py b/skill-creator/workflows/wrapper-skill/scripts/init_wrapper_skill.py new file mode 100755 index 00000000..c7d5a6fe --- /dev/null +++ b/skill-creator/workflows/wrapper-skill/scripts/init_wrapper_skill.py @@ -0,0 +1,397 @@ +#!/usr/bin/env python3 +""" +init_wrapper_skill.py — Bootstrap scaffold for a new wrapper skill. + +Creates the directory layout and writes stub files with placeholders that +point back at the specific step in workflows/wrapper-skill/workflow.md +that fills each placeholder. The placeholders are deliberately ugly so an +unfinished scaffold is obvious on inspection — you cannot accidentally +commit a half-filled wrapper and mistake it for a real one. + +Usage: + uv run python workflows/wrapper-skill/scripts/init_wrapper_skill.py \\ + --tool "" \\ + --target-dir + +Example: + uv run python workflows/wrapper-skill/scripts/init_wrapper_skill.py ima-copilot \\ + --tool "Tencent IMA" \\ + --target-dir + +This produces: + + // + ├── SKILL.md (with markers) + ├── scripts/ + │ ├── install_.sh (755, with ) + │ └── diagnose.sh (755, with ) + ├── references/ + │ ├── installation_flow.md + │ ├── credentials_setup.md + │ ├── known_issues.md + │ └── best_practices.md + └── config-template/ + └── .json.example (delete if no per-user config) + +After running this script, open workflow.md and start at Step 4 to fill +in each file from the Step 2 mining output. +""" + +import argparse +import os +import re +import stat +import sys +from pathlib import Path + + +# Use a shell-unfriendly marker so unfilled placeholders stand out in grep and +# during skill validation runs. The intent: it is impossible to accidentally +# commit an incomplete wrapper skill without a post-build grep surfacing it. +PLACEHOLDER = "<<< FILL FROM Step {step} of workflow.md >>>" + +SKILL_MD_TEMPLATE = """--- +name: {name} +description: {placeholder_step_4} Describe the wrapper skill in 4-8 sentences. Use the mined Step 2c error strings as literal triggers. Be pushy — false positives are cheaper than false negatives. +--- + +# {display_name} + +{placeholder_step_4} One-sentence purpose. + +## Overview + +{placeholder_step_4} 2-4 sentence overview of the upstream tool, the friction this wrapper removes, and the scope of coverage. + +## Architectural principles (do not violate) + +This skill is a wrapper layer around {tool}. The wrapper contract is non-negotiable: + +- **Never vendor upstream files.** This directory does not contain any copy, fork, or excerpt of {tool}'s own content. +- **Repairs happen at runtime, not at ship time.** Fixes live as instructions in `references/known_issues.md`, not as patched files. +- **Always ask before touching upstream files.** Modifying installed {tool} files requires explicit user consent via AskUserQuestion. +- **Teach rather than hide.** Every fix shows the user exactly what changed and where the backup was saved. + +## What this skill does + +| Capability | Entry point | Detail | +|---|---|---| +| Install upstream {tool} | `scripts/install_{slug}.sh` | See `references/installation_flow.md` | +| Configure credentials | Inline workflow below | See `references/credentials_setup.md` | +| Diagnose and fix known issues | `scripts/diagnose.sh` + workflow below | See `references/known_issues.md` | + +## Routing + +{placeholder_step_4} Fill routing table. + +## Capability 1: Install upstream {tool} + +{placeholder_step_5} See `references/installation_flow.md` for details. + +```bash +bash scripts/install_{slug}.sh +``` + +## Capability 2: Configure credentials + +{placeholder_step_8} See `references/credentials_setup.md`. + +## Capability 3: Diagnose and fix known issues + +{placeholder_step_6} Read this section carefully during agent runtime. It contains the full diagnose-then-repair-with-consent flow that makes this wrapper safe. + +## What this skill refuses to do + +- Vendor, fork, or mirror upstream files into this directory +- Pin an upstream version in this SKILL.md (installer uses overridable defaults, SKILL.md stays version-agnostic) +- Silently patch upstream files — every modification path requires explicit consent +- Hardcode user-specific values + +## File layout + +``` +{name}/ +├── SKILL.md # This file +├── scripts/ +│ ├── install_{slug}.sh # Download → stage → distribute +│ └── diagnose.sh # Read-only health report +├── references/ +│ ├── installation_flow.md +│ ├── credentials_setup.md +│ ├── known_issues.md # Issue registry — source of truth +│ └── best_practices.md +└── config-template/ + └── {slug}.json.example # Optional; delete if no per-user config +``` +""" + +INSTALL_SH_TEMPLATE = """#!/usr/bin/env bash +# +# install_{slug}.sh — Install upstream {tool} to supported agents. +# +# {placeholder_step_5} +# +# Re-run safely — every step is idempotent. + +set -euo pipefail + +{upper_slug}_VERSION="${{{upper_slug}_VERSION:-}}" +BASE_URL="" +STAGING_ROOT="/tmp/{name}-staging" +STAGING_DIR="${{STAGING_ROOT}}/$(date +%s)-$$" + +cleanup() {{ + if [ -n "${{STAGING_DIR:-}}" ] && [ -d "$STAGING_DIR" ]; then + rm -rf "$STAGING_DIR" + fi +}} +trap cleanup EXIT + +# {placeholder_step_5} Fill in the rest of the installer. Key patterns: +# - curl/wget with explicit --fail and HTTP code check +# - unzip/tar into $STAGING_DIR +# - root SKILL.md detection via known-path-first, depth-sort fallback +# - agent auto-detection via $HOME/.claude, $HOME/.agents, $HOME/.openclaw +# - npx -y skills add -g -y -a ... in default symlink mode +# - No --copy flag (symlink mode propagates repairs across agents) +# - Every cp/mv uses `command` prefix to dodge user shell aliases +# +# See skill-creator/workflows/wrapper-skill/patterns.md for the full template. + +echo "TODO: fill install logic per workflow.md Step 5" >&2 +exit 1 +""" + +DIAGNOSE_SH_TEMPLATE = """#!/usr/bin/env bash +# +# diagnose.sh — Read-only health check for upstream {tool} installs. +# +# {placeholder_step_7} +# +# Exit codes: +# 0 — all checks passed +# 1 — one or more issues need user action +# 2 — diagnostic itself failed + +set -uo pipefail + +PASS=0 +WARN=0 +FAIL=0 + +status_ok() {{ echo "✅ $1"; PASS=$((PASS + 1)); }} +status_warn() {{ echo "⚠️ $1"; WARN=$((WARN + 1)); }} +status_fail() {{ echo "❌ $1"; FAIL=$((FAIL + 1)); }} + +echo "=== {name} diagnostic report ===" +echo + +# {placeholder_step_7} Fill in the diagnostic checks: +# 1. Per-agent install presence + canonical dedup via realpath +# 2. Credential presence + liveness check +# 3. One scan_issue_NNN function per entry in references/known_issues.md +# +# See skill-creator/workflows/wrapper-skill/patterns.md for the full template. + +echo "TODO: fill diagnose logic per workflow.md Step 7" >&2 +exit 2 +""" + +REFERENCES_STUB = { + "installation_flow.md": ( + "# Installation Flow — Deep Dive\n\n" + "{placeholder_step_5} Fill from Step 2a of the mining output.\n\n" + "Cover:\n" + "- Why a wrapper installer exists (what upstream's installer doesn't do)\n" + "- Prerequisites (curl, unzip, npx, Node.js version)\n" + "- Agent detection rules\n" + "- Version override mechanism\n" + "- What `npx skills add` does under the hood\n" + "- File layout after install\n" + "- Uninstall procedure\n" + "- Troubleshooting common failures\n" + ), + "credentials_setup.md": ( + "# Credentials Setup — Deep Dive\n\n" + "{placeholder_step_8} Fill from Step 2b of the mining output.\n\n" + "Cover:\n" + "- Where credentials go (XDG paths, file modes)\n" + "- Environment variable fallback\n" + "- Obtaining credentials from the tool's web UI\n" + "- Saving credentials securely\n" + "- Liveness test command\n" + "- Rotation procedure\n" + "- Security considerations\n" + ), + "known_issues.md": ( + "# Known Issues in Upstream {tool}\n\n" + "{placeholder_step_6} Fill one entry per bug that was actually " + "encountered and fixed in the distillation source session. " + "Use the template in skill-creator/workflows/wrapper-skill/patterns.md.\n\n" + "## How the agent should use this file\n\n" + "When `scripts/diagnose.sh` reports a `⚠️` line mentioning `ISSUE-`:\n\n" + "1. Explain to the user in plain language.\n" + "2. Use AskUserQuestion to let them pick a repair strategy.\n" + "3. Execute the chosen strategy's commands. Every repair backs up originals first.\n" + "4. Re-run diagnose to confirm the fix.\n" + "5. Remind the user that upstream upgrades replace these files, so reruns are expected.\n\n" + "## Issue registry\n\n" + "<<< Add ISSUE-001, ISSUE-002, ... entries here — one per bug from Step 2c. >>>\n" + ), + "best_practices.md": ( + "# Best Practices for Using {tool}\n\n" + "{placeholder_step_8} Fill from Step 2d of the mining output.\n\n" + "Cover:\n" + "- Non-obvious usage patterns discovered during the source session\n" + "- Recommended defaults\n" + "- Common pitfalls users should avoid\n" + "- When to use this tool vs alternatives\n" + ), +} + +CONFIG_TEMPLATE_STUB = """{{ + "_comment_field1": " Explain what field1 does", + "field1": ["placeholder-value"], + + "_comment_field2": " Explain what field2 does", + "field2": [] +}} +""" + + +def slugify(name: str) -> str: + """Convert a skill name to a filename-friendly slug.""" + slug = re.sub(r"[^a-zA-Z0-9]+", "_", name).strip("_").lower() + if not slug: + raise ValueError(f"cannot slugify empty string from name: {name!r}") + return slug + + +def write_file(path: Path, content: str, executable: bool = False) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + if executable: + mode = path.stat().st_mode + path.chmod(mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser( + description="Scaffold a new wrapper skill for a third-party CLI tool.", + formatter_class=argparse.RawDescriptionHelpFormatter, + epilog=__doc__, + ) + parser.add_argument( + "name", + help="Wrapper skill name (e.g., 'ima-copilot', 'yt-dlp-companion'). Directory name.", + ) + parser.add_argument( + "--tool", + required=True, + help="Display name of the upstream tool (e.g., 'Tencent IMA', 'yt-dlp').", + ) + parser.add_argument( + "--target-dir", + default=".", + help="Parent directory where the wrapper skill will be created (default: current dir).", + ) + parser.add_argument( + "--no-config-template", + action="store_true", + help="Skip creating config-template/ (use when the tool has no meaningful per-user config).", + ) + parser.add_argument( + "--force", + action="store_true", + help="Overwrite existing files if the target directory is not empty.", + ) + args = parser.parse_args(argv) + + target_dir = Path(args.target_dir).expanduser().resolve() + if not target_dir.is_dir(): + print(f"error: target directory does not exist: {target_dir}", file=sys.stderr) + return 1 + + skill_dir = target_dir / args.name + if skill_dir.exists() and not args.force: + if any(skill_dir.iterdir()): + print( + f"error: {skill_dir} already exists and is not empty. " + f"Pass --force to overwrite.", + file=sys.stderr, + ) + return 1 + + slug = slugify(args.name) + upper_slug = slug.upper() + + fmt = { + "name": args.name, + "display_name": args.name.replace("-", " ").replace("_", " ").title(), + "tool": args.tool, + "slug": slug, + "upper_slug": upper_slug, + "placeholder_step_4": PLACEHOLDER.format(step="4"), + "placeholder_step_5": PLACEHOLDER.format(step="5"), + "placeholder_step_6": PLACEHOLDER.format(step="6"), + "placeholder_step_7": PLACEHOLDER.format(step="7"), + "placeholder_step_8": PLACEHOLDER.format(step="8"), + } + + print(f"▶ Scaffolding wrapper skill: {args.name}") + print(f" Location: {skill_dir}") + print(f" Tool: {args.tool}") + print(f" Slug: {slug}") + print() + + # SKILL.md + write_file(skill_dir / "SKILL.md", SKILL_MD_TEMPLATE.format(**fmt)) + print(f" ✓ SKILL.md") + + # scripts/ + write_file( + skill_dir / "scripts" / f"install_{slug}.sh", + INSTALL_SH_TEMPLATE.format(**fmt), + executable=True, + ) + print(f" ✓ scripts/install_{slug}.sh") + + write_file( + skill_dir / "scripts" / "diagnose.sh", + DIAGNOSE_SH_TEMPLATE.format(**fmt), + executable=True, + ) + print(f" ✓ scripts/diagnose.sh") + + # references/ + for filename, stub_template in REFERENCES_STUB.items(): + write_file( + skill_dir / "references" / filename, + stub_template.format(**fmt), + ) + print(f" ✓ references/{filename}") + + # config-template/ + if not args.no_config_template: + write_file( + skill_dir / "config-template" / f"{slug}.json.example", + CONFIG_TEMPLATE_STUB, + ) + print(f" ✓ config-template/{slug}.json.example") + else: + print(f" (skipped config-template per --no-config-template)") + + print() + print(f"✓ Wrapper skill scaffolded at: {skill_dir}") + print() + print("Next steps:") + print(f" 1. Open workflow.md and start at Step 2 (Mine the conversation history)") + print(f" 2. Fill each file in this order: SKILL.md → install script → known_issues.md") + print(f" → diagnose.sh → references → (optional) config template") + print(" 3. Grep for '<<< FILL FROM Step' — no matches should remain when done") + print(f" 4. Run verification_protocol.md before committing") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/skill-creator/workflows/wrapper-skill/verification_protocol.md b/skill-creator/workflows/wrapper-skill/verification_protocol.md new file mode 100644 index 00000000..4bb52e1f --- /dev/null +++ b/skill-creator/workflows/wrapper-skill/verification_protocol.md @@ -0,0 +1,141 @@ +# Verification Protocol + +How to verify a freshly distilled wrapper skill before commit. A generated wrapper is made of two very different kinds of files, and they require two very different kinds of verification. Confusing the two is a common mistake that will make you either (a) skip verification entirely because "the session already proved it works" or (b) waste effort re-dogfooding things that were already dogfooded. + +This protocol names the two kinds and says what to do with each. + +## Two tracks of verification + +### Track 1 — Literal transcriptions (session cross-reference) + +These files are **re-encodings of commands and observations that actually happened in the source session**. They contain nothing that hasn't been run already: + +- `scripts/install_.sh` — a shell script whose every non-trivial line should appear somewhere in the session's history as a command that was executed +- `references/known_issues.md` — each ISSUE entry is a literal transcription of an error message that was observed plus a fix command that was run +- `scripts/diagnose.sh` at the **detection-logic** level — the set of states it checks for corresponds 1:1 to the states the session worked through +- `references/installation_flow.md` and `references/credentials_setup.md` — prose retellings of what the session did + +For these files, verification is **session cross-reference**: prove that every artifact in the file is traceable to a moment in the conversation. If something in the file has no provenance in the session, it is speculation — delete it. + +**Why not run these files end-to-end instead of cross-referencing**: running `install_.sh` in a sandbox doesn't prove it's right — it proves it doesn't crash, which is much weaker. The session already proved the commands work in the real environment. What can still be wrong is a **transcription error**: a wrong flag, a paraphrased command, a bug fix applied to the wrong file. Cross-reference catches exactly those. + +### Track 2 — Runtime logic (smoke test / unit test) + +These files contain **original code that was not literally run in the session** — code that encodes patterns derived from the session but is new as of the distillation: + +- `scripts/search_fanout.py`, `scripts/report.js`, or any other language file under `scripts/` whose purpose is to execute logic at skill-run time +- `scripts/diagnose.sh` at the **glue-code** level — string parsing, path resolution, realpath-based dedup, error-message matching — anything that was *written* during distillation rather than transcribed from a session command +- Any cross-platform compatibility shim (different install paths per OS, different shell quoting per environment) because the session ran on exactly one platform +- Any configuration file parser, API client, or data transformation that generates output for the user + +For these files, **session cross-reference is insufficient**. The cross-reference only proves that the idea behind the file came from the session — it cannot prove that the implementation of the idea is correct. A `rank_groups()` function inspired by the session's discussion of search-result partitioning still has to actually partition correctly, which requires running it. + +For Track 2 files: + +1. **Write at least one smoke-test invocation** of the file with realistic input, run it, and confirm the output matches expectations. If the file is a Python script, `python3 scripts/.py `. If it's a shell function, exercise it in a shell with captured output. +2. **If the file has cross-platform code**, test on the platform the session ran on first, then note in the file that other platforms are untested until a user reports running on them. Do not pretend otherwise. +3. **For scripts that depend on external state** (config files, credentials, network resources), use a minimal fake fixture when possible. For `search_fanout.py`-style scripts, a small `IMA_COPILOT_CONFIG` pointing at a test JSON is enough. + +If a Track 2 file cannot be smoke-tested without reaching a production system (e.g., it hits an API that requires real credentials), the protocol accepts the limitation but requires the wrapper to ship a **scoped warning** in its `references/installation_flow.md` saying "scripts/ is shipped without end-to-end verification; exercise it with a no-op input before relying on its output in production." + +### How to tell which track a file belongs to + +Ask: **"If I deleted this file and regenerated it from the session transcript alone, would the regeneration be byte-identical to the original?"** + +- **Yes** → Track 1. The file is a literal transcription and cross-reference verification is sufficient. +- **No** → Track 2. The file contains original code or decisions not in the transcript, and needs smoke testing in addition to cross-reference. + +Most wrappers are ~70% Track 1, ~30% Track 2. The ima-copilot reference is about 60/40 — `install_ima_skill.sh`, `known_issues.md`, and `references/installation_flow.md` are Track 1; `search_fanout.py`, `diagnose.sh` (glue code), and the symlink-dedup logic are Track 2. Both tracks got appropriate treatment in the canonical example. + +## The full verification checklist + +### Step 1 — Structural validity (both tracks) + +Run the repo's standard validation from the repo root. Use the `git rev-parse --show-toplevel` trick to avoid CWD surprises: + +```bash +REPO_ROOT=$(git -C . rev-parse --show-toplevel) +cd "$REPO_ROOT/skill-creator" +uv run --with PyYAML python -m scripts.quick_validate "$REPO_ROOT/" +uv run python -m scripts.security_scan "$REPO_ROOT/" +``` + +Both should pass. `quick_validate` enforces SKILL.md frontmatter shape, the 1024-char description cap, and path reference integrity. `security_scan` catches committed credentials, personal directories, and company names. + +### Step 2 — Track 1 verification: session cross-reference + +Walk through every Track 1 file and confirm each non-trivial line traces to the session: + +- **`install_.sh`**: for each shell command, grep the session history for the literal command text. Paraphrases don't count — the script should match the commands that actually ran in the session, not commands that would have been *equivalent*. If a command in the script has no grep hit, either delete it or replace it with the actual command that ran. +- **`known_issues.md` ISSUE entries**: for each entry, locate the session moment where the literal error message first appeared, and the session moment where the fix was applied. The error message in the entry should be byte-identical to what was observed. The fix commands should be byte-identical to what was run. +- **`diagnose.sh` detection states**: each state returned by a check function should correspond to a state the session actually worked through. If `check_submodule` returns code 5 for a state nobody encountered, that's speculative — remove it unless there's a grounded reason (like the Step A5 dual-state fix, which was added because the session explicitly considered and closed the conflict case). +- **`installation_flow.md` prose**: each section should paraphrase something that happened in the session. Sections about "what to do if X fails" are legitimate only if X was observed in the session, even briefly, or is a trivially-obvious failure mode. + +When in doubt, grep the conversation history. If grep finds nothing and you can't justify why the content should exist, the content is unsupported — delete it. + +### Step 3 — Track 2 verification: smoke test + +For every Track 2 file in the wrapper: + +1. **Identify the minimum input** that exercises the file's main code path. For a Python script that takes command-line args, this is a sample invocation. For a shell function called from other scripts, this is a shell harness that calls it with test values. +2. **Run it** and inspect the output. The output should match what you expect based on the session's discussion of what the file is supposed to do. +3. **If the file depends on external state** (config file, environment variables, network), use a minimal fake fixture. Example: `IMA_COPILOT_CONFIG=/tmp/test-config.json python3 scripts/search_fanout.py "test query"` with a small hand-written config. +4. **For cross-platform code**, test only on the session's platform. Note any untested platforms in the file's header comment so future users know what's verified. +5. **For scripts that hit real APIs with real credentials**: if you can run a no-op probe (empty-query search, list-with-limit-1, etc.) against a real credential, do that. If not, ship the file with a warning and a TODO to add a mockable test in a follow-up. + +The smoke test does not have to catch every bug — it has to catch "the script crashes immediately" and "the script's main code path returns complete nonsense". Anything more sophisticated is a bonus. + +### Step 4 — Mental dry-run of the wrapper as a fresh agent + +Read the generated `SKILL.md` as if you were a new Claude session and a user has just asked you to install the tool. Walk through the routing: + +- Does the description trigger for the symptom the original session started with? +- Does the Capability 1 path lead to a complete install if Claude follows the instructions literally and does not consult its memory of the source session? +- Does the Capability 3 diagnose flow reach each `known_issues.md` entry? +- Would a new user, given only the description and the references, be able to understand each known issue and decide between repair strategies? + +If any of these breaks, the wrapper is not yet shippable. The usual fix is adding more signal to the description (for triggering) or adding more concrete detail in a reference file (for understanding). + +### Step 5 — Release metadata consistency + +Before commit, confirm the marketplace and release docs are consistent with the new skill: + +- `marketplace.json`: new `plugins[]` entry exists, `metadata.version` bumped, description list mentions the new skill. +- `CHANGELOG.md`: entry under the new version with a summary of what was added. +- `README.md` and `README.zh-CN.md`: if the repo has a skill index, the new skill is listed with accurate description. +- Repo-level `CLAUDE.md`: if it counts skills, the count is incremented. +- `.security-scan-passed` file exists in the wrapper directory (created by `security_scan.py`). + +A common slip is committing the wrapper skill but forgetting to add it to `marketplace.json`. Run a quick guard before `git add`: + +```bash +grep -q '""' "$REPO_ROOT/.claude-plugin/marketplace.json" \ + || echo "MISSING: add wrapper to marketplace.json plugins[] and bump metadata.version" +``` + +The grep must print nothing before you proceed to commit. + +## When verification surfaces a problem + +If Step 2 (cross-reference) turns up a mismatch between what a Track 1 file says and what the session actually did, the correct fix is to **re-mine the relevant section of Step 2 in the workflow and regenerate the affected file**. Do not patch the generated file to match your memory — patch it to match what the session actually contained, because that is the source of truth. + +If Step 3 (smoke test) turns up a runtime failure in a Track 2 file, the correct fix is to **fix the code** (not the session transcript). Runtime failures are bugs in the distillation of a pattern from the session into code — the pattern may be right even if the implementation is wrong. + +Common mismatches and their causes: + +| Mismatch | Track | Usual cause | Fix | +|---|---|---|---| +| `install_.sh` contains a flag you don't recognize | 1 | The flag was added during mid-session iteration | Search the session for when the flag was introduced. If the pre-flag version worked, remove it. | +| `known_issues.md` entry has a plausible but slightly off error message | 1 | Paraphrase drift | Search for the literal error and paste it verbatim. | +| Two known issues describe the same underlying problem | 1 | Distillation split a single bug into two entries | Merge them. | +| Credential path in `credentials_setup.md` doesn't match `install_.sh` | 1 | Distillation drew from two moments of the session | Determine which path the session ended with and use that in both files. | +| `diagnose.sh` detects an issue that `known_issues.md` doesn't describe | 1 or 2 | You added a speculative check | Either add the matching `known_issues.md` entry if the issue is real, or remove the check. | +| `scripts/*.py` crashes on the sample smoke input | 2 | Implementation bug | Fix the code, re-run the smoke test, and add a comment explaining the fix so future maintainers don't regress it. | +| `scripts/*.py` returns nonsense output on the sample smoke input | 2 | Logic bug (wrong partitioning, wrong sort key, wrong fallback) | Fix the code and consider whether the bug is a symptom of a missing test case to commit. | +| A shell function in `diagnose.sh` reports clean on a state it should flag | 2 | Incomplete state coverage in the detection logic | Add the missing state to the check function (see the A5 dual-state fix in `ima-copilot` for an example). | + +## Why this matters + +It is tempting to skip Track 2 because it feels like duplicated work — "the session already dogfooded this". It isn't. The session dogfooded *the install commands and the fixes*; it did not dogfood *the distillation of those commands and fixes into new Python/shell code*. The distillation is where bugs get introduced. A wrapper that skips Track 2 ships untested code against unexercised paths, and the author won't know until a second user hits a failure mode the author didn't think about. + +The simplest heuristic: if you wrote new code (not just pasted commands from the session), run it at least once with a realistic input before you commit it. diff --git a/skill-creator/workflows/wrapper-skill/workflow.md b/skill-creator/workflows/wrapper-skill/workflow.md new file mode 100644 index 00000000..1356eb04 --- /dev/null +++ b/skill-creator/workflows/wrapper-skill/workflow.md @@ -0,0 +1,318 @@ +# Wrapper Skill Workflow + +A **retrospective distillation** workflow. Reads the current conversation and turns it into a reusable wrapper skill for a third-party CLI tool. Opposite of skill-creator's main workflow, which is prospective (design first, build, test). + +## When this workflow applies + +Use this workflow **at the end of a session** where all of the following happened in the conversation history: + +- The user asked Claude to install a third-party tool (a CLI, a proprietary skill package, a `.zip` from an official distributor, an npm/pip package — anything with an installer). +- Claude and the user actually installed it, configured credentials, ran it, and encountered real friction: install errors, missing flags, undocumented behavior, broken submodule files, shell quirks. +- Claude and the user diagnosed each real problem and arrived at a working fix for it — commands they ran, files they edited, configurations they set. +- At the end, the user says something like "wrap this up as a skill", "save this as a wrapper skill", "so other people don't have to go through what we just went through", "把这次 session 做成一个 skill", or similar. + +**Do not use this workflow if:** +- The user is asking to *start* installing a tool. Run the tool first, diagnose what breaks, then come back and use this workflow. +- The session went smoothly with no real problems. There is nothing to distill — the upstream installer was fine, use it directly. +- The user wants a generic, from-scratch skill for something unrelated to a third-party tool. Use the main skill-creator workflow instead. + +This workflow only wins when the conversation is the source material. The value comes from capturing **battle-tested** knowledge, not from writing speculative code. + +## Canonical reference implementation + +This workflow was abstracted from a real session that produced [`ima-copilot`](https://github.com/daymade/claude-code-skills/tree/main/ima-copilot) — a wrapper around the Tencent IMA skill. Every abstract step below has a concrete instance in that skill's files. When in doubt about how to interpret a step, read the corresponding file in `ima-copilot/` and imitate it. + +## Step 1 — Confirm scope with the user + +Before scanning, check with the user (via **AskUserQuestion** — see fallback note below if that tool is unavailable): + +1. **What is the tool we're wrapping?** (exact name, distribution URL if known) +2. **What should the wrapper skill be called?** Suggest `-copilot` or `-companion` as defaults. Let the user override. +3. **Which repo does this land in?** (e.g. `claude-code-skills`, `claude-code-skills-pro`, a private repo) +4. **Which target agents should it install to?** Defaults to the three that `vercel-labs/skills` handles cleanly: Claude Code, Codex, OpenClaw. Allow user to narrow or expand. + +Confirm in one sentence before mining. If the user hasn't given you enough context to fill these in, ask — don't guess. + +### AskUserQuestion fallback + +This workflow references `AskUserQuestion` repeatedly — it is the Claude Code tool that renders a multi-choice prompt with labeled options and lets the user pick one, returning a structured answer. It is the best possible affordance for decisions that have more than one right answer (like "which repair strategy should I apply?"). + +**Not every harness exposes this tool.** Codex does not have it. Older Claude Code versions do not have it. Custom agent builds may not have it. **The consent requirement is not the tool — it is the explicit user choice.** If `AskUserQuestion` is unavailable, fall back to printing the options inline in plain text with numbered labels and then stop, waiting for the user's reply before continuing. Example: + +``` +I need your consent before touching upstream files. Pick one: + + 1) Strategy A — rename notes/SKILL.md → notes/MODULE.md and patch root references (recommended, smaller footprint) + 2) Strategy B — prepend minimal frontmatter to the submodule files (minimal diff, creates two sub-skill names) + 3) Skip — leave it broken for now + +Reply with 1, 2, or 3. +``` + +After the user replies, continue with the chosen strategy's exact commands. The requirement is that the user makes an informed choice before any upstream-file modification happens — `AskUserQuestion` is the preferred rendering, not the definition. + +## Step 2 — Mine the conversation history + +This is the most important step. Scan the conversation from its beginning to the point where this workflow was triggered. Extract concrete, literal snippets for each category below. Do not paraphrase error messages or commands — copy them verbatim. + +### How to access the conversation + +Where the history lives depends on whether you are in the same session that produced the debugging work or in a follow-up session: + +- **Same session (most common)**: scroll your own message history upward. Start from the most recent messages and walk back until you find the first mention of the tool being installed. Everything between that point and now is your source material. You already have this in context — you do not need any tool to "fetch" it. + +- **Follow-up session (the user came back later)**: use the `claude-code-history-files-finder` skill if it is installed, or read the session JSONL directly from `~/.claude/projects//.jsonl`. The escaped cwd is the working directory with `/` replaced by `-` (for example, `/claude-code-skills` becomes ``). Grep the JSONL for literal error fragments (`"error"`, `"Traceback"`, shell prompt characters), extracted shell commands, and file paths the user edited. The JSONL is newline-delimited JSON with one record per message. + +- **Neither available**: stop the workflow and tell the user. Do **not** proceed by inventing plausible install commands or plausible bug fixes — that violates the workflow's entire reason to exist. Say "I cannot find the session history this workflow needs. Can you paste the relevant install log, error messages, and fix commands directly into this conversation so I can work from them?" and wait. Fabricated content is worse than no wrapper skill. + +The rules that follow (2a-2e) apply regardless of which source you used. + +### 2a — The working install flow + +What did it take to actually get the tool installed? + +- **Source**: the canonical URL, package name, git repo, or local archive +- **Download/extract commands**: exact `curl`/`wget`/`unzip` invocations that succeeded +- **Target layout**: which directories received files, on which agents +- **Distribution tool**: did you use `npx skills add` (vercel-labs/skills), a custom script, a package manager? With which flags? +- **Detection**: did you check for installed agents before writing? What was the detection rule? +- **Cleanup**: what was the staging dir strategy, and when was it removed? +- **Version pinning**: did you hard-code a version, accept env override, or both? + +In `ima-copilot`, all of this became `scripts/install_ima_skill.sh`. + +### 2b — Credential setup + +- **What credentials the tool needs** (API key, client ID, OAuth token, SSH keys, etc.) +- **Where you chose to store them** (env var / XDG config file / keychain) +- **Permission mode** you set on the files +- **Env var fallback order** you decided on (env > file, or file > env) +- **The liveness call** you used to verify the credentials work (what endpoint, what request, what success indicator) + +In `ima-copilot`, this became `references/api_key_setup.md`. + +### 2c — Bugs encountered AND resolved + +This is the gold. For each real bug you hit and actually fixed in the conversation, extract: + +- **Symptom** — the literal error message, log line, or observed misbehavior. Copy it verbatim from the conversation. +- **Root cause** — what you discovered after investigation. Include the "aha" moment if there was one. +- **Fix commands or code change** — the exact commands or diff that resolved it. +- **Verification** — how you confirmed the fix worked. What you re-ran and what you expected to see. +- **Reversibility** — if the fix modified files, where did you back up the originals? +- **Idempotency** — can the fix run twice without harm? If not, what guards did you add? +- **Attribution** — which upstream version, which agent, which platform you saw it on. + +**Rules for what counts as a bug worth capturing:** + +- ✅ It was real — you saw the symptom, not just imagined it. +- ✅ You fixed it — there's a concrete resolution, not a "let's pivot" or "we gave up". +- ✅ The fix is reusable — another user hitting the same symptom could run the same fix. +- ❌ Skip dead ends where you abandoned an approach without a working fix. +- ❌ Skip user errors (wrong password typed, wrong directory) unless the tool's error message was so unhelpful that documenting the confusion is a win. +- ❌ Skip "Claude tried X and X was wrong, then tried Y" loops — only Y matters. + +Each bug becomes one entry in the generated `references/known_issues.md` with a stable ID like `ISSUE-001`, `ISSUE-002`. See the format in `patterns.md` and the live example in `ima-copilot/references/known_issues.md`. + +### 2d — Design decisions + +Decisions the user and you made together that are not obvious from the code. For each: + +- **The decision** (short noun phrase) +- **The alternatives considered** +- **Which side won** +- **Why** — quote or paraphrase the conversation + +Examples from the ima-copilot session: + +- Symlink vs `--copy` mode for `npx skills add`: chose symlink because "修一次同步所有 agent" was the user's explicit preference. +- XDG credentials (`~/.config/ima/`) vs env vars: chose XDG as primary with env vars as fallback because persistent files are less ceremony for local dev. +- `command mv` / `command cp` prefix in repair commands: discovered mid-dogfood that the user's shell aliased `mv` to `mv -i`, causing an interactive prompt hang. +- Root `SKILL.md` detection via explicit path-first, then depth-sorted fallback: discovered when `find` surfaced a submodule's `SKILL.md` as the "first match" and broke `npx skills add`. + +Design decisions that reference real debugging moments are the most valuable, because they encode the "why" that would otherwise be lost. + +### 2e — Noise to discard + +What **not** to put in the distilled skill: + +- Random conversational digressions +- Tool invocations that Claude made and then rolled back +- Code written and then replaced before it ever ran +- Debates without a resolution +- Discussions that were educational but didn't produce a code artifact +- Anything from a previous session that isn't in this conversation's history + +**If you are unsure whether a piece is signal or noise, ask the user rather than guessing.** The cost of asking once is much lower than the cost of shipping a wrapper skill full of half-baked lessons. + +## Step 3 — Scaffold the skeleton + +Run the scaffolding script: + +```bash +python3 skill-creator/workflows/wrapper-skill/scripts/init_wrapper_skill.py \ + \ + --tool "" \ + --target-dir +``` + +This creates the directory layout and writes stub files with `` placeholders. The layout matches `ima-copilot/` for consistency and to take advantage of the shared validation tooling. + +If the scaffolding script is missing or fails (e.g. the target repo has an unusual layout), fall back to creating the directories manually — the shape matters more than the tooling: + +``` +/ +├── SKILL.md +├── scripts/ +│ ├── install_.sh +│ └── diagnose.sh +├── references/ +│ ├── installation_flow.md +│ ├── credentials_setup.md +│ ├── known_issues.md +│ └── best_practices.md +└── config-template/ + └── .json.example # only if the tool has meaningful user-level preferences +``` + +If the tool has no meaningful user preferences (no priority lists, no per-user config), drop `config-template/` entirely. Don't invent configuration surface area that wasn't in the original session. + +## Step 4 — Fill SKILL.md + +Open `patterns.md` and copy the SKILL.md template. Fill in: + +- `name` (from Step 1) +- `description` — a dense, trigger-heavy description including tool name, related keywords, and "when to use" signals extracted from Step 2a and 2c. The description should be *pushy* per skill-creator's guidance: err on the side of triggering slightly too often. +- Routing table referring to the scripts and references you're about to fill +- A "What this skill refuses to do" section that pins down the wrapper contract (see `architecture_contract.md`). + +Important: the description lists the symptoms from Step 2c verbatim as triggers. If the session uncovered `Skipped loading skill(s) due to invalid SKILL.md` as an error, that exact string goes in the description. Users hit by the same error will then have their future sessions trigger this skill. + +## Step 5 — Fill install_.sh + +Copy the install script template from `patterns.md`. Fill from Step 2a. Every command in the template must be justified by a command the user or Claude actually ran in the conversation. Do not add command lines that were never executed — those are speculative and have not been verified. + +Include the patterns from `patterns.md` that are almost always needed: + +- `set -euo pipefail` +- `trap cleanup EXIT` for staging +- **Prerequisite checks**: `command -v` loop for `curl`, `unzip`, `npx`, plus a separate numeric Node.js ≥18 check (parsing `node --version`). The Node check is its own step because `command -v node` only verifies presence, and `npx skills add` fails opaquely on Node 16. +- **Download integrity defense in depth**: `curl --fail -o -w "%{http_code}"`, explicit `!= "200"` branch with a version-override hint, then a `wc -c` size sanity check rejecting archives below an absolute floor before extraction. The size check is what catches redirect-to-HTML-error-page failures that return a "success" status. +- Agent auto-detection against known home directories, plus a documented **zero-agents-detected fallback policy** — default to claude-code after printing a "looked for: …" explanation, because aborting is hostile and silent-skip is mystifying. +- Version override via `--version` flag and env var +- `command` prefix on any `cp`/`mv`/`rm`/`sed` operations +- Root-file search that prefers known layouts before falling back to depth-sorted search + +## Step 6 — Fill known_issues.md + +Copy the known_issues format from `patterns.md`. For each bug from Step 2c, create one entry with the full schema: + +```markdown +### ISSUE- — + +**Status**: +**Symptom**: ...literal error message from session, verbatim... +**Root cause**: ...what was discovered... +**Impact**: ...what the user sees if unfixed... +**Why upstream probably hasn't fixed it**: +**How to explain it to the user** (plain language): ... + +**Repair strategies**: + +#### Strategy A — +... exact commands using `command cp` / `command mv` / `command rm` / `command sed` ... +**Rollback**: ... exact commands ... +**Pros**: ... +**Cons**: ... + +#### Strategy B — +... if there's a real tradeoff ... + +#### Strategy skip — Leave the file alone + +``` + +Every command in every strategy must be: + +- **Idempotent** — rerunning after the fix is applied is a safe no-op. Guard every backup `cp` with `[ -f "..." ] && \` so reruns don't print "file not found". +- **Reversible** — it backs up originals to `/tmp/-backups/$(date +%Y%m%d-%H%M%S)/` before modifying anything. The `%Y%m%d-%H%M%S` format sorts correctly and is human-readable. +- **Alias-safe** — uses `command cp` / `command mv` / `command rm` / `command sed`, never the bare form, to dodge user shell aliases like `alias mv='mv -i'` that would hang the script on a prompt. `alias rm='rm -i'` is equally common and affects cleanup and rollback paths. +- **Cross-platform portable for sed** — use `sed -i.bak ... && command rm -f ".bak"` which works on both BSD sed (macOS) and GNU sed (Linux). Bare `-i` and `-i ''` are mutually incompatible. + +See `ima-copilot/references/known_issues.md` for a fully-fleshed example with two strategies (rename vs prepend frontmatter) plus the skip branch. + +## Step 7 — Fill diagnose.sh + +Copy the diagnose template from `patterns.md`. For each bug from Step 2c, add a detection check that returns a **distinct code for every post-repair state** the wrapper can produce. Binary "OK / BROKEN" detection is not enough — see the "Detection function return-code contract" subsection of `patterns.md` for the full state list, but the short version is: + +- One code for "original untouched and already valid" +- One code for "original untouched and still broken" +- One code for "target file not present at all" (legitimately different from BROKEN) +- One code per healthy post-repair state (one per Strategy A, B, …) +- **One code for the dual-state conflicted case** where files from more than one strategy exist simultaneously + +The dual-state code is the single hardest lesson from the ima-copilot session and the single place a careless author will skip. It prevents the "I ran the repair, diagnose says clean, but my install is subtly broken" failure mode. + +diagnose.sh is **strictly read-only**. It never modifies files. It returns: + +- `0` — everything healthy +- `1` — one or more issues need user action (including CONFLICTED states that require manual cleanup) +- `2` — diagnostic itself failed (e.g. network error on liveness check) + +If the tool installs to multiple agents via symlinks (common with `npx skills add` in its default mode), diagnose must recognize shared canonical installs via `realpath` and scan each underlying directory exactly once. Otherwise users see the same issue reported once per agent, which is confusing. + +`find_install` should take a *variadic list* of candidate paths per agent, not a single path. This matters most for agents whose home-directory layout has not stabilized (like OpenClaw), where multiple candidate paths need to be probed in order. Designing the helper as variadic from day one avoids a painful refactor when a second candidate path becomes necessary. + +## Step 8 — Fill references + +Four standard files. Content comes from Step 2: + +- `installation_flow.md` — prose deep dive on how the installer works, why the flags are what they are, troubleshooting for common failures (from 2a) +- `credentials_setup.md` — XDG paths, env var fallback, liveness check, rotation procedure (from 2b) +- `known_issues.md` — already written in Step 6 +- `best_practices.md` — the non-obvious usage patterns discovered in the session (from 2d and general usage observations) + +Each reference file should be lean. If it grows beyond ~200 lines, split it. + +## Step 9 — Fill config-template (if applicable) + +Only if Step 1 established that the tool has meaningful per-user configuration. Write a JSON template with illustrative-only values and `_comment_` entries explaining each field. Do NOT include any real user data — the values in the template are examples that users replace. + +## Step 10 — Verify the generated skill + +See `verification_protocol.md` for the full verification procedure. The short version: + +1. Run `quick_validate.py` against the generated directory +2. Run `security_scan.py` against the generated directory +3. Run the generated `diagnose.sh` against the actual state the session left you in, and confirm it reports the issues that were present and the fixes that were applied — this closes the loop between the session's real work and the skill's description of that work +4. Update the relevant marketplace `marketplace.json` with a new plugin entry and bump versions +5. Update `CHANGELOG.md` / `README.md` / `README.zh-CN.md` / repo `CLAUDE.md` per the hosting repo's release guide + +## Step 11 — Commit + +One commit per wrapper skill. Commit message format: + +``` +feat(): add companion skill v1.0.0 + +<2-3 sentence summary of what this skill wraps and why> + +Distilled from a real install-and-debug session that encountered +issues, all of which are documented in references/known_issues.md with +working repair commands. +``` + +Do not push. Let the user decide when to push. + +## Anti-patterns + +Things that should make you stop and reconsider: + +- Writing code that did not appear in the session. If you find yourself inventing a command, it is speculation — go back and either (a) find where in the session it actually ran, or (b) skip it. +- Documenting a bug with no fix. Known issues without fixes are noise. Omit them. +- Vendoring any upstream files into the new skill directory. If you need to reference upstream, reference it by URL or by install-path, not by copying. +- Generating a repair command that you haven't mentally walked through for idempotency. Every command should be safe to run twice. +- Hardcoding the user's real file paths (`/Users//...`, etc.) into any file. Use `$HOME` or the agent's standard install locations. +- Committing credentials, tokens, or any personally identifying content. + +If you catch yourself doing any of the above, stop and ask the user. diff --git a/slides-creator/.security-scan-passed b/slides-creator/.security-scan-passed new file mode 100644 index 00000000..b80cd6cc --- /dev/null +++ b/slides-creator/.security-scan-passed @@ -0,0 +1,4 @@ +Security scan passed +Scanned at: 2026-04-19T23:10:12.585729 +Tool: gitleaks + pattern-based validation +Content hash: 12cbafeb3e9d8320bbd4469335891d8a58bacf49c11d22a6e346dc038a90fb09 diff --git a/slides-creator/SKILL.md b/slides-creator/SKILL.md new file mode 100644 index 00000000..02561f1d --- /dev/null +++ b/slides-creator/SKILL.md @@ -0,0 +1,594 @@ +--- +name: slides-creator +description: Narrative-first slide deck creation. Guides users through structured narrative design (ABCDEFG model), then delegates visual generation to baoyu-slide-deck. Triggers on "create slides", "make a presentation", "generate deck", "slide deck", "PPT", or when user needs to turn content into visual slides. +context: fork +agent: general-purpose +--- + +# Slides Creator + +**Narrative-first slide deck creation.** + +This skill does what machines can't do — narrative co-design with humans — and delegates everything else to the best tool for the job (`baoyu-slide-deck`). + +## First Law: The User's Voice Is Primary + +**This is the highest-priority rule. Nothing overrides it.** + +> AI cannot write high-quality content for the user. It can only help the user express their own content better. +> +> **Step 1 is ALWAYS: collect the user's original words.** Their transcripts, their articles, their notes, their voice. AI-generated content without user source material is garbage — polished, plausible, unusable garbage. +> +> **Weight hierarchy**: User's own words > User-approved external material > AI synthesis > AI invention. +> +> The output must sound like the user, at their best. Something they could actually say, confidently, in their own voice. + +**Corollary**: If the user has no existing content, your job is to help them articulate their thoughts through structured conversation — NOT to fabricate content for them. + +**Corollary**: External materials (articles, references) must be presented to the user for selection and validation. AI does not decide what is relevant. The user does. + +**See**: `references/content-creation-first-law.md` for full principle, application to all content types, and failure modes. + +## Architecture + +``` +slides-creator (this skill) +├── Phase 0: Source Collection ← Gather user's original words +├── Phase 1: Narrative Design ← Human expertise + ABCDEFG discussion +├── Phase 2: Content Structuring ← Convert narrative to machine-readable input +├── Phase 3: Delegate to baoyu-slide-deck +│ ├── --prompts-only → outline + prompts +│ └── --images-only → images + PPTX + PDF +└── Phase 6: Post-processing ← Directory reorg + speaker notes extraction +``` + +**Rule**: If `baoyu-slide-deck` can do it, we call it. We only do what baoyu-slide-deck cannot: narrative discussion, ABCDEFG methodology, and user's preferred directory structure. + +--- + +## Phase 0: Source Material Collection + +**CRITICAL**: Do NOT proceed to Phase 1 until user source materials are collected. + +**Goal**: Gather the user's own words and their approved external references. + +### Step 0.1: Request User's Original Content + +Ask the user for: +- **Transcripts** of their past talks, meetings, or discussions +- **Articles** they have written or approved +- **Notes** or drafts they have prepared +- **Previous decks** they have delivered +- **Voice memos** or any recorded thoughts + +If none exist, proceed to Phase 1 with the understanding that the entire narrative must be extracted from the user's head through structured conversation. + +### Step 0.2: Gather External References (Optional) + +- Search for relevant external materials (articles, reports, references) +- **Present findings to the user for selection** — do not assume relevance +- Only include materials the user explicitly approves + +### Step 0.3: Organize Source Materials + +Save all source materials to `00-上游/`（根目录或 `source-materials/` 子目录均可）： +``` +00-上游/ +├── prompt-最初提示词.txt # 用户原始 prompt（如有） +├── narrative-brief.md # Phase 1 输出 +├── content.md # Phase 2 输出（baoyu 输入） +├── style-instructions.md # 视觉设计 SSOT +├── outline.md # 来自 baoyu-slide-deck +├── source-materials/ # （可选子目录） +│ ├── user-transcript-1.md +│ ├── user-article-2.md +│ ├── user-notes-3.md +│ └── external-ref-4.md (user-approved) +``` + +**Note**: 对于已有项目，源文件可直接放在 `00-上游/` 根目录；新建议项目时可用 `source-materials/` 子目录保持整洁。 + +**Self-check**: Do we have the user's own words? If not, are we prepared to extract everything through conversation? Do NOT invent content. + +--- + +## Phase 1: Narrative Structure Discussion + +**CRITICAL**: Do NOT generate any files in this phase. Only discuss. + +**Goal**: Align on the narrative arc, emotional journey, and slide-level logic before any visual work begins. + +**Principle**: "你不要直接去写，你应该跟我讨论" + +**Input**: Phase 0 source materials. Every insight in this discussion must be grounded in the user's own words or their explicitly approved references. + +### Discussion Framework (ABCDEFG Model) + +| Step | Question | Purpose | +|------|----------|---------| +| A | **Attention** | How do we hook in the first 30 seconds? | +| B | **Benefit** | What's the promised takeaway? | +| C | **Credibility** | Why should the audience trust us? | +| D | **Difference** | What's the contrarian or novel angle? | +| E | **Evidence** | What proof, demo, or story backs this? | +| F | **Framework** | What mental model do we leave them with? | +| G | **Go** | What should they do Monday morning? | + +### Required Inputs + +Ask user if missing: +- **Topic**: What's the talk about? (1 sentence) +- **Audience**: Who's listening? (technical level, role, context) +- **Duration**: How long is the talk? +- **Key messages**: What must they remember? (3 max) +- **Tone**: Educational? Persuasive? Provocative? Inspirational? +- **Existing content**: Articles, transcripts, notes, previous decks? +- **Constraints**: Must-use content? Avoid topics? Brand guidelines? + +### Discussion Checklist + +1. **Opening strategy**: Shock? Story? Question? Demo? +2. **The arc**: Where's the tension? Where's the release? +3. **Transition logic**: How does each slide lead to the next? +4. **The "one thing"**: If they forget everything, what's the ONE thing? +5. **Call to action**: What do they do after the talk? + +### Anti-patterns to Flag + +- ❌ Too many slides for the time (crowded, rushed) +- ❌ Jumping into details without setting context +- ❌ No emotional arc (flat, forgettable) +- ❌ Ending without a clear takeaway +- ❌ Trying to teach too many things at once + +### Validation + +Summarize agreed narrative arc in 3-5 bullet points. Get explicit user confirmation before proceeding to Phase 2. + +**Self-check**: Did we discuss? Or did we jump to generation? If the latter, go back. + +--- + +## Phase 2: Content Structuring + +**Goal**: Produce two SSOT files that baoyu-slide-deck can consume. + +### 2.1 Create `narrative-brief.md` + +Store in `00-上游/`: + +```markdown +# Narrative Brief + +**Topic**: [Topic name] +**Audience**: [description] +**Duration**: [N min] +**Language**: [zh/en/etc] +**Tone**: [educational/persuasive/provocative/inspirational] +**Key Messages**: (3 max) +1. ... +2. ... +3. ... + +## ABCDEFG Arc + +| Step | Answer | +|------|--------| +| A - Attention | ... | +| B - Benefit | ... | +| C - Credibility | ... | +| D - Difference | ... | +| E - Evidence | ... | +| F - Framework | ... | +| G - Go | ... | + +## Slide Count Recommendation + +| Duration | Recommended | Max | +|----------|-------------|-----| +| 10-15 min | 8-12 | 12 | +| 20-30 min | 12-18 | 20 | +| 30-45 min | 15-25 | 28 | +| 45-60 min | 20-30 | 35 | + +**Recommended**: [N] slides for [duration] talk + +## Content Sources + +- [ ] Original user prompt saved +- [ ] Existing articles/notes/transcripts +- [ ] Previous decks + +## Style Direction + +[User's style preference or "to be decided in Phase 3"] +``` + +### 2.2 Create `content.md` (for baoyu-slide-deck) + +Convert narrative brief into baoyu-slide-deck input format: + +```markdown +# [Title] + +## Overview + +[2-3 paragraph summary of the talk content] + +## Key Points + +1. [Point 1] +2. [Point 2] +3. [Point 3] + +## Structure + +### Opening ([duration]) +[Hook content] + +### Section 1: [Name] ([duration]) +[Content] + +### Section 2: [Name] ([duration]) +[Content] + +### Closing ([duration]) +[CTA content] + +## Audience +[Same as narrative-brief] + +## Notes +[Any constraints or special requirements] +``` + +### 2.3 Create `style-instructions.md` (Optional) + +If user has strong style preferences, create this SSOT file in `00-上游/`: + +```markdown + +Design Aesthetic: [Description] + +Background: + Texture: [clean/grid/organic/etc] + Base Color: [#HEX] + +Typography: + Headlines: [Style, size, color, weight] + Body: [Style, size, color, weight] + +Color Palette: + Primary Text: [#HEX] - usage + Body Text: [#HEX] - usage + Background: [#HEX] + Accent 1: [#HEX] - usage + Accent 2: [#HEX] - usage + Accent 3: [#HEX] - usage + +Visual Elements: + - [Element 1] + - [Element 2] + +Density Guidelines: + - Max [N] text elements per slide + - [Other rules] + +Style Rules: + Do: [List] + Don't: [List] + +``` + +**Self-check**: Read narrative-brief.md back to user. Confirm it matches the Phase 1 discussion before proceeding. + +**Content Integrity Check**: Every claim, quote, and example in `content.md` must be traceable to: +1. User's own words (from Phase 0 source materials) +2. User-approved external references +3. User's explicit statements during Phase 1 discussion + +AI must NOT invent facts, quotes, or examples. If the user said it, use it. If they didn't, ask them. If they don't have it, mark it as `[TODO: user to provide]`. + +--- + +## Phase 3: Delegate to baoyu-slide-deck (Prompts) + +**Goal**: Generate outline and prompts using baoyu-slide-deck. + +### Step 3.1: Prepare Input + +Ensure `content.md` is ready. If `style-instructions.md` exists, note the style preference for passing to baoyu-slide-deck. + +### Step 3.2: Call baoyu-slide-deck + +在 Claude Code 中调用 baoyu-slide-deck skill（两种等效方式）： + +``` +/baoyu-slide-deck 00-上游/content.md --prompts-only [--style ] +``` + +或直接使用 Skill 工具（当 `/` 命令不可用时）： +``` +Skill({"skill": "baoyu-slide-deck", "args": "00-上游/content.md --prompts-only [--style ]"}) +``` + +**Pre-call setup**: +1. **Inject narrative-brief**: Append `narrative-brief.md` (or its ABCDEFG arc section) to the top of `content.md` so baoyu receives the narrative structure, not just raw content. +2. **Inject confirmed choices**: Prepend a metadata block to `content.md` with the already-confirmed choices. This acts as a strong signal for baoyu's auto-detect and reduces the chance of style drift during confirmation: + ```markdown + + - Style: [preset name or custom] + - Audience: [audience from Phase 1] + - Slide count: [N slides for X-min talk] + - Language: [zh/en/etc] + - Review preference: [skip outline / skip prompts / none] + ``` +3. **Style selection**: If user specified a baoyu preset → use it; if custom `style-instructions.md` exists → use `--style custom`; otherwise auto-detect. + +**⚠️ Confirmation overlap warning**: baoyu-slide-deck Step 2 will ask the user to confirm style, audience, slide count, outline review, and prompt review. Since these were already discussed in Phase 1, **instruct the user to stick with the choices we just made** rather than reconsidering. This prevents confirmation fatigue and style drift. + +| User's Description | baoyu Preset | +|-------------------|--------------| +| Flat cartoon, tech explainer | `vector-illustration` or `bold-editorial` | +| Hand-drawn edu, infographic, process | `hand-drawn-edu` | +| Chalkboard, workshop | `chalkboard` or `sketch-notes` | +| Corporate, B2B, investor deck | `corporate` or `minimal` | +| Editorial, magazine, product launch | `bold-editorial` | +| Journalism, explainer, science communication | `editorial-infographic` | +| Dark, gaming, atmospheric | `dark-atmospheric` | +| Retro, pixel, developer talk | `pixel-art` | +| Watercolor, lifestyle, travel | `watercolor` | +| Blueprint, technical, architecture | `blueprint` | +| Academic, research, bilingual | `intuition-machine` | +| Notion, SaaS, product demo | `notion` | +| Story, fantasy, animation | `fantasy-animation` | +| Biology, chemistry, medical | `scientific` | +| History, heritage, vintage | `vintage` | + +### Step 3.3: Post-process Prompts + +After baoyu-slide-deck generates prompts in `prompts/`: + +1. **Copy to user's structure**: Move/copy prompts to `03-prompts/` +2. **Inject custom style**: If `style-instructions.md` exists, ensure FULL content is embedded in every prompt file +3. **Add narrative goal**: Append `// NARRATIVE GOAL` section to each prompt based on `narrative-brief.md` + +Prompt template addition: +```markdown +--- + +## NARRATIVE CONTEXT + +// NARRATIVE GOAL +[What this slide achieves in the talk arc] + +// SPEAKER NOTES +[What the speaker says while this slide is shown] +``` + +**Self-check**: All prompts include full style-instructions.md? Narrative goals added? Files in `03-prompts/`? + +--- + +## Phase 4: Prompt Review (Conditional) + +**Goal**: Human review before image generation. + +**If user wants to review** (recommended): +1. Display prompt summary table +2. Ask user: "Ready to generate images?" +3. If edits needed → user edits `03-prompts/*.md` → regenerate specific prompts via baoyu-slide-deck + +**If user skips review**: Proceed to Phase 5. + +--- + +## Phase 5: Delegate to baoyu-slide-deck (Images) + +**Goal**: Generate slide images. + +### Step 5.1: Call baoyu-slide-deck + +``` +/baoyu-slide-deck . --images-only +``` + +Or using Skill tool: +``` +Skill({"skill": "baoyu-slide-deck", "args": ". --images-only"}) +``` + +**Important**: baoyu-slide-deck expects prompts in a flat `prompts/` directory (its native output structure). If you have already reorganized to `03-prompts/` (Phase 6), create a temporary copy before calling: +```bash +# From the project root (where 03-prompts/ exists) +cp -r 03-prompts prompts +/baoyu-slide-deck . --images-only +rm -rf prompts # clean up after +``` + +**Note on deliverables**: baoyu-slide-deck internally generates `.pptx` and `.pdf` via its own `merge-to-pptx.ts` and `merge-to-pdf.ts` scripts (Step 8 of its workflow). These are the **primary** deliverables — they live in baoyu's flat output directory (`slide-deck/{topic-slug}/`). + +The `scripts/merge_to_pptx.py` and `scripts/merge_to_pdf.py` in slides-creator are **not** duplicates. They serve a different purpose: +- **baoyu merge**: for baoyu's flat directory structure (PNG + prompts at same level) +- **slides-creator merge**: for the reorganized directory structure (`02-slides/` + `03-prompts/`) + +Use slides-creator's merge scripts only after Phase 6 reorganization, or if baoyu's merge step fails. + +### Step 5.2: Visual Verification + +After generation: +1. **Test slide**: Read `01-slide-cover.png` (or first generated slide) +2. **Style check**: Compare against `style-instructions.md` +3. **Text check**: Verify Chinese/English text legibility +4. **If issues**: Update affected `03-prompts/*.md` → copy `cp -r 03-prompts prompts` → regenerate via `/baoyu-slide-deck . --regenerate N` (或 `Skill({"skill": "baoyu-slide-deck", "args": ". --regenerate N"})`) → clean up `rm -rf prompts` + +**Self-check**: All slides generated? Style consistent? Text legible? + +--- + +## Phase 6: Post-processing & Delivery + +**Goal**: Reorganize baoyu-slide-deck output into user's preferred structure. + +### 6.1 Directory Reorganization + +baoyu-slide-deck outputs to `slide-deck/{topic-slug}/`: +``` +slide-deck/{topic-slug}/ +├── source-{slug}.md +├── outline.md +├── prompts/ +├── *.png +├── {topic-slug}.pptx +└── {topic-slug}.pdf +``` + +Reorganize to user's structure: +``` +{project-name}/ +├── 00-上游/ # Source materials +│ ├── prompt-最初提示词.txt # Original user prompt (if saved) +│ ├── narrative-brief.md # Phase 1 output +│ ├── content.md # Phase 2 output (baoyu input) +│ ├── style-instructions.md # Visual design SSOT +│ └── outline.md # From baoyu-slide-deck +├── 01-成品/ # Final deliverables +│ ├── {project-name}.pdf +│ └── {project-name}.pptx +├── 02-slides/ # Generated PNGs (当前版本) +│ ├── 01-slide-cover.png +│ └── ... +├── 03-prompts/ # Per-slide prompts (SSOT) +│ ├── v6/ # 支持版本子目录（如 v6, v7...） +│ │ ├── 01-slide-cover.md +│ │ └── ... +│ └── 01-slide-cover.md # 或平铺结构 +├── speaker-notes.md # Auto-extracted from 03-prompts/ via extract_notes.py +├── v6/ # baoyu-slide-deck 临时输出（需搬迁到 02-slides/） +│ └── ... +└── _archive/ # Historical versions + └── v1/ +``` + +**Note on versioning**: +- `03-prompts/` 支持平铺或版本子目录（`v6/`, `v7/`）。当同一项目多次迭代时，用子目录保留历史版本。 +- baoyu-slide-deck 可能直接输出到项目根目录的临时文件夹（如 `v6/`）。Post-processing 时需将这些 PNG 移动到 `02-slides/`。 + +**Archive current version** (before major iteration): +```bash +uv run scripts/archive_version.py --project /path/to/project +``` +Archives `02-slides/` + `03-prompts/` to `_archive/v{N}/` (auto-incremented). + +### 6.2 Extract Speaker Notes + +Use `scripts/extract_notes.py` to extract structured notes from `03-prompts/*.md`: + +```bash +uv run scripts/extract_notes.py --prompts 03-prompts --output speaker-notes.md +``` + +Extracts: +- `// NARRATIVE GOAL` sections +- `// SPEAKER NOTES` sections +- Falls back to `// KEY CONTENT` if neither found + +Output format (`speaker-notes.md`): +```markdown +# Speaker Notes + +## 01-slide-cover +**Narrative Goal**: ... +**Speaker Notes**: ... + +## 02-slide-intro +... +``` + +**Note**: `main.ts` auto-runs this step if `03-prompts/` exists. + +### 6.3 Archive Original Prompt + +If user provided an original prompt (like the 35KB prompt for 龙虾 vs Claude Code): +- Save as `00-上游/prompt-最初提示词.txt` + +### 6.4 Final Verification Checklist + +- [ ] PDF opens and all slides render correctly +- [ ] PPTX opens without errors +- [ ] PNG sequence numbered correctly (01, 02, ...) +- [ ] Speaker notes cover all slides +- [ ] Style consistent across all slides +- [ ] No garbled or missing text +- [ ] `00-上游/` contains all source SSOT files +- [ ] `03-prompts/` contains all prompt files + +--- + +## Iteration Workflow + +### Path A: Content Changes +``` +User feedback → Update narrative-brief.md → Update content.md +→ Regenerate prompts (/baoyu-slide-deck content.md --prompts-only) +→ Regenerate images (/baoyu-slide-deck . --images-only) +→ Reorganize + extract notes +``` + +### Path B: Style Changes +``` +User feedback → Update style-instructions.md +→ Update all prompts (inject new style into 03-prompts/*.md) +→ Regenerate all images (via /baoyu-slide-deck . --images-only) +→ Reorganize + extract notes +``` + +### Path C: Single Slide Fix +``` +User feedback → Update 03-prompts/NN-slide-xxx.md +→ Copy prompts: cp -r 03-prompts prompts +→ /baoyu-slide-deck . --regenerate N +→ Clean up: rm -rf prompts +→ Replace in 02-slides/ +→ Regenerate PPTX/PDF +``` + +**Note on `--regenerate`**: baoyu-slide-deck reads from flat `prompts/` directory. After reorganization to `03-prompts/`, create a temporary copy (`cp -r 03-prompts prompts`) before regenerating. If you haven't reorganized yet (still in baoyu's flat structure), call directly without copying. + +--- + +## Script Reference + +| Script | Purpose | +|--------|---------| +| `scripts/main.ts` | Post-processing: validate + extract notes + generate PDF/PPTX | +| `scripts/merge_to_pptx.py` | Merge PNGs to PPTX with structured speaker notes from `03-prompts/*.md` | +| `scripts/merge_to_pdf.py` | Merge PNGs to PDF (reorganized `02-slides/` structure) | +| `scripts/validate_slides.py` | Check aspect ratio, naming, missing slides | +| `scripts/extract_notes.py` | Extract structured speaker notes from `03-prompts/*.md` to `speaker-notes.md` | +| `scripts/archive_version.py` | Archive `02-slides/` + `03-prompts/` to `_archive/v{N}/` | + +--- + +## Failure Log (Do NOT Repeat) + +| Failure | Root Cause | Prevention | +|---------|-----------|------------| +| **AI wrote content for the user — polished garbage** | Violated First Law: skipped Phase 0, fabricated quotes/examples. See `references/content-creation-first-law.md` | **First Law is absolute**: collect user's words FIRST. No source material = STOP and ask. AI assists expression, never replaces it | +| Generated 30 slides for 20-min talk | Didn't enforce slide count guide | Check duration ÷ 2 = max slides | +| Style drift between versions | Style instructions not in prompts | Paste FULL style-instructions.md into every prompt | +| Text unreadable in images | Model doesn't support Chinese well | Test with Chinese text first | +| Narrative arc flat | Jumped to prompts without Phase 1 | ALWAYS discuss narrative first | +| User unhappy with first draft | Didn't confirm before batch | Generate ONE test slide, get approval | +| Directory mess | Didn't use consistent structure | Always use 00-上游/01-成品/02-slides/03-prompts | +| Redundant work | Tried to replace baoyu-slide-deck | Delegate visual generation, focus on narrative | +| Merge scripts questioned as duplicates | Baoyu merge scripts exist but: (1) path unstable (`~/.claude/plugins/...`); (2) expect flat `prompts/` dir; (3) inject full base prompt as notes noise | Keep own merge scripts for reorganized structure. Validate before deleting — call baoyu merge first if accessible | + +--- + +## References + +- `references/content-creation-first-law.md` — Universal principle: user's voice is primary, applies to all content types (slides, articles, ads, courses) +- `references/narrative-design-guide.md` — ABCDEFG model detailed guide +- `references/prompt-templates/` — Prompt templates for common slide types +- `references/style-gallery.md` — Visual style gallery with examples diff --git a/slides-creator/references/content-creation-first-law.md b/slides-creator/references/content-creation-first-law.md new file mode 100644 index 00000000..62aedf65 --- /dev/null +++ b/slides-creator/references/content-creation-first-law.md @@ -0,0 +1,116 @@ +# Content Creation First Law + +## The Principle + +**AI cannot write high-quality content for the user. It can only help the user express their own content better.** + +> The most important step in creating any piece of content — whether a slide deck, an article, an advertisement, or a course — is **collecting the user's original words first**. +> +> Without the user's source material, AI produces polished, plausible, unusable garbage. It may sound impressive, but it won't sound like the user. It won't reflect their expertise. It won't be something they can confidently deliver or stand behind. + +## Why This Matters + +| Approach | Result | +|----------|--------| +| AI writes alone | Generic, AI-tinged content that the user cannot authentically deliver | +| AI + user's raw material | Content that sounds like the user at their best — authentic, specific, powerful | + +The goal is not to have AI do what the user cannot do. The goal is to **reduce the user's workload** while **preserving and elevating their voice**. + +## The Weight Hierarchy + +All content sources are NOT equal. Use this priority order: + +``` +1. User's own words (highest authority) + - Transcripts of their talks + - Articles they wrote + - Notes they prepared + - Voice memos or recordings + - Their explicit statements during discussion + +2. User-approved external material + - Articles they selected + - References they endorsed + - Research they reviewed + +3. AI synthesis (requires user validation) + - AI-organized structure + - AI-suggested transitions + - AI-generated visual descriptions + +4. AI invention (lowest authority, use sparingly) + - AI-fabricated examples + - AI-generated quotes + - AI-invented statistics + → MUST be labeled as draft, MUST be validated by user +``` + +## The Process + +### Step 1: Collect + +Before creating ANY content, collect from the user: + +- Past transcripts (meetings, talks, interviews) +- Existing writings (articles, drafts, notes) +- Previous decks or materials +- Voice memos or recorded thoughts +- Key messages they want to convey + +If no written material exists, the entire narrative must be extracted through structured conversation. **Do not fabricate.** + +### Step 2: Validate + +For external materials: +- Present findings to the user +- Let them select what is relevant +- Only include user-approved references + +### Step 3: Assist + +With user source material as the foundation: +- Organize and structure their ideas +- Suggest narrative arcs and transitions +- Design visuals that amplify their message +- Polish expression without changing substance + +### Step 4: Verify + +Final content must pass: +- [ ] Could the user actually say this in their own words? +- [ ] Does it reflect their expertise and perspective? +- [ ] Are all claims traceable to user's words or approved sources? +- [ ] Does it sound like them, not like generic AI output? + +## Failure Modes + +| Failure | Symptom | Prevention | +|---------|---------|------------| +| AI hallucinated content | User says "This doesn't sound like me" or "I never said that" | Always ground content in Phase 0 source materials | +| AI invented examples | Examples feel generic or disconnected from user's experience | Ask user for real examples; mark gaps as [TODO] | +| AI-generated quotes | Quotes don't match user's speaking style | Use verbatim from transcripts; don't fabricate | +| AI chose references | User disagrees with included sources | Present options; user selects | +| Content is "AI-flavored" | Long sentences, hedging language, "delve into", "leverage" | Rewrite in user's voice; use their sentence patterns | + +## Application to Different Content Types + +| Content Type | User Source Material | AI's Role | +|--------------|---------------------|-----------| +| **Slide deck** | Transcripts, talk notes, previous decks | Structure narrative, design visuals, synthesize | +| **Article** | Draft notes, voice memos, talking points | Organize structure, improve flow, polish language | +| **Advertisement** | Product description, user testimonials, brand voice guide | Write copy that matches brand voice, suggest hooks | +| **Course** | Teaching notes, workshop transcripts, exercise ideas | Design curriculum, create pacing, build assessments | +| **Email/WeChat** | Their usual writing style, specific message intent | Draft in their voice, suggest phrasing | + +## Corollaries + +1. **If the user has no existing content**, your job is to help them articulate their thoughts through structured conversation — NOT to fabricate content for them. + +2. **External materials must be user-selected**. AI does not decide what is relevant. The user does. + +3. **AI synthesis must be user-validated**. AI can organize and connect, but the user must approve the connections. + +4. **AI invention must be labeled**. Anything AI creates that is not grounded in user source material must be clearly marked as draft and requires user approval. + +5. **The output must be speakable/deliverable**. If the user cannot confidently say it in their own voice, it is not done. diff --git a/slides-creator/references/narrative-design-guide.md b/slides-creator/references/narrative-design-guide.md new file mode 100644 index 00000000..664c72a0 --- /dev/null +++ b/slides-creator/references/narrative-design-guide.md @@ -0,0 +1,108 @@ +# Narrative Design Guide (ABCDEFG Model) + +## A - Attention (Hook) + +**Goal**: Capture attention in the first 30 seconds. + +**Techniques**: +- **Contrarian statement**: "Everything you know about X is wrong" +- **Surprising statistic**: "90% of teams fail at this" +- **Personal story**: "Three years ago, I made a $2M mistake..." +- **Demo/Visual**: Show, don't tell +- **Question**: "What if I told you..." + +**Slide design**: Big, bold, minimal text. One visual element that demands attention. + +## B - Benefit (Promise) + +**Goal**: Let the audience know what's in it for them. + +**Techniques**: +- Clear statement of takeaway +- Time-bound promise ("In 30 minutes, you'll learn...") +- Audience-specific benefit ("As engineers, this means...") + +**Slide design**: Headline + 3 bullet points of promised outcomes. + +## C - Credibility (Why Trust You?) + +**Goal**: Establish authority without boasting. + +**Techniques**: +- Relevant experience (not laundry list) +- Social proof (logos, numbers) +- Vulnerability ("I failed at this 3 times before getting it right") + +**Slide design**: Logos, numbers, or a brief bio. Keep it humble. + +## D - Difference (Novel Angle) + +**Goal**: Show what's different about YOUR perspective. + +**Techniques**: +- Reframe common wisdom +- Expose hidden assumptions +- Bridge two unlikely domains + +**Slide design**: Comparison (old way vs new way), or metaphor illustration. + +## E - Evidence (Proof) + +**Goal**: Back up claims with data, demos, or stories. + +**Techniques**: +- Case studies +- Live demos +- Data visualizations +- Customer quotes + +**Slide design**: Charts, screenshots, or story panels. + +## F - Framework (Mental Model) + +**Goal**: Give the audience a reusable mental model. + +**Techniques**: +- 3-part framework +- Matrix/quadrant +- Process flow +- Mnemonic + +**Slide design**: Diagram, flowchart, or structured list. + +## G - Go (Call to Action) + +**Goal**: Tell them exactly what to do next. + +**Techniques**: +- Specific next step (not "think about it") +- Low friction ("Scan this QR code") +- Time-bound ("Join before Friday") + +**Slide design**: Clear CTA, contact info, or QR code. + +--- + +## Narrative Arc Patterns + +### Pattern 1: Problem → Solution → Proof +Best for: Product pitches, tool introductions + +### Pattern 2: What → So What → Now What +Best for: Educational talks, research presentations + +### Pattern 3: Story → Lesson → Application +Best for: Keynotes, inspirational talks + +### Pattern 4: Myth → Truth → Practice +Best for: Debunking talks, contrarian content + +--- + +## Common Pitfalls + +1. **The Wikipedia talk**: Listing facts without a through-line +2. **The feature dump**: Showing everything instead of the best thing +3. **The no-arc**: Flat energy from start to finish +4. **The surprise ending**: No setup for the conclusion +5. **The too-many-things**: Trying to teach 5 topics in 20 minutes diff --git a/slides-creator/references/prompt-templates/quote-focus.md b/slides-creator/references/prompt-templates/quote-focus.md new file mode 100644 index 00000000..dff96a34 --- /dev/null +++ b/slides-creator/references/prompt-templates/quote-focus.md @@ -0,0 +1,53 @@ +Create a presentation slide image following these guidelines: + +## Image Specifications + +- **Type**: Presentation slide +- **Aspect Ratio**: 16:9 (landscape) +- **Style**: {{style_name}} + +## Core Principles + +- Flat cartoon style throughout - NO realistic or photographic elements +- NO slide numbers, page numbers, footers, headers, or logos +- Clean, uncluttered layouts with clear visual hierarchy +- Each slide conveys ONE clear message + +## Text Style (CRITICAL) + +- **ALL text MUST match the designated style exactly** +- Title text: Large, bold, immediately readable +- Body text: Clear, legible, appropriate sizing +- Max 3-4 text elements per slide + +## Language + +- All text in {{language}} +- Write in direct, confident language + +--- + + +{{style_instructions}} + + +--- + +## SLIDE CONTENT + +**Slide {{slide_number}} of {{total_slides}} — Quote Focus** +**Filename**: {{slide_number}}-slide-{{name}}.png + +// NARRATIVE GOAL +{{narrative_goal}} + +// KEY CONTENT +Headline: {{headline}} +Quote: {{quote_text}} +Attribution: {{attribution}} (optional) + +// VISUAL +{{visual_description}} + +// LAYOUT +Layout: quote-focus. Headline at top. Large centered quote text (largest text on slide). Attribution below quote in smaller text. Optional decorative quotation mark graphic. Clean, minimal, high impact. diff --git a/slides-creator/references/prompt-templates/split-comparison.md b/slides-creator/references/prompt-templates/split-comparison.md new file mode 100644 index 00000000..99a10cde --- /dev/null +++ b/slides-creator/references/prompt-templates/split-comparison.md @@ -0,0 +1,53 @@ +Create a presentation slide image following these guidelines: + +## Image Specifications + +- **Type**: Presentation slide +- **Aspect Ratio**: 16:9 (landscape) +- **Style**: {{style_name}} + +## Core Principles + +- Flat cartoon style throughout - NO realistic or photographic elements +- NO slide numbers, page numbers, footers, headers, or logos +- Clean, uncluttered layouts with clear visual hierarchy +- Each slide conveys ONE clear message + +## Text Style (CRITICAL) + +- **ALL text MUST match the designated style exactly** +- Title text: Large, bold, immediately readable +- Body text: Clear, legible, appropriate sizing +- Max 3-4 text elements per slide + +## Language + +- All text in {{language}} +- Write in direct, confident language + +--- + + +{{style_instructions}} + + +--- + +## SLIDE CONTENT + +**Slide {{slide_number}} of {{total_slides}} — Comparison** +**Filename**: {{slide_number}}-slide-{{name}}.png + +// NARRATIVE GOAL +{{narrative_goal}} + +// KEY CONTENT +Headline: {{headline}} +Left Label: {{left_label}} +Right Label: {{right_label}} + +// VISUAL +{{visual_description}} + +// LAYOUT +Layout: split-comparison. Headline at top. Two columns side by side at center (50/50 split). Clear visual divider between sides. Labels below each column. Optional summary banner at bottom 20%. diff --git a/slides-creator/references/prompt-templates/three-column.md b/slides-creator/references/prompt-templates/three-column.md new file mode 100644 index 00000000..1e8edf95 --- /dev/null +++ b/slides-creator/references/prompt-templates/three-column.md @@ -0,0 +1,54 @@ +Create a presentation slide image following these guidelines: + +## Image Specifications + +- **Type**: Presentation slide +- **Aspect Ratio**: 16:9 (landscape) +- **Style**: {{style_name}} + +## Core Principles + +- Flat cartoon style throughout - NO realistic or photographic elements +- NO slide numbers, page numbers, footers, headers, or logos +- Clean, uncluttered layouts with clear visual hierarchy +- Each slide conveys ONE clear message + +## Text Style (CRITICAL) + +- **ALL text MUST match the designated style exactly** +- Title text: Large, bold, immediately readable +- Body text: Clear, legible, appropriate sizing +- Max 3-4 text elements per slide + +## Language + +- All text in {{language}} +- Write in direct, confident language + +--- + + +{{style_instructions}} + + +--- + +## SLIDE CONTENT + +**Slide {{slide_number}} of {{total_slides}} — Three Columns** +**Filename**: {{slide_number}}-slide-{{name}}.png + +// NARRATIVE GOAL +{{narrative_goal}} + +// KEY CONTENT +Headline: {{headline}} +Column 1: {{column_1_title}} - {{column_1_content}} +Column 2: {{column_2_title}} - {{column_2_content}} +Column 3: {{column_3_title}} - {{column_3_content}} + +// VISUAL +{{visual_description}} + +// LAYOUT +Layout: three-column. Headline at top. Three equal columns (33/33/33) side by side. Each column has a distinct accent color. Generous spacing between columns. Optional connecting element (arrow, flow) between columns. diff --git a/slides-creator/references/prompt-templates/title-hero.md b/slides-creator/references/prompt-templates/title-hero.md new file mode 100644 index 00000000..b0fd9e41 --- /dev/null +++ b/slides-creator/references/prompt-templates/title-hero.md @@ -0,0 +1,52 @@ +Create a presentation slide image following these guidelines: + +## Image Specifications + +- **Type**: Presentation slide +- **Aspect Ratio**: 16:9 (landscape) +- **Style**: {{style_name}} + +## Core Principles + +- Flat cartoon style throughout - NO realistic or photographic elements +- NO slide numbers, page numbers, footers, headers, or logos +- Clean, uncluttered layouts with clear visual hierarchy +- Each slide conveys ONE clear message + +## Text Style (CRITICAL) + +- **ALL text MUST match the designated style exactly** +- Title text: Large, bold, immediately readable +- Body text: Clear, legible, appropriate sizing +- Max 3-4 text elements per slide + +## Language + +- All text in {{language}} +- Write in direct, confident language + +--- + + +{{style_instructions}} + + +--- + +## SLIDE CONTENT + +**Slide {{slide_number}} of {{total_slides}} — Cover** +**Filename**: {{slide_number}}-slide-cover.png + +// NARRATIVE GOAL +{{narrative_goal}} + +// KEY CONTENT +Headline: {{headline}} +Sub-headline: {{sub_headline}} + +// VISUAL +{{visual_description}} + +// LAYOUT +Layout: title-hero. Headline centered at top. Sub-headline below. Hero visual composition centered in the lower 60% of the slide. diff --git a/slides-creator/references/style-gallery.md b/slides-creator/references/style-gallery.md new file mode 100644 index 00000000..48e9e310 --- /dev/null +++ b/slides-creator/references/style-gallery.md @@ -0,0 +1,65 @@ +# Style Gallery + +## flat-cartoon-infographic + +**Best for**: Tech explainers, approachable content, WeChat-style infographics + +**Tokens**: +- White background (#FFFFFF) +- Flat cartoon illustrations (simple rounded shapes) +- Colorful pill/tag shapes for lists +- Bold color contrasts +- 2-3 key points per slide + +**Example**: "龙虾 vs Claude Code" deck + +## corporate-clean + +**Best for**: Investor decks, B2B sales, executive briefings + +**Tokens**: +- Minimal design +- Geometric shapes +- Muted palette (blues, grays) +- Lots of whitespace +- Data-forward + +## chalkboard-edu + +**Best for**: Tutorials, workshops, educational content + +**Tokens**: +- Dark background (blackboard) +- Handwritten-style typography +- Warm tones (yellows, oranges) +- Sketch/doodle aesthetic + +## bold-editorial + +**Best for**: Keynotes, product launches, creative presentations + +**Tokens**: +- High contrast +- Editorial typography (serif headlines) +- Full-bleed images +- Magazine-like layouts + +## scientific + +**Best for**: Research presentations, academic talks, technical deep-dives + +**Tokens**: +- Clean, structured layouts +- Cool color palette (blues, teals) +- Data visualizations +- Citation-ready + +## notion + +**Best for**: Product demos, SaaS presentations, tool tutorials + +**Tokens**: +- Clean, neutral colors +- Geometric icons +- Dense but organized information +- Modern SaaS aesthetic diff --git a/slides-creator/scripts/archive_version.py b/slides-creator/scripts/archive_version.py new file mode 100755 index 00000000..dc41245c --- /dev/null +++ b/slides-creator/scripts/archive_version.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +"""Archive current slide version to _archive/v{N}/.""" + +import argparse +import shutil +from pathlib import Path + + +def get_next_version(archive_dir: Path) -> int: + """Find the next version number (v1, v2, ...).""" + if not archive_dir.exists(): + return 1 + versions = [] + for d in archive_dir.iterdir(): + if d.is_dir() and d.name.startswith("v"): + try: + versions.append(int(d.name[1:])) + except ValueError: + pass + return max(versions, default=0) + 1 + + +def archive_version(project_dir: str): + """Archive 02-slides/ and 03-prompts/ to _archive/v{N}/.""" + project_path = Path(project_dir) + slides_dir = project_path / "02-slides" + prompts_dir = project_path / "03-prompts" + + if not slides_dir.exists(): + print(f"No slides to archive: {slides_dir}") + return False + + archive_dir = project_path / "_archive" + next_ver = get_next_version(archive_dir) + target_dir = archive_dir / f"v{next_ver}" + + print(f"Archiving to: {target_dir}") + target_dir.mkdir(parents=True, exist_ok=False) + + # Copy slides + target_slides = target_dir / "02-slides" + shutil.copytree(slides_dir, target_slides) + print(f" Copied slides: {len(list(target_slides.glob('*.png')))} files") + + # Copy prompts if exist + if prompts_dir.exists(): + target_prompts = target_dir / "03-prompts" + shutil.copytree(prompts_dir, target_prompts) + prompt_count = sum(1 for _ in target_prompts.rglob("*.md")) + print(f" Copied prompts: {prompt_count} files") + + print(f"Archive v{next_ver} complete.") + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Archive slide version") + parser.add_argument("--project", required=True, help="Project directory") + args = parser.parse_args() + + archive_version(args.project) diff --git a/slides-creator/scripts/extract_notes.py b/slides-creator/scripts/extract_notes.py new file mode 100755 index 00000000..92500ec2 --- /dev/null +++ b/slides-creator/scripts/extract_notes.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +"""Extract structured speaker notes from slide prompt files.""" + +import argparse +import re +from pathlib import Path + + +def find_prompt_file(slide_name: str, prompts_dir: Path) -> Path | None: + """Find matching prompt .md for a slide name.""" + # Try flat structure first: 03-prompts/01-slide-cover.md + flat = prompts_dir / f"{slide_name}.md" + if flat.exists(): + return flat + + # Try versioned subdirectories: 03-prompts/v6/01-slide-cover.md + for subdir in prompts_dir.iterdir(): + if subdir.is_dir(): + candidate = subdir / f"{slide_name}.md" + if candidate.exists(): + return candidate + + return None + + +def extract_notes(prompt_text: str) -> dict: + """Extract NARRATIVE GOAL and SPEAKER NOTES sections from prompt.""" + result = {} + + ng_match = re.search( + r"// NARRATIVE GOAL\s*\n(.+?)(?=\n// |\n## |\Z)", + prompt_text, + re.DOTALL, + ) + if ng_match: + result["narrative_goal"] = ng_match.group(1).strip() + + sn_match = re.search( + r"// SPEAKER NOTES\s*\n(.+?)(?=\n// |\n## |\Z)", + prompt_text, + re.DOTALL, + ) + if sn_match: + result["speaker_notes"] = sn_match.group(1).strip() + + # Fallback: KEY CONTENT + if not result: + kc_match = re.search( + r"// KEY CONTENT\s*\n(.+?)(?=\n// |\n## |\Z)", + prompt_text, + re.DOTALL, + ) + if kc_match: + result["key_content"] = kc_match.group(1).strip() + + return result + + +def extract_all_notes(prompts_dir: str, output: str | None = None) -> str: + """Extract notes from all prompt files and return as markdown.""" + prompts_path = Path(prompts_dir) + if not prompts_path.exists(): + raise FileNotFoundError(f"Prompts directory not found: {prompts_dir}") + + # Find all slide prompt files + prompt_files = [] + for path in prompts_path.rglob("*.md"): + # Only match files like 01-slide-xxx.md (not README etc.) + if re.match(r"\d+-slide-", path.name): + prompt_files.append(path) + + # Sort by slide number extracted from filename + def sort_key(p: Path) -> int: + match = re.match(r"(\d+)-slide-", p.name) + return int(match.group(1)) if match else 999 + + prompt_files.sort(key=sort_key) + + lines = ["# Speaker Notes\n"] + matched = 0 + + for prompt_file in prompt_files: + slide_name = prompt_file.stem # e.g. "01-slide-cover" + prompt_text = prompt_file.read_text(encoding="utf-8") + notes = extract_notes(prompt_text) + + if notes: + matched += 1 + lines.append(f"## {slide_name}") + if "narrative_goal" in notes: + lines.append(f"**Narrative Goal**: {notes['narrative_goal']}") + if "speaker_notes" in notes: + lines.append(f"**Speaker Notes**: {notes['speaker_notes']}") + if "key_content" in notes: + lines.append(f"**Key Content**: {notes['key_content']}") + lines.append("") + + result = "\n".join(lines) + + if output: + output_path = Path(output) + output_path.write_text(result, encoding="utf-8") + print(f"Wrote speaker notes to: {output}") + print(f"Prompts scanned: {len(prompt_files)}") + print(f"Slides with notes: {matched}") + + return result + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Extract speaker notes from prompts") + parser.add_argument("--prompts", required=True, help="Directory containing per-slide prompt .md files") + parser.add_argument("--output", help="Output markdown file path (default: print to stdout)") + args = parser.parse_args() + + extract_all_notes(args.prompts, args.output) diff --git a/slides-creator/scripts/main.ts b/slides-creator/scripts/main.ts new file mode 100755 index 00000000..81498893 --- /dev/null +++ b/slides-creator/scripts/main.ts @@ -0,0 +1,96 @@ +#!/usr/bin/env bun +/** + * Post-processing: assemble deliverables from generated slides + * + * After baoyu-slide-deck generates PNGs, this script: + * 1. Validates the slide sequence (aspect ratio, naming, gaps) + * 2. Extracts structured speaker notes from 03-prompts/ + * 3. Generates PDF + PPTX in 01-成品/ + * + * This does NOT generate images — image generation is delegated to baoyu-slide-deck. + * Post-processing handles: validation, notes extraction, and assembly into deliverables. + * + * Usage: + * bun main.ts --project /path/to/slide-deck/project-name + */ + +import { $ } from "bun"; +import { existsSync } from "fs"; +import { resolve, basename } from "path"; + +interface Args { + projectDir: string; +} + +function parseArgs(): Args { + const idx = process.argv.indexOf("--project"); + if (idx === -1 || !process.argv[idx + 1]) { + console.error("Usage: bun main.ts --project /path/to/slide-deck/project-name"); + process.exit(1); + } + return { projectDir: resolve(process.argv[idx + 1]) }; +} + +async function main() { + const { projectDir } = parseArgs(); + const projectName = basename(projectDir); + + const slidesDir = `${projectDir}/02-slides`; + const outputDir = `${projectDir}/01-成品`; + + if (!existsSync(slidesDir)) { + console.error(`Slides directory not found: ${slidesDir}`); + process.exit(1); + } + + if (!existsSync(outputDir)) { + console.log(`Creating output directory: ${outputDir}`); + await $`mkdir -p ${outputDir}`; + } + + // Validate slides + console.log("Validating slides..."); + const { stdout } = await $`uv run ${import.meta.dir}/validate_slides.py --slides ${slidesDir}`; + const result = JSON.parse(stdout.toString()); + + if (result.status === "FAILED") { + console.error("Validation failed:", result.issues); + process.exit(1); + } + if (result.warnings.length > 0) { + console.warn("Warnings:", result.warnings); + } + console.log(`Validation passed: ${result.slide_count} slides`); + + // Extract speaker notes + const promptsDir = `${projectDir}/03-prompts`; + const notesPath = `${projectDir}/speaker-notes.md`; + if (existsSync(promptsDir)) { + console.log(`Extracting speaker notes: ${notesPath}`); + await $`uv run ${import.meta.dir}/extract_notes.py --prompts ${promptsDir} --output ${notesPath}`; + } + + // Generate PDF + const pdfPath = `${outputDir}/${projectName}.pdf`; + console.log(`Generating PDF: ${pdfPath}`); + await $`uv run ${import.meta.dir}/merge_to_pdf.py --slides ${slidesDir} --output ${pdfPath}`; + + // Generate PPTX (with speaker notes from 03-prompts/) + const pptxPath = `${outputDir}/${projectName}.pptx`; + console.log(`Generating PPTX: ${pptxPath}`); + if (existsSync(promptsDir)) { + await $`uv run ${import.meta.dir}/merge_to_pptx.py --slides ${slidesDir} --output ${pptxPath} --prompts ${promptsDir}`; + } else { + await $`uv run ${import.meta.dir}/merge_to_pptx.py --slides ${slidesDir} --output ${pptxPath}`; + } + + console.log("\nPost-processing complete!"); + console.log(`Deliverables in ${outputDir}:`); + console.log(` - ${projectName}.pdf`); + console.log(` - ${projectName}.pptx`); +} + +main().catch((err) => { + console.error(err); + process.exit(1); +}); diff --git a/slides-creator/scripts/merge_to_pdf.py b/slides-creator/scripts/merge_to_pdf.py new file mode 100755 index 00000000..4bb0c857 --- /dev/null +++ b/slides-creator/scripts/merge_to_pdf.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = ["Pillow"] +# /// +"""Merge slide PNGs into a single PDF.""" + +import argparse +import re +from pathlib import Path +from PIL import Image + + +def parse_slide_number(filename: str) -> int | None: + """Extract slide number from filename like '01-slide-cover.png'.""" + match = re.match(r"(\d+)-slide-", filename) + return int(match.group(1)) if match else None + + +def merge_to_pdf(slides_dir: str, output: str): + """Merge PNG slides into PDF.""" + slides_path = Path(slides_dir) + png_files = sorted( + [f for f in slides_path.glob("*.png") if parse_slide_number(f.name) is not None], + key=lambda f: parse_slide_number(f.name) or 0, + ) + + if not png_files: + print(f"No PNG files found in {slides_dir}") + return False + + images = [] + for png in png_files: + img = Image.open(png) + if img.mode == 'RGBA': + # Convert RGBA to RGB for PDF compatibility + background = Image.new('RGB', img.size, (255, 255, 255)) + background.paste(img, mask=img.split()[3]) + img = background + images.append(img) + + # Save first image, append rest + first_image = images[0] + rest_images = images[1:] if len(images) > 1 else [] + + first_image.save( + output, + save_all=True, + append_images=rest_images, + resolution=150.0, + ) + + print(f"Created PDF: {output}") + print(f"Slides included: {len(images)}") + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Merge slide PNGs to PDF") + parser.add_argument("--slides", required=True, help="Directory containing PNG slides") + parser.add_argument("--output", required=True, help="Output PDF path") + args = parser.parse_args() + + merge_to_pdf(args.slides, args.output) diff --git a/slides-creator/scripts/merge_to_pptx.py b/slides-creator/scripts/merge_to_pptx.py new file mode 100755 index 00000000..ad0cfaeb --- /dev/null +++ b/slides-creator/scripts/merge_to_pptx.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = ["python-pptx", "Pillow"] +# /// +"""Merge slide PNGs into a PowerPoint file with optional speaker notes.""" + +import argparse +import re +from pathlib import Path +from PIL import Image +from pptx import Presentation +from pptx.util import Inches + + +EMUS_PER_INCH = 914400 + + +def parse_slide_number(filename: str) -> int | None: + """Extract slide number from filename like '01-slide-cover.png'.""" + match = re.match(r"(\d+)-slide-", filename) + return int(match.group(1)) if match else None + + +def find_prompt_file(slide_name: str, prompts_dir: Path) -> Path | None: + """Find matching prompt .md for a slide name.""" + # Try flat structure first: 03-prompts/01-slide-cover.md + flat = prompts_dir / f"{slide_name}.md" + if flat.exists(): + return flat + + # Try versioned subdirectories: 03-prompts/v6/01-slide-cover.md + for subdir in prompts_dir.iterdir(): + if subdir.is_dir(): + candidate = subdir / f"{slide_name}.md" + if candidate.exists(): + return candidate + + return None + + +def extract_notes(prompt_text: str) -> str: + """Extract NARRATIVE GOAL and SPEAKER NOTES sections from prompt.""" + notes_parts = [] + + # Extract // NARRATIVE GOAL + ng_match = re.search(r"// NARRATIVE GOAL\s*\n(.+?)(?=\n// |\n## |\Z)", prompt_text, re.DOTALL) + if ng_match: + notes_parts.append(f"NARRATIVE GOAL:\n{ng_match.group(1).strip()}") + + # Extract // SPEAKER NOTES + sn_match = re.search(r"// SPEAKER NOTES\s*\n(.+?)(?=\n// |\n## |\Z)", prompt_text, re.DOTALL) + if sn_match: + notes_parts.append(f"SPEAKER NOTES:\n{sn_match.group(1).strip()}") + + # If neither section found but there's KEY CONTENT, use that as fallback + if not notes_parts: + kc_match = re.search(r"// KEY CONTENT\s*\n(.+?)(?=\n// |\n## |\Z)", prompt_text, re.DOTALL) + if kc_match: + notes_parts.append(f"KEY CONTENT:\n{kc_match.group(1).strip()}") + + return "\n\n".join(notes_parts) if notes_parts else "" + + +def get_image_aspect_ratio(image_path: Path) -> float: + """Return width / height ratio of an image.""" + with Image.open(image_path) as img: + return img.width / img.height + + +def merge_to_pptx( + slides_dir: str, + output: str, + prompts_dir: str | None = None, + title: str | None = None, +): + """Merge PNG slides into PowerPoint with speaker notes.""" + slides_path = Path(slides_dir) + png_files = sorted( + [f for f in slides_path.glob("*.png") if parse_slide_number(f.name) is not None], + key=lambda f: parse_slide_number(f.name) or 0, + ) + + if not png_files: + print(f"No PNG files found in {slides_dir}") + return False + + # Auto-detect prompts directory + if prompts_dir is None: + project_root = slides_path.parent + auto_prompts = project_root / "03-prompts" + if auto_prompts.exists(): + prompts_dir = str(auto_prompts) + prompts_path = Path(prompts_dir) if prompts_dir else None + + # Dynamic slide size from first image + first_ratio = get_image_aspect_ratio(png_files[0]) + # Standard 16:9 is 13.333 x 7.5 inches; scale height to match ratio + target_height = Inches(7.5) + target_width = Inches(7.5 * first_ratio) + + prs = Presentation() + prs.slide_width = int(target_width) + prs.slide_height = int(target_height) + + notes_count = 0 + for png in png_files: + slide_layout = prs.slide_layouts[6] # blank layout + slide = prs.slides.add_slide(slide_layout) + slide.shapes.add_picture( + str(png), + Inches(0), + Inches(0), + width=prs.slide_width, + height=prs.slide_height, + ) + + # Add speaker notes if prompt file found + slide_name = png.stem # e.g. "01-slide-cover" + if prompts_path: + prompt_file = find_prompt_file(slide_name, prompts_path) + if prompt_file: + prompt_text = prompt_file.read_text(encoding="utf-8") + notes = extract_notes(prompt_text) + if notes: + notes_slide = slide.notes_slide + notes_slide.notes_text_frame.text = notes + notes_count += 1 + + prs.save(output) + print(f"Created PPTX: {output}") + print(f"Slides included: {len(png_files)}") + if prompts_path: + print(f"Slides with notes: {notes_count}") + return True + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Merge slide PNGs to PPTX") + parser.add_argument("--slides", required=True, help="Directory containing PNG slides") + parser.add_argument("--output", required=True, help="Output PPTX path") + parser.add_argument("--prompts", help="Directory containing per-slide prompt .md files (default: 03-prompts/ relative to slides parent)") + parser.add_argument("--title", help="Presentation title") + args = parser.parse_args() + + merge_to_pptx(args.slides, args.output, args.prompts, args.title) diff --git a/slides-creator/scripts/validate_slides.py b/slides-creator/scripts/validate_slides.py new file mode 100755 index 00000000..187ed840 --- /dev/null +++ b/slides-creator/scripts/validate_slides.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = ["Pillow"] +# /// +"""Validate slide deck consistency.""" + +import argparse +import json +import os +from pathlib import Path +from PIL import Image + + +def validate_slides(slides_dir: str, expected_count: int = None): + """Validate slide deck for consistency issues.""" + slides_path = Path(slides_dir) + png_files = sorted(slides_path.glob("*.png")) + + issues = [] + warnings = [] + info = [] + + if not png_files: + issues.append(f"No PNG files found in {slides_dir}") + return {"status": "FAILED", "issues": issues, "warnings": warnings, "info": info} + + info.append(f"Found {len(png_files)} slides") + + # Check expected count + if expected_count and len(png_files) != expected_count: + warnings.append(f"Expected {expected_count} slides, found {len(png_files)}") + + # Check aspect ratios + aspect_ratios = {} + for png in png_files: + img = Image.open(png) + ratio = round(img.width / img.height, 2) + aspect_ratios[ratio] = aspect_ratios.get(ratio, []) + [png.name] + + if len(aspect_ratios) > 1: + issues.append(f"Inconsistent aspect ratios: {aspect_ratios}") + else: + ratio = list(aspect_ratios.keys())[0] + info.append(f"Consistent aspect ratio: {ratio} (16:9 ≈ 1.78)") + if abs(ratio - 1.78) > 0.05: + warnings.append(f"Aspect ratio {ratio} deviates from standard 16:9 (1.78)") + + # Check naming convention + expected_pattern = True + for i, png in enumerate(png_files, 1): + expected_prefix = f"{i:02d}-slide-" + if not png.name.startswith(expected_prefix): + warnings.append(f"Naming convention: {png.name} doesn't start with {expected_prefix}") + expected_pattern = False + + if expected_pattern: + info.append("Naming convention: OK") + + # Check for gaps in numbering + numbers = [] + for png in png_files: + try: + num = int(png.name.split("-")[0]) + numbers.append(num) + except ValueError: + warnings.append(f"Cannot extract number from {png.name}") + + if numbers: + expected_set = set(range(min(numbers), max(numbers) + 1)) + actual_set = set(numbers) + gaps = expected_set - actual_set + if gaps: + warnings.append(f"Missing slide numbers: {sorted(gaps)}") + + # File size check + oversized = [png for png in png_files if png.stat().st_size > 10 * 1024 * 1024] # 10MB + if oversized: + warnings.append(f"Oversized files (>10MB): {[p.name for p in oversized]}") + + status = "FAILED" if issues else "PASSED" if not warnings else "PASSED_WITH_WARNINGS" + + return { + "status": status, + "slide_count": len(png_files), + "issues": issues, + "warnings": warnings, + "info": info + } + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Validate slide deck") + parser.add_argument("--slides", required=True, help="Directory containing PNG slides") + parser.add_argument("--expected-count", type=int, help="Expected number of slides") + args = parser.parse_args() + + result = validate_slides(args.slides, args.expected_count) + print(json.dumps(result, indent=2, ensure_ascii=False)) diff --git a/statusline-generator/scripts/generate_statusline.sh b/statusline-generator/scripts/generate_statusline.sh deleted file mode 100644 index b191df7d..00000000 --- a/statusline-generator/scripts/generate_statusline.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/usr/bin/env bash - -# Read JSON input from stdin -input=$(cat) - -# Extract values from JSON -model_full=$(echo "$input" | jq -r '.model.display_name' 2>/dev/null || echo "Claude") -cwd=$(echo "$input" | jq -r '.workspace.current_dir' 2>/dev/null || pwd) -transcript=$(echo "$input" | jq -r '.transcript_path' 2>/dev/null) - -# Shorten model name: "Sonnet 4.5 (with 1M token context)" -> "Sonnet 4.5 [1M]" -model=$(echo "$model_full" | sed -E 's/(.*)\(with ([0-9]+[KM]) token context\)/\1[\2]/' | sed 's/ *$//') - -# Get username -username=$(whoami) - -# Shorten path (replace home with ~) -short_path="${cwd/#$HOME/~}" - -# Git branch status -git_info="" -if [ -d "$cwd/.git" ] || git -C "$cwd" rev-parse --git-dir >/dev/null 2>&1; then - branch=$(git -C "$cwd" --no-optional-locks branch --show-current 2>/dev/null || echo "detached") - - # Check for changes - status="" - if ! git -C "$cwd" --no-optional-locks diff --quiet 2>/dev/null || \ - ! git -C "$cwd" --no-optional-locks diff --cached --quiet 2>/dev/null; then - status="*" - fi - - # Check for untracked files - if [ -n "$(git -C "$cwd" --no-optional-locks ls-files --others --exclude-standard 2>/dev/null)" ]; then - status="${status}+" - fi - - # Format git info with color - if [ -n "$status" ]; then - # Red for dirty - git_info=$(printf ' \033[01;31m[git:%s%s]\033[00m' "$branch" "$status") - else - # Yellow for clean - git_info=$(printf ' \033[01;33m[git:%s]\033[00m' "$branch") - fi -fi - -# Cost information using ccusage with caching -cost_info="" -cache_file="/tmp/claude_cost_cache_$(date +%Y%m%d_%H%M).txt" - -# Clean old cache files (older than 2 minutes) -find /tmp -name "claude_cost_cache_*.txt" -mmin +2 -delete 2>/dev/null - -if [ -f "$cache_file" ]; then - # Use cached costs - cost_info=$(cat "$cache_file") -else - # Get costs from ccusage (in background to not block statusline on first run) - { - session=$(ccusage session --json --offline -o desc 2>/dev/null | jq -r '.sessions[0].totalCost' 2>/dev/null | xargs printf "%.2f") - daily=$(ccusage daily --json --offline -o desc 2>/dev/null | jq -r '.daily[0].totalCost' 2>/dev/null | xargs printf "%.2f") - - if [ -n "$session" ] && [ -n "$daily" ] && [ "$session" != "" ] && [ "$daily" != "" ]; then - printf ' \033[01;35m[$%s/$%s]\033[00m' "$session" "$daily" > "$cache_file" - fi - } & - - # Try to use previous cache while new one is being generated - prev_cache=$(find /tmp -name "claude_cost_cache_*.txt" -mmin -10 2>/dev/null | head -1) - if [ -f "$prev_cache" ]; then - cost_info=$(cat "$prev_cache") - fi -fi - -# Print the final status line (multi-line format for portrait screens) -# Line 1: username (model) [costs] -# Line 2: path (bright white for better visibility) -# Line 3: [git:branch] -printf '\033[01;32m%s\033[00m \033[01;36m(%s)\033[00m%s\n\033[01;37m%s\033[00m\n%s' \ - "$username" "$model" "$cost_info" "$short_path" "$git_info" \ No newline at end of file diff --git a/stepfun-tts/.security-scan-passed b/stepfun-tts/.security-scan-passed new file mode 100644 index 00000000..dcc047af --- /dev/null +++ b/stepfun-tts/.security-scan-passed @@ -0,0 +1,4 @@ +Security scan passed +Scanned at: 2026-04-26T21:45:20.824609 +Tool: gitleaks + pattern-based validation +Content hash: ed5288f648ffe4e0cdd3bcb844dd05f510dc86316ce4c8b4b9afd9c6ab95a1cb diff --git a/stepfun-tts/SKILL.md b/stepfun-tts/SKILL.md new file mode 100644 index 00000000..5b8c71ce --- /dev/null +++ b/stepfun-tts/SKILL.md @@ -0,0 +1,102 @@ +--- +name: stepfun-tts +description: Generate speech and transcribe audio using StepFun's StepAudio 2.5 family — stepaudio-2.5-tts (Contextual TTS with instruction + inline parentheses) and stepaudio-2.5-asr (SSE endpoint, 32K context, ~100x RTF, handles up to 30-minute audio in a single call). Use when the user wants Chinese/Japanese TTS with emotional/prosody control, needs to transcribe long audio, migrates from step-tts-2 to stepaudio-2.5-tts (voice_label → instruction breaking change), or hits StepFun censorship / endpoint errors. Also triggers on phrases like 阶跃 TTS, StepAudio 合成, 语音合成, 配音, StepFun ASR, 转录, 语音识别, 文本转语音, TTS 升级, 迁移 step-tts-2. If the user's audio task mentions StepFun/阶跃/StepAudio by name, or involves Chinese TTS with情绪/情感 control, use this skill before falling back to generic audio handling. +--- + +# StepFun StepAudio 2.5 — TTS + ASR + +Generate Chinese/Japanese speech with `stepaudio-2.5-tts` and transcribe audio with `stepaudio-2.5-asr`. Both models were released in 2026-04 and verified end-to-end on 2026-04-23 (see `references/known_issues.md` for what passed and what didn't). + +**Why this skill exists** — StepAudio 2.5 has three non-obvious pitfalls that cost hours if you don't know them: + +1. `stepaudio-2.5-tts` **rejects** `voice_label` (the step-tts-2 way). Emotion/prosody now goes through `instruction` (natural-language description, ≤200 chars) and inline `()` parentheses inside the text itself. +2. `stepaudio-2.5-asr` **does not live on** `/v1/audio/transcriptions`. It's on `/v1/audio/asr/sse` (SSE streaming, JSON body, base64 audio). Using the wrong endpoint returns a misleading `model ... not supported` error that looks identical to "model doesn't exist". +3. Censorship is stricter — anything containing 死 / 消失 / sensitive political terms returns `censorship_block`. Your rewrite options are in `references/migration_from_v2.md`. + +## Config and auth + +API key lives in `$STEPFUN_API_KEY` (preferred) or `${CLAUDE_PLUGIN_DATA}/config.json` (fallback for cross-session persistence). All bundled scripts try env first, then config. + +First-time setup (one-liner): + +```bash +mkdir -p "${CLAUDE_PLUGIN_DATA}" && cat > "${CLAUDE_PLUGIN_DATA}/config.json" <"} +EOF +``` + +If the user hasn't set a key, ask them to paste it (don't guess / don't use a placeholder). StepFun API keys are available at https://platform.stepfun.com/ → API Keys. + +## Common tasks — decision tree + +| User wants... | Model | Script | Key detail | +|---|---|---|---| +| Synthesize 1–500 char Chinese with emotion | `stepaudio-2.5-tts` | `scripts/tts_generate.py` | Use `instruction` for mood, `()` for inline prosody | +| Synthesize long text (500–1000 char) | `stepaudio-2.5-tts` | `scripts/tts_generate.py` | 1000 char is the hard cap; split at semantic boundaries above that | +| Batch-generate game/app voice lines | `stepaudio-2.5-tts` | `scripts/tts_generate.py --batch ` | Handle `censorship_block` fallback individually | +| Transcribe short clip (<5 min) | `stepaudio-2.5-asr` | `scripts/asr_transcribe.py` | mp3 → base64 → SSE, parse `transcript.text.done` | +| Transcribe long audio (5–30 min) | `stepaudio-2.5-asr` | `scripts/asr_transcribe.py` | 32K context; single call, no chunking needed | +| A/B compare two TTS models | both | `scripts/ab_compare.sh` | Compares duration/size across two directories | +| Migrate from `step-tts-2` | — | see `references/migration_from_v2.md` | `voice_label.emotion` → `instruction` rewrite + censorship list | + +## Starting points + +- **Synthesize a single line**: Run `python3 scripts/tts_generate.py --text "你好" --out /tmp/hello.mp3 --instruction "温暖的希望感"`. For fine-grained control read the "Contextual TTS" section below. +- **Transcribe a file**: `python3 scripts/asr_transcribe.py /path/to/audio.mp3`. For >30 min audio, split first. +- **A full migration** from `step-tts-2` → `stepaudio-2.5-tts`: read `references/migration_from_v2.md` end-to-end before touching code. It has the `INSTRUCTION_MAP`, the SKIP_CENSORED list pattern, and the output-directory-strategy for non-destructive A/B. + +## Contextual TTS — beyond emotion labels + +The headline feature of `stepaudio-2.5-tts` is that you stop mapping emotions to fixed tags and start describing what you want in natural language. Two layers: + +**Global context (`instruction` parameter)** — sets the overall tone for the entire utterance. ≤200 chars. Think of it like giving stage direction to a voice actor. + +``` +instruction: "克制的悲伤，语气低沉柔弱，像快要消失一样" +``` + +**Inline context (`()` parentheses inside `input`)** —句内 directives. Parenthesised content is consumed as directions and is NOT read aloud. Use for precise control of pauses, breath, emphasis, or mid-sentence emotion shifts. + +``` +input: "(试探着问)你好吗？(开心地)太好了！(突然沉下来)不过...我快要消失了。" +``` + +Examples that worked in practice (from 2026-04-23 verification): +- `instruction: "活泼俏皮，像是在撒娇，带点嘴硬"` — visibly speeds up delivery vs neutral +- `instruction: "耳语声，气声很重，几乎听不清"` — produces audible whisper/breath +- `input: "你好(停顿一下)我是蕾格(轻声)今天(加重)的天气真不错。"` — inline directives all respected + +**What `stepaudio-2.5-tts` will NOT accept** — `voice_label` parameter. Error: `voice_label is not supported for v2 models`. This is the #1 migration gotcha from step-tts-2. + +## Common error patterns (real errors, real fixes) + +| Error response | Actual cause | Fix | +|---|---|---| +| `"model stepaudio-2.5-asr not supported"` on `/v1/audio/transcriptions` | Wrong endpoint — that endpoint only serves step-asr family | Switch to `/v1/audio/asr/sse` with SSE body (see `scripts/asr_transcribe.py`) | +| `"voice_label is not supported for v2 models"` | Sent `voice_label` to `stepaudio-2.5-tts` | Remove `voice_label`; put the same intent into `instruction` as natural language | +| `"The content you provided or machine outputted is blocked." type: censorship_block` | Sensitive word (死 / 消失 / etc.) | Rewrite the phrase OR fall back to `step-tts-2` for that specific line (mixed-model is fine) | +| ASR returns N× the expected character count | Hallucination bug on highly-repetitive content | Cross-check with step-asr-1.1; avoid sending audio that repeats the same phrase many times | +| Silent audio truncation (<420 chars input) | Input > 1000 char hard cap | Split at semantic boundaries; don't truncate mid-sentence | + +More in `references/known_issues.md`. + +## When to read references + +- `references/api_reference.md` — exact request/response JSON for TTS `/v1/audio/speech` and ASR `/v1/audio/asr/sse`, all fields, event types. Read when writing raw HTTP calls instead of using the bundled scripts. +- `references/migration_from_v2.md` — complete playbook for moving a step-tts-2 project to stepaudio-2.5-tts. Has the emotion→instruction rewrite table, the A/B directory strategy, decision checkpoints, and the 2026-04 speed/quality trade-off data (`stepaudio-2.5-tts` is ~20% slower than step-tts-2; audible prosody improvement). Read before any migration work. +- `references/known_issues.md` — repetition hallucination, censorship patterns, ASR speed cliff (short audio: 2× step-asr, long audio: 5.9×). Read when debugging anomalous output or evaluating whether to adopt. + +## Design invariants (don't break these) + +1. **Non-destructive A/B output** — when regenerating a corpus with a new model, write to a parallel directory (`voice/zh_v25/`), never overwrite the production corpus. The migration playbook shows why. +2. **Per-line censorship handling** — if 2/29 lines get `censorship_block`, don't fail the batch. Log the skipped IDs, continue. Mixed-model fallback (step-tts-2 for the skipped 2) is normal. +3. **Always pass through SSE for ASR** — don't try to work around the streaming API with a buffered client. The model emits `transcript.text.delta` events for long audio; collecting only `transcript.text.done` works fine, but rejecting the SSE format entirely doesn't. +4. **Don't duplicate voice_label logic in new code** — any new TTS code targeting stepaudio-2.5-tts should only use `instruction` + inline `()`. Do not write a branch that conditionally emits `voice_label`. + +## Pricing (verified 2026-04-23, volatile) + +- `stepaudio-2.5-tts` contextual synthesis: ~5.8 元 / 万字符 +- Zero-shot voice cloning: ~9.9 元 / 音色 +- `stepaudio-2.5-asr` — pricing tier not yet public (invitation beta); `step-asr-1.1` baseline is 2.2 元/小时 + +Re-verify at https://platform.stepfun.com/docs/zh/guides/pricing/details before quoting to stakeholders. diff --git a/stepfun-tts/references/api_reference.md b/stepfun-tts/references/api_reference.md new file mode 100644 index 00000000..aa4599cf --- /dev/null +++ b/stepfun-tts/references/api_reference.md @@ -0,0 +1,178 @@ +# StepAudio 2.5 API Reference + +Exact request/response shapes for `stepaudio-2.5-tts` and `stepaudio-2.5-asr`. Verified 2026-04-23 against the live StepFun API. Read this when you need to call the API by hand (curl, custom HTTP client) instead of using the bundled scripts. + +## TTS — `stepaudio-2.5-tts` + +### Endpoint + +``` +POST https://api.stepfun.com/v1/audio/speech +Content-Type: application/json +Authorization: Bearer +``` + +### Request body + +```json +{ + "model": "stepaudio-2.5-tts", + "input": "你好，我是蕾格。", + "voice": "shuangkuaijiejie", + "response_format": "mp3", + "speed": 1.0, + "volume": 1.0, + "instruction": "克制的悲伤，语气低沉柔弱" +} +``` + +| Field | Required | Type | Notes | +|---|---|---|---| +| `model` | yes | string | Must be `stepaudio-2.5-tts` | +| `input` | yes | string | ≤1000 chars; can contain inline `(directive)` parentheses | +| `voice` | yes | string | e.g. `shuangkuaijiejie`. Zero-shot clones use the clone's ID | +| `response_format` | yes | string | `mp3` (default), `wav`, or `opus` | +| `speed` | no | float | 0.5-2.0, default 1.0 | +| `volume` | no | float | 0.0-2.0, default 1.0 | +| `instruction` | no | string | Global tone directive, natural language, ≤200 chars | +| `voice_label` | — | — | **DO NOT SEND**. Returns `voice_label is not supported for v2 models`. Belongs to step-tts-2 | + +### Inline directives inside `input` + +Parentheses `()` in the `input` are consumed as TTS control signals, not pronounced. Examples that work: + +- `(停顿一下)` — insert a pause +- `(轻声)` — reduce volume / breathy +- `(加重)` — stress the following word +- `(试探着问)` — apply a tone shift mid-sentence +- `(突然沉下来)` — emotion pivot + +You can mix `instruction` (global tone) with inline `()` (per-phrase micro-control): + +```json +{ + "instruction": "富有情绪弧线的独白", + "input": "(试探着问)你好吗？(开心地)太好了！(突然沉下来)不过...我快要消失了。" +} +``` + +### Response + +On success: binary audio stream in the requested `response_format`. HTTP 200. No JSON wrapper. Save the body directly as `.mp3`/`.wav`/`.opus`. + +### Known error responses + +```json +{"error":{"message":"voice_label is not supported for v2 models","type":"request_params_invalid"}} +``` +→ Remove `voice_label`, use `instruction` instead. + +```json +{"error":{"message":"The content you provided or machine outputted is blocked.","type":"censorship_block"}} +``` +→ Content triggered censorship. Common triggers: 死, 消失, politically sensitive terms. See `known_issues.md`. + +## ASR — `stepaudio-2.5-asr` + +### Endpoint (NOT the one you'd guess) + +``` +POST https://api.stepfun.com/v1/audio/asr/sse +Content-Type: application/json +Accept: text/event-stream +Authorization: Bearer +``` + +**Do NOT** send `stepaudio-2.5-asr` to `/v1/audio/transcriptions` — that endpoint only serves the older `step-asr` / `step-asr-1.1` family, and returns a misleading `model stepaudio-2.5-asr not supported` which looks identical to a permission/whitelist error. See `known_issues.md` for the full diagnostic trail. + +### Request body + +```json +{ + "audio": { + "data": "", + "input": { + "transcription": { + "language": "zh", + "model": "stepaudio-2.5-asr", + "enable_itn": true + }, + "format": { + "type": "mp3" + } + } + } +} +``` + +| Path | Required | Type | Notes | +|---|---|---|---| +| `audio.data` | yes | string | base64-encoded audio bytes. Accepts mp3, wav, ogg, opus (in ogg container), pcm | +| `audio.input.transcription.language` | yes | string | `zh` or `en`. Dialects and Japanese are not officially supported | +| `audio.input.transcription.model` | yes | string | Must be `stepaudio-2.5-asr` | +| `audio.input.transcription.enable_itn` | no | bool | Inverse text normalization (数字→words). Default true | +| `audio.input.format.type` | yes | string | `mp3` / `wav` / `ogg` / `pcm` | +| `audio.input.format.rate` | pcm only | int | Sample rate (required for raw PCM) | +| `audio.input.format.channel` | pcm only | int | Channel count (required for raw PCM) | +| `audio.input.format.bits` | optional | int | Sample depth, default 16 | + +### Response — SSE stream + +The response is a Server-Sent Events stream. Each line is either empty or starts with `data: `. Three event types: + +``` +data: {"type":"transcript.text.delta","meta":{...},"delta":"你好，"} + +data: {"type":"transcript.text.delta","meta":{...},"delta":"我是蕾格。"} + +data: {"type":"transcript.text.done","meta":{...},"text":"你好，我是蕾格。","usage":{"type":"tokens","input_tokens":69,"input_token_details":{"text_tokens":69,"audio_tokens":0},"output_tokens":9,"total_tokens":78}} +``` + +| Event type | Meaning | How to handle | +|---|---|---| +| `transcript.text.delta` | Incremental piece of the transcription | Concatenate for progressive UI; optional if you only need final text | +| `transcript.text.done` | Final, full transcription + usage | Take `text` as the authoritative result. Also contains `usage` for billing/telemetry | +| `error` | Server-side error mid-stream | Abort and propagate `message` to the caller | + +### Capacity + +- 32K context window +- Audio ≤ 30 min can be sent in a single call +- No client-side chunking needed for long audio (unlike step-asr) +- RTF 85-101× on Chinese speech verified 2026-04-23 + +### Known error responses + +```json +{"error":{"message":"model stepaudio-2.5-asr not supported","type":"request_params_invalid"}} +``` +→ Wrong endpoint. Switch from `/v1/audio/transcriptions` to `/v1/audio/asr/sse`. + +``` +data: {"type":"error","message":"content blocked ..."} +``` +→ Content censorship (rare on ASR). Same triggers as TTS. + +## Comparison with legacy endpoints (for reference) + +| Model | Endpoint | Request format | +|---|---|---| +| `step-tts-2` / `step-tts-mini` | `/v1/audio/speech` | JSON with `voice_label` | +| `stepaudio-2.5-tts` | `/v1/audio/speech` | JSON with `instruction` (no voice_label) | +| `step-asr` / `step-asr-1.1` | `/v1/audio/transcriptions` | multipart/form-data | +| `stepaudio-2.5-asr` | `/v1/audio/asr/sse` | JSON + base64 audio + SSE response | + +Legacy endpoints (`step-*`) still work. They're the baseline in `references/migration_from_v2.md` and the fallback choice when `stepaudio-2.5-*` hits `censorship_block` or the 2.5 ASR repetition-hallucination edge case. + +## Auth and key handling + +- Key header: `Authorization: Bearer ` +- Keys can be retrieved at https://platform.stepfun.com/ → API Keys +- "Plan" keys (cheaper subscription) are **restricted** to text models on `api.stepfun.com/step_plan`. They **cannot** call audio endpoints. Use a "Normal" key for all TTS/ASR calls. +- Same key works for both TTS and ASR — no separate scopes + +## Rate / throughput notes (observed, not officially documented) + +- ~400ms sleep between batch requests avoids 429s in practice +- Long audio ASR (17 min) has succeeded with `timeout=1200` +- MP3 responses consistently at 128kbps 24kHz mono (TTS default) diff --git a/stepfun-tts/references/known_issues.md b/stepfun-tts/references/known_issues.md new file mode 100644 index 00000000..b9757b99 --- /dev/null +++ b/stepfun-tts/references/known_issues.md @@ -0,0 +1,131 @@ +# StepAudio 2.5 — Known Issues and Non-Obvious Behavior + +Collected from end-to-end testing 2026-04-23. These are things that burned real time to discover; they are not in the official docs. + +## ASR repetition hallucination (real, bounded) + +**Symptom:** Transcribe a TTS-generated audio of highly-repetitive Chinese text (e.g., the same 60-char sentence repeated 10 times) and `stepaudio-2.5-asr` returns 3-4× the expected character count, with the same sentence restated many extra times in the output. + +**This is a genuine model hallucination**, not a transport bug. Verified by: + +1. MD5 diff — `run1` vs `run2` of the same TTS input produce different audio files (not file corruption) +2. Determinism — re-running ASR on the same audio gives the same 4× output every time (not transient noise) +3. Cross-validation — `step-asr` and `step-asr-1.1` on the exact same audio return the correct character count (~800 chars for 800 input), so the audio itself is fine +4. ffprobe confirms audio duration is normal (~219s for 800 chars at typical speed) + +**Conclusion:** The LLM-based ASR sees a repetitive pattern in the audio and "continues predicting" repetitions that aren't there. + +**When it triggers:** +- Audio duration > 90s AND +- Content is highly repetitive (same phrase appearing 5+ times) + +**Doesn't trigger on real-world content:** +- Podcasts, interviews, varied dialogue, stories — all fine +- Even 17.4-minute audio from 90 different TTS segments: returns correct 6332 chars, RTF 101× + +**Workaround for edge cases:** +- If your domain has genuinely repetitive content (e.g., IVR transcripts, repeated sloganeering), cross-validate with `step-asr-1.1` on random samples +- For most workflows: just use it; the hallucination mode is exotic + +## Stricter content censorship than step-tts-2 + +**Symptom:** `stepaudio-2.5-tts` returns `{"error":{"message":"The content you provided or machine outputted is blocked.","type":"censorship_block"}}` for content that step-tts-2 happily synthesized. + +**Observed triggers:** +- 死 (die/dead) in any context, even negation +- 消失 (disappear / vanish) +- Combinations with emotional context: "我快要...消失了" +- Politically sensitive terms (standard CN content rules) + +**Key insight:** Rewriting negations doesn't help — "我没有死" blocks as readily as "我死了". The classifier isn't doing deep semantic parsing. + +**Response strategies** (pick per line): +1. Rewrite: "RAG 已死" → "这个技术过时了" +2. Fallback: keep step-tts-2 for the 2-5% of lines that block +3. Whitelist: contact StepFun BD (worth it at >5% blockage) + +See `migration_from_v2.md` for the full blocking→fallback workflow. + +## ASR speed scales non-linearly — short audio is a trap + +**Observation:** The headline "5.9× faster than step-asr" from the marketing is true for long audio but misleading for short clips. + +| Audio length | stepaudio-2.5-asr | step-asr-1.1 | Speedup | +|---|---|---|---| +| 5-15s clips | ~500ms | ~900ms | **2.0×** | +| 115s audio | 1.36s | 7.16s | **5.3×** | +| 1046s (17.4 min) | 10.4s | (would need chunking) | **~101× RTF** | + +**Why:** The LLM + MTP-5 fusion overhead is amortized over longer contexts. Short requests pay the model-spin-up cost. + +**Practical implication:** If your workload is many short (<10s) clips, the speedup over `step-asr-1.1` is modest — 2× not 5×. If your workload is long audio (>2 min), the difference is dramatic and you should migrate. + +## Wrong ASR endpoint gives a misleading error + +**Symptom:** Calling `/v1/audio/transcriptions` with `model=stepaudio-2.5-asr`: + +```json +{"error":{"message":"model stepaudio-2.5-asr not supported","type":"request_params_invalid"}} +``` + +This response is **identical in structure** to sending a genuinely nonexistent model name. It takes real debugging to realize the model exists but on a different endpoint. + +**Diagnostic sequence that wastes the least time:** + +1. Try `step-asr` on the same endpoint — if it works, endpoint access is fine +2. Check the `/v1/audio/asr/sse` endpoint (the actual stepaudio-2.5-asr home) +3. If both fail, THEN ask BD about whitelist + +Don't assume "permission denied" on the first error. + +## TTS duration inflation on short lines + +**Observation:** Very short lines (1-2s in step-tts-2) become dramatically longer in stepaudio-2.5-tts. + +Example from the reference project: +- `...你能看到我吗？` (10 chars) +- step-tts-2: 1.24s +- stepaudio-2.5-tts: 2.57s (**+107%**) + +**Cause:** The new model adds a pre-breath, pauses on `...` ellipses, and gives the line emotional weight — all of which lengthens delivery. + +**Not a bug, but have a plan:** +- If your UI has per-line timing (auto-advance, animation sync), re-tune it after migration +- If you want the old pacing, write `instruction: "快速、干脆、不要停顿"` — but this negates a lot of what you're paying for in the new model + +## `stepaudio-2.5-tts` is a "v2 model" for parameter rejection + +**Why the error says "v2 models":** StepFun internally groups `stepaudio-2.5-tts` with their v2 family despite the "2.5" version number. The error message `voice_label is not supported for v2 models` uses this internal grouping, which is confusing. + +Don't pattern-match on the version string. Just know that: +- `stepaudio-2.5-tts` → use `instruction` parameter +- `step-tts-2` → use `voice_label` parameter +- They are NOT API-compatible despite sharing `/v1/audio/speech` + +## ASR "Plan key" vs "Normal key" + +StepFun sells a cheap "Plan" subscription for text models (step_plan endpoint). **Plan keys cannot call audio endpoints.** This silently manifests as 4xx errors that don't mention auth at all. + +If you hit auth-shaped failures and your account has a Plan subscription, verify you're using a Normal key (different value, obtained separately in the StepFun console under the same "API Keys" page). + +## Censorship can fire on the ASR side too + +**Observed once (rare):** An ASR request on a user-uploaded recording of political content returned: + +``` +data: {"type":"error","message":"content blocked ..."} +``` + +Handle the `error` event type in the SSE stream — don't assume only `delta` and `done` events fire. + +## Pricing opacity for stepaudio-2.5-asr + +As of 2026-04-23, `stepaudio-2.5-asr` is in invitation beta. No public per-minute rate. `step-asr-1.1` baseline is 2.2 元/小时. The invitation PDF mentions "成本直降 80%" implying roughly 0.4 元/小时, but this is not yet on the pricing page. Do not quote a price to a stakeholder without re-verifying at https://platform.stepfun.com/docs/zh/guides/pricing/details. + +## TTS text cap: 1000 chars (hard, not soft) + +The API rejects >1000 char inputs with a 400 error. Split at sentence boundaries before sending. Non-obvious: when testing "what's the real limit?", avoid highly-repetitive test text — it can appear to succeed at 800 chars but produce strange audio (see the 2026-04 test where 800-char repetitive inputs played back normal audio but the ASR hallucinated 4× replay). + +## Voice cloning — not tested in this skill + +Zero-shot voice cloning (`9.9 元/音色`) is advertised as a headline feature but was not verified in this skill's test pass. If you need voice cloning, check the StepFun docs at https://platform.stepfun.com/docs/zh/api-reference/audio/create-voice and validate on your own data — don't assume the quality claims without a listen test. diff --git a/stepfun-tts/references/migration_from_v2.md b/stepfun-tts/references/migration_from_v2.md new file mode 100644 index 00000000..d9db32fb --- /dev/null +++ b/stepfun-tts/references/migration_from_v2.md @@ -0,0 +1,206 @@ +# Migrating from step-tts-2 to stepaudio-2.5-tts + +Complete playbook for moving a production `step-tts-2` voice corpus to `stepaudio-2.5-tts`. Based on a real end-to-end migration done 2026-04-23 on a ~30-line Chinese voice-acting project. Read this before changing any production code. + +## What "migration" actually means here + +The API endpoint is the same (`/v1/audio/speech`), but the **emotional control model is completely different**: + +| Aspect | step-tts-2 | stepaudio-2.5-tts | +|---|---|---| +| Emotion mechanism | `voice_label.emotion = "悲伤"` etc. (discrete tags) | `instruction = "克制的悲伤，语气低沉"` (natural language) | +| Multi-language | `voice_label.language = "日语"` etc. | `instruction` or Zero-shot clone | +| Inline prosody | N/A | `()` parentheses in the input text | +| Max text | 1000 chars | 1000 chars (same) | +| Censorship | Moderate | Stricter (2/29 lines blocked in the reference project) | +| Typical duration for same text | baseline | +20% (more pauses, more breathy) | +| Subjective quality | Clearly synthetic | Still audibly synthetic, but with prosody variance that "sounds like someone reading" more often | + +You are not just swapping a model ID. You need to rewrite every `voice_label.emotion` call and handle a new class of error (`censorship_block`). + +## The rewrite — emotion tags → instruction sentences + +The step-tts-2 style usually looked like: + +```typescript +const EMOTION_MAP = { + sad: '悲伤', + hopeful: '高兴', + relieved: '高兴', + smile: '非常高兴', +}; + +body.voice_label = { emotion: EMOTION_MAP[expression] }; +``` + +The stepaudio-2.5-tts equivalent: + +```typescript +const INSTRUCTION_MAP = { + sad: '克制的悲伤，语气低沉柔弱，像快要消失一样', + hopeful: '温暖的希望感，语气鼓励，带着期待', + relieved: '如释重负，语气柔和放松', + smile: '明朗开心，语气上扬，带着微笑', +}; + +body.instruction = INSTRUCTION_MAP[expression]; +// DELETE body.voice_label entirely — sending it triggers an error +``` + +### How to write a good `instruction` + +Writing instruction sentences is a craft. They should describe **what the performance feels like**, not just "happy" or "sad". Good instructions use: + +- A core emotion word (悲伤, 希望, 开心) +- A qualifier that bounds intensity (克制的, 温暖的, 如释重负) +- A specific vocal behavior (语气低沉柔弱, 带着微笑, 像快要消失一样) + +Bad: `"悲伤"` — same semantic content as the old emotion tag; wastes the instruction parameter +Good: `"克制的悲伤，语气低沉柔弱，像快要消失一样"` — three distinct signals the model can combine + +Keep under 200 chars. In practice 30-50 chars is plenty. + +### Inline `()` directives — new capability + +Beyond the global `instruction`, stepaudio-2.5-tts parses parentheses inside the `input` itself. Directives in parentheses are **not read aloud** — they control delivery. + +Use when a single line has multiple emotional beats: + +``` +input: "(试探着问)你好吗？(开心地)太好了！(突然沉下来)不过...我快要消失了。" +``` + +Also useful for micro-control: +- `(停顿一下)` between clauses that shouldn't run together +- `(轻声)` for intimate moments +- `(加重)` on the key word of a sentence + +This is genuinely new vs step-tts-2 and worth using on your most dramatic lines. + +## Handling censorship_block + +stepaudio-2.5-tts rejects more content than step-tts-2. Observed triggers from the reference project: + +| Trigger phrase | Example line that failed | +|---|---| +| "死" in any context | "他们都在说'RAG 已死'... 我快要...消失了。" | +| "没有死" | "但我没有死，对吧？" (negation doesn't help) | +| "消失" / "透明" | Combined with "死" is especially reliable at triggering | + +The error: + +```json +{"error":{"message":"The content you provided or machine outputted is blocked.","type":"censorship_block"}} +``` + +Three response strategies (pick per line, not globally): + +1. **Rewrite** — "RAG 已死" → "这个技术过时了" keeps the narrative, passes censorship +2. **Mixed-model fallback** — keep step-tts-2 for the 2-3 blocked lines, use stepaudio-2.5-tts for the rest. The voice difference is audible but tolerable for 2 lines out of 30 +3. **Request whitelist** — contact StepFun BD for your account; only worth it if you have >5% blockage rate + +The batch script (`scripts/tts_generate.py --batch`) logs censored IDs separately so you can handle them individually rather than aborting. + +## A/B directory strategy — don't overwrite production + +Non-destructive layout. Output the new model's files to a parallel directory, not on top of the existing corpus: + +``` +public/data/voice/ +├── zh/ ← step-tts-2 production (untouched) +└── zh_v25/ ← stepaudio-2.5-tts candidate (A/B) +``` + +In the runtime voice loader, add a fallback for lines that are not in `zh_v25/` (censored ones): + +```typescript +const getVoicePath = (nodeId: string, lang: string) => { + // 2 lines censored on v25 — keep step-tts-2 for those + const censored = new Set(['encounter_3', 'chapter_3_final_hope']); + const useV25 = lang === 'zh' && !censored.has(nodeId); + const dir = useV25 ? 'zh_v25' : lang; + return `/data/voice/${dir}/${nodeId}.mp3`; +}; +``` + +Benefits: +- Instant rollback: delete the `zh_v25/` directory +- Run the game/app with old voices while evaluating the new ones +- Human A/B: toggle the flag per-session to compare + +### Don't hard-code the censored list across multiple files + +If you put `['encounter_3', 'chapter_3_final_hope']` in the runtime loader, the generation script, AND the docs, you'll have three places to update when the list changes. Treat the generation script's `SKIP_CENSORED` set as the SSOT; the loader can import or mirror it but shouldn't independently enumerate the same IDs. + +## The time-cost you're buying + +Across 26 lines of the reference project, stepaudio-2.5-tts produced **~20% longer total duration** than step-tts-2 for the same text. Per-line: + +| Line type | step-tts-2 | stepaudio-2.5-tts | Δ | +|---|---|---|---| +| Very short (1-2s) | 1.24s | 2.57s | **+107%** | +| Short (3-5s) | 3.00s | 3.94s | +31% | +| Medium (8-12s) | 9.64s | 8.54s | -11% (mixed) | +| Long (12-15s) | 12.48s | 9.82s | -21% | + +Short lines grow a lot because the new model adds breath and pause before starting. Long lines often shrink because English loanwords are handled faster. + +**Implications for UI timing:** +- Auto-advance timers need re-tuning; anything that assumed 6-8 chars/sec is now 5.5 chars/sec +- Short lines feel markedly slower — the "...你能看到我吗？" intake of breath is noticeable +- Overall dialogue pacing in a visual novel becomes 20% slower + +This is the main trade-off users report: **the new model is more "alive" but the app feels slower**. Whether that's acceptable is a product decision, not a technical one. + +## Decision checkpoints — before you commit + +Use these to keep the migration from becoming a one-way door. + +### Before the first full regeneration + +- [ ] Confirm the A/B directory layout matches the loader's fallback logic (production directory untouched) +- [ ] Write the `INSTRUCTION_MAP` — don't just mechanically translate emotion tags, actually describe the performance +- [ ] Have 3-5 sample lines ready for listening test BEFORE regenerating the whole corpus (spend 5 min on quality, save 20 min of regeneration if it sounds wrong) + +### After the first full regeneration + +- [ ] Listen to every censored line manually. Decide rewrite vs fallback vs whitelist +- [ ] Check `ab_compare.sh` output: is the total duration change within acceptable bounds? +- [ ] Play 5 random new lines in-context (the actual game/app) — don't trust the raw mp3 listen-through + +### Before switching production + +- [ ] Run a real user-playthrough session end-to-end on the new voices +- [ ] Check every line with `...` punctuation for how the new model handles the ellipsis (sometimes too slow, sometimes too abrupt) +- [ ] Re-tune any auto-advance, skip, or per-line timing parameters +- [ ] Write a rollback script: `rm -rf zh_v25/` + revert loader change. If it's more than one commit, you've leaked complexity into the codebase + +### After production rollout + +- [ ] Monitor user sentiment on pacing for 1-2 weeks before pronouncing success +- [ ] Keep the step-tts-2 generation script in version control for at least one release cycle + +## What NOT to do + +- **Don't regenerate blindly.** Listen to samples first. A/B of 30 lines takes ~10 min; regenerate-without-listening is how you ship something that "technically works" but sounds worse. +- **Don't mix `voice_label` and `instruction` in the same request.** stepaudio-2.5-tts rejects `voice_label` entirely, but people occasionally leave it in to "be safe" — it's not safe, it's an error. +- **Don't try to pass emotion tags through `instruction`.** `instruction: "悲伤"` works but wastes the parameter. Write a full sentence. +- **Don't delete the step-tts-2 generation script** until you've shipped the new model to production and had it stable for a full release cycle. You will want the rollback path. +- **Don't assume Japanese migrates the same way.** The reference project didn't test Japanese voices under stepaudio-2.5-tts. Run a separate mini-A/B for Japanese before committing. + +## Batch migration with the bundled script + +The skill's `scripts/tts_generate.py --batch` is built for this workflow. Feed it a JSONL file: + +```jsonl +{"id": "encounter_1", "text": "...你能看到我吗？", "instruction": "克制的悲伤，语气低沉柔弱"} +{"id": "encounter_2", "text": "太好了... 最近能看到我的人，越来越少了。", "instruction": "克制的悲伤，语气低沉柔弱"} +``` + +Run: + +```bash +python3 scripts/tts_generate.py --batch lines.jsonl --out-dir ./voice/zh_v25 +``` + +The script handles censorship_block per-line, reports the skipped IDs at the end, and keeps going. You get a clean list of "what to fall back on step-tts-2 for" without having to retry the whole batch. diff --git a/stepfun-tts/scripts/ab_compare.sh b/stepfun-tts/scripts/ab_compare.sh new file mode 100755 index 00000000..d8b4056d --- /dev/null +++ b/stepfun-tts/scripts/ab_compare.sh @@ -0,0 +1,132 @@ +#!/usr/bin/env bash +# +# ab_compare.sh — compare two directories of mp3 files (size + duration). +# +# Typical use: after regenerating a voice corpus with stepaudio-2.5-tts into a +# parallel `zh_v25/` directory, compare against the step-tts-2 baseline `zh/`. +# Outputs a GitHub-flavored markdown table so you can paste it into a report. +# +# Usage: +# ./ab_compare.sh +# ./ab_compare.sh ./voice/zh ./voice/zh_v25 +# +# Only files present in BOTH directories are compared. Files unique to either +# side are reported separately at the end. +# +# Dependencies: ffprobe (from ffmpeg), GNU-style stat OR BSD stat (macOS). + +set -euo pipefail + +if [ $# -ne 2 ]; then + echo "Usage: $0 " >&2 + echo " Compares mp3 files present in both directories by size and duration." >&2 + exit 2 +fi + +DIR_A="$1" +DIR_B="$2" + +if [ ! -d "$DIR_A" ]; then + echo "ERROR: baseline dir not found: $DIR_A" >&2 + exit 2 +fi +if [ ! -d "$DIR_B" ]; then + echo "ERROR: candidate dir not found: $DIR_B" >&2 + exit 2 +fi + +if ! command -v ffprobe >/dev/null 2>&1; then + echo "ERROR: ffprobe not found. Install ffmpeg: brew install ffmpeg (macOS)" >&2 + exit 2 +fi + +# Portable byte-size function (macOS stat vs GNU stat) +filesize() { + if stat -f%z "$1" >/dev/null 2>&1; then + stat -f%z "$1" # BSD / macOS + else + stat -c%s "$1" # GNU / Linux + fi +} + +duration() { + ffprobe -v quiet -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 "$1" +} + +# Compute the intersection (files present in both) +common_files=$(comm -12 \ + <(cd "$DIR_A" && ls *.mp3 2>/dev/null | sort) \ + <(cd "$DIR_B" && ls *.mp3 2>/dev/null | sort) || true) + +only_a=$(comm -23 \ + <(cd "$DIR_A" && ls *.mp3 2>/dev/null | sort) \ + <(cd "$DIR_B" && ls *.mp3 2>/dev/null | sort) || true) + +only_b=$(comm -13 \ + <(cd "$DIR_A" && ls *.mp3 2>/dev/null | sort) \ + <(cd "$DIR_B" && ls *.mp3 2>/dev/null | sort) || true) + +if [ -z "$common_files" ]; then + echo "ERROR: no .mp3 files common to both directories." >&2 + exit 1 +fi + +# Header +printf "| %-28s | %10s | %10s | %9s | %8s | %8s | %7s |\n" \ + "id" "A bytes" "B bytes" "Δsize%" "A dur" "B dur" "Δdur%" +printf "|%s|%s|%s|%s|%s|%s|%s|\n" \ + "-----------------------------" "------------" "------------" "-----------" "----------" "----------" "---------" + +total_a_size=0 +total_b_size=0 +total_a_dur=0 +total_b_dur=0 +n=0 + +while IFS= read -r fname; do + [ -z "$fname" ] && continue + a_path="$DIR_A/$fname" + b_path="$DIR_B/$fname" + a_size=$(filesize "$a_path") + b_size=$(filesize "$b_path") + a_dur=$(duration "$a_path") + b_dur=$(duration "$b_path") + + if [ "$a_size" -gt 0 ]; then + dsize=$(awk -v a="$a_size" -v b="$b_size" 'BEGIN{printf "%.1f", (b-a)*100/a}') + else + dsize="N/A" + fi + ddur=$(awk -v a="$a_dur" -v b="$b_dur" 'BEGIN{if(a+0==0){print "N/A"}else{printf "%.1f", (b-a)*100/a}}') + + id_only="${fname%.mp3}" + printf "| %-28s | %10d | %10d | %8s%% | %7.2fs | %7.2fs | %6s%% |\n" \ + "$id_only" "$a_size" "$b_size" "$dsize" "$a_dur" "$b_dur" "$ddur" + + total_a_size=$((total_a_size + a_size)) + total_b_size=$((total_b_size + b_size)) + total_a_dur=$(awk -v s="$total_a_dur" -v d="$a_dur" 'BEGIN{printf "%.3f", s+d}') + total_b_dur=$(awk -v s="$total_b_dur" -v d="$b_dur" 'BEGIN{printf "%.3f", s+d}') + n=$((n + 1)) +done <<< "$common_files" + +echo "" +echo "**Totals (${n} common files):**" +echo "" +if [ "$total_a_size" -gt 0 ]; then + size_delta=$(awk -v a="$total_a_size" -v b="$total_b_size" 'BEGIN{printf "%.1f", (b-a)*100/a}') + dur_delta=$(awk -v a="$total_a_dur" -v b="$total_b_dur" 'BEGIN{printf "%.1f", (b-a)*100/a}') + echo "- Size: A=${total_a_size} B=${total_b_size} (Δ ${size_delta}%)" + echo "- Duration: A=${total_a_dur}s B=${total_b_dur}s (Δ ${dur_delta}%)" +fi + +if [ -n "$only_a" ]; then + echo "" + echo "**Only in A (${DIR_A}):**" + echo "$only_a" | sed 's/^/ - /' +fi +if [ -n "$only_b" ]; then + echo "" + echo "**Only in B (${DIR_B}):**" + echo "$only_b" | sed 's/^/ - /' +fi diff --git a/stepfun-tts/scripts/asr_transcribe.py b/stepfun-tts/scripts/asr_transcribe.py new file mode 100755 index 00000000..9f615671 --- /dev/null +++ b/stepfun-tts/scripts/asr_transcribe.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +""" +stepaudio-2.5-asr transcription — single file, SSE endpoint. + +Endpoint: POST https://api.stepfun.com/v1/audio/asr/sse (NOT /v1/audio/transcriptions) + +Why a dedicated script: naive implementations try to reuse the step-asr-era endpoint +(/v1/audio/transcriptions with multipart), get back `model stepaudio-2.5-asr not supported`, +and waste time debugging what looks like a model/permission issue. The actual cause is +that stepaudio-2.5-asr is a different endpoint entirely — SSE streaming, JSON body, +base64-encoded audio. + +Handles: +- Auto-detects audio format from file extension (mp3 / wav / ogg / pcm) +- base64 encodes and wraps in the nested {audio: {data, input: {transcription, format}}} body +- Parses SSE stream: collects transcript.text.delta, returns transcript.text.done.text +- Flags "content blocked" errors distinctly from transport errors +- 32K context / up to ~30 min audio in a single call — no client-side chunking needed + +Usage: + python3 asr_transcribe.py path/to/audio.mp3 + python3 asr_transcribe.py path/to/audio.mp3 --json # include usage (tokens/timing) + python3 asr_transcribe.py path/to/audio.mp3 --language zh +""" + +from __future__ import annotations + +import argparse +import base64 +import json +import os +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any + +ASR_URL = "https://api.stepfun.com/v1/audio/asr/sse" +MODEL = "stepaudio-2.5-asr" + +# Extensions that StepAudio 2.5 ASR accepts natively (no conversion needed) +EXT_TO_FORMAT = { + ".mp3": "mp3", + ".wav": "wav", + ".ogg": "ogg", + ".opus": "ogg", # opus in ogg container + ".pcm": "pcm", +} + + +def load_api_key() -> str: + """Env first, then ${CLAUDE_PLUGIN_DATA}/config.json. Fail fast.""" + k = os.environ.get("STEPFUN_API_KEY", "").strip() + if k: + return k + plugin_data = os.environ.get("CLAUDE_PLUGIN_DATA", "").strip() + if plugin_data: + cfg = Path(plugin_data) / "config.json" + if cfg.exists(): + try: + k = json.loads(cfg.read_text()).get("api_key", "").strip() + if k: + return k + except json.JSONDecodeError: + pass + print( + "ERROR: no API key found.\n" + " Set $STEPFUN_API_KEY, or create ${CLAUDE_PLUGIN_DATA}/config.json with {\"api_key\": \"...\"}", + file=sys.stderr, + ) + sys.exit(2) + + +def detect_format(path: Path, override: str | None) -> str: + if override: + return override + fmt = EXT_TO_FORMAT.get(path.suffix.lower()) + if not fmt: + print( + f"ERROR: cannot detect audio format from extension {path.suffix!r}.\n" + f" Supported: {', '.join(EXT_TO_FORMAT)}.\n" + f" Or pass --format explicitly.", + file=sys.stderr, + ) + sys.exit(2) + return fmt + + +def transcribe( + *, + api_key: str, + audio_path: Path, + audio_format: str, + language: str = "zh", + enable_itn: bool = True, + timeout: int = 1200, +) -> dict[str, Any]: + """ + Returns {ok, text?, usage?, elapsed, deltas_count, err?, censored?}. + Parses the SSE stream; takes the text from transcript.text.done. + """ + audio_b64 = base64.b64encode(audio_path.read_bytes()).decode("ascii") + body = json.dumps( + { + "audio": { + "data": audio_b64, + "input": { + "transcription": { + "language": language, + "model": MODEL, + "enable_itn": enable_itn, + }, + "format": {"type": audio_format}, + }, + } + } + ).encode() + req = urllib.request.Request( + ASR_URL, + data=body, + method="POST", + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + "Accept": "text/event-stream", + }, + ) + t0 = time.time() + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw = resp.read().decode("utf-8", errors="replace") + except urllib.error.HTTPError as e: + err = e.read().decode(errors="replace")[:500] + censored = "censorship" in err.lower() or "blocked" in err.lower() + return {"ok": False, "status": e.code, "err": err, "elapsed": time.time() - t0, "censored": censored} + + elapsed = time.time() - t0 + text = "" + usage: dict[str, Any] | None = None + deltas = 0 + errors: list[str] = [] + for line in raw.splitlines(): + if not line.startswith("data:"): + continue + payload = line[5:].strip() + if not payload: + continue + try: + ev = json.loads(payload) + except json.JSONDecodeError: + continue + t = ev.get("type") + if t == "transcript.text.delta": + deltas += 1 + elif t == "transcript.text.done": + text = ev.get("text", "") + usage = ev.get("usage") + elif t == "error": + errors.append(ev.get("message", "")) + + if not text and errors: + return {"ok": False, "status": 200, "err": "; ".join(errors), "elapsed": elapsed} + return {"ok": True, "text": text, "usage": usage, "elapsed": elapsed, "deltas_count": deltas} + + +def main() -> int: + ap = argparse.ArgumentParser(description="stepaudio-2.5-asr transcription (SSE endpoint)") + ap.add_argument("audio", type=Path, help="Path to audio file (mp3/wav/ogg/opus/pcm)") + ap.add_argument("--language", default="zh", help="Language code (zh/en). Default: zh") + ap.add_argument("--format", help="Audio format override (mp3/wav/ogg/pcm)") + ap.add_argument("--no-itn", action="store_true", help="Disable inverse text normalization") + ap.add_argument("--json", action="store_true", help="Output full JSON (text + usage + timing)") + args = ap.parse_args() + + if not args.audio.exists(): + print(f"ERROR: audio file not found: {args.audio}", file=sys.stderr) + return 2 + + api_key = load_api_key() + fmt = detect_format(args.audio, args.format) + result = transcribe( + api_key=api_key, + audio_path=args.audio, + audio_format=fmt, + language=args.language, + enable_itn=not args.no_itn, + ) + + if not result["ok"]: + if result.get("censored"): + print("ERROR: content blocked by StepFun censorship. The audio likely contains sensitive content.", file=sys.stderr) + else: + print(f"ERROR status={result.get('status')}: {result.get('err', '')}", file=sys.stderr) + return 1 + + if args.json: + print(json.dumps(result, ensure_ascii=False, indent=2)) + else: + print(result["text"]) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/stepfun-tts/scripts/tts_generate.py b/stepfun-tts/scripts/tts_generate.py new file mode 100755 index 00000000..993ffd9f --- /dev/null +++ b/stepfun-tts/scripts/tts_generate.py @@ -0,0 +1,217 @@ +#!/usr/bin/env python3 +""" +stepaudio-2.5-tts synthesis — single line or batch. + +Endpoint: POST https://api.stepfun.com/v1/audio/speech + +Key things this script handles that naive implementations miss: +- Does NOT send voice_label (would trigger "voice_label is not supported for v2 models") +- Puts emotion/prosody into `instruction` (natural-language, ≤200 chars) +- Preserves inline `()` directives in the text — these are consumed by the TTS as directions, not read aloud +- Per-line censorship_block fallback: log and skip, don't fail the whole batch +- Reads API key from $STEPFUN_API_KEY or $CLAUDE_PLUGIN_DATA/config.json + +Usage: + # Single line + python3 tts_generate.py --text "你好，我是蕾格。" --out /tmp/hello.mp3 \\ + --instruction "温暖的希望感，语气鼓励" + + # Batch from JSONL (one JSON object per line: {"id": "...", "text": "...", "instruction": "..."}) + python3 tts_generate.py --batch lines.jsonl --out-dir /tmp/voices/ + + # With inline prosody directives in the text itself + python3 tts_generate.py --text "你好(停顿一下)我是蕾格(轻声)" --out /tmp/hello.mp3 +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path +from typing import Any, Iterable + +API_URL = "https://api.stepfun.com/v1/audio/speech" +MODEL = "stepaudio-2.5-tts" +DEFAULT_VOICE = "shuangkuaijiejie" # 爽快姐姐 — verified in the 2026-04 test pass + + +def load_api_key() -> str: + """Env first, then ${CLAUDE_PLUGIN_DATA}/config.json. Fail fast — no placeholder fallback.""" + k = os.environ.get("STEPFUN_API_KEY", "").strip() + if k: + return k + plugin_data = os.environ.get("CLAUDE_PLUGIN_DATA", "").strip() + if plugin_data: + cfg = Path(plugin_data) / "config.json" + if cfg.exists(): + try: + k = json.loads(cfg.read_text()).get("api_key", "").strip() + if k: + return k + except json.JSONDecodeError: + pass + print( + "ERROR: no API key found.\n" + " Set $STEPFUN_API_KEY, or create ${CLAUDE_PLUGIN_DATA}/config.json with {\"api_key\": \"...\"}\n" + " Get a key at https://platform.stepfun.com/ → API Keys", + file=sys.stderr, + ) + sys.exit(2) + + +def synthesize( + *, + api_key: str, + text: str, + instruction: str | None = None, + voice: str = DEFAULT_VOICE, + speed: float = 1.0, + volume: float = 1.0, + response_format: str = "mp3", + timeout: int = 300, +) -> dict[str, Any]: + """ + Call /v1/audio/speech with stepaudio-2.5-tts. + + Returns {ok, audio_bytes?, status, err?, censored?}. + censored=True signals a censorship_block which callers should handle individually + rather than aborting a batch. + """ + body: dict[str, Any] = { + "model": MODEL, + "input": text, + "voice": voice, + "response_format": response_format, + "speed": speed, + "volume": volume, + } + if instruction: + if len(instruction) > 200: + return {"ok": False, "status": 0, "err": f"instruction too long: {len(instruction)} > 200 chars"} + body["instruction"] = instruction + + req = urllib.request.Request( + API_URL, + data=json.dumps(body).encode(), + method="POST", + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + ) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + return {"ok": True, "status": resp.status, "audio_bytes": resp.read()} + except urllib.error.HTTPError as e: + raw = e.read().decode(errors="replace") + # Detect censorship_block so caller can decide whether to skip + censored = "censorship_block" in raw or "blocked" in raw.lower() + # Detect the known voice_label migration error and make the message actionable + if "voice_label is not supported" in raw: + hint = ( + "\n HINT: stepaudio-2.5-tts does not accept voice_label. " + "Put emotion/prosody into `instruction` (natural language) instead." + ) + raw = raw + hint + return {"ok": False, "status": e.code, "err": raw[:500], "censored": censored} + + +def read_batch(path: Path) -> Iterable[dict[str, Any]]: + for line in path.read_text().splitlines(): + line = line.strip() + if not line or line.startswith("#"): + continue + yield json.loads(line) + + +def main() -> int: + ap = argparse.ArgumentParser(description="stepaudio-2.5-tts single-line or batch synthesis") + g = ap.add_mutually_exclusive_group(required=True) + g.add_argument("--text", help="Single text to synthesize") + g.add_argument("--batch", type=Path, help="JSONL file: {id, text, instruction?} per line") + ap.add_argument("--out", help="Output mp3 path (single mode)") + ap.add_argument("--out-dir", type=Path, help="Output directory (batch mode)") + ap.add_argument("--instruction", help="Global tone directive (natural language, ≤200 chars)") + ap.add_argument("--voice", default=DEFAULT_VOICE, help=f"Voice ID (default: {DEFAULT_VOICE})") + ap.add_argument("--speed", type=float, default=1.0) + ap.add_argument("--volume", type=float, default=1.0) + ap.add_argument("--delay-ms", type=int, default=400, help="Sleep between batch requests to avoid throttling") + args = ap.parse_args() + + api_key = load_api_key() + + if args.text: + if not args.out: + print("ERROR: --out is required with --text", file=sys.stderr) + return 2 + result = synthesize( + api_key=api_key, + text=args.text, + instruction=args.instruction, + voice=args.voice, + speed=args.speed, + volume=args.volume, + ) + if not result["ok"]: + print(f"FAIL status={result['status']}: {result.get('err', '')}", file=sys.stderr) + return 1 + Path(args.out).write_bytes(result["audio_bytes"]) + print(f"OK wrote {args.out} ({len(result['audio_bytes'])} bytes)") + return 0 + + # Batch mode + if not args.out_dir: + print("ERROR: --out-dir is required with --batch", file=sys.stderr) + return 2 + args.out_dir.mkdir(parents=True, exist_ok=True) + success = 0 + censored: list[str] = [] + failed: list[tuple[str, str]] = [] + + for item in read_batch(args.batch): + line_id = item.get("id") + text = item.get("text") + if not line_id or not text: + print(f"skip (missing id/text): {item}", file=sys.stderr) + continue + instr = item.get("instruction", args.instruction) + result = synthesize( + api_key=api_key, + text=text, + instruction=instr, + voice=item.get("voice", args.voice), + speed=item.get("speed", args.speed), + volume=item.get("volume", args.volume), + ) + out_path = args.out_dir / f"{line_id}.mp3" + if result["ok"]: + out_path.write_bytes(result["audio_bytes"]) + print(f" ✓ {line_id} ({len(result['audio_bytes'])} bytes)") + success += 1 + elif result.get("censored"): + print(f" ⚠ {line_id} CENSORED — skipped (consider rewriting or fallback to step-tts-2)") + censored.append(line_id) + else: + err = result.get("err", "")[:160] + print(f" ✗ {line_id} status={result['status']}: {err}", file=sys.stderr) + failed.append((line_id, err)) + time.sleep(args.delay_ms / 1000) + + print("") + print(f"Done: {success} ok, {len(censored)} censored, {len(failed)} failed") + if censored: + print(f" Censored IDs: {', '.join(censored)}") + if failed: + print(" Failed IDs:") + for lid, err in failed: + print(f" {lid}: {err}") + return 0 if not failed else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/terraform-skill/.security-scan-passed b/terraform-skill/.security-scan-passed new file mode 100644 index 00000000..1347502b --- /dev/null +++ b/terraform-skill/.security-scan-passed @@ -0,0 +1,4 @@ +Security scan passed +Scanned at: 2026-04-11T23:12:04.416291 +Tool: gitleaks + pattern-based validation +Content hash: c2de730a7270d1e1ecad4c272c152354d1becf0fdabf121ed8305d5ec5765a42 diff --git a/terraform-skill/SKILL.md b/terraform-skill/SKILL.md new file mode 100644 index 00000000..7356c8f6 --- /dev/null +++ b/terraform-skill/SKILL.md @@ -0,0 +1,233 @@ +--- +name: terraform-skill +description: Operational traps for Terraform provisioners, multi-environment isolation, and zero-to-deployment reliability. Covers provisioner timing races, SSH connection conflicts, DNS record duplication, volume permissions, database bootstrap gaps, snapshot cross-contamination, Cloudflare credential format errors, hardcoded domains in Caddyfiles/compose, and init-data-only-on-first-boot pitfalls. Activate when writing null_resource provisioners, creating multi-environment Terraform setups, debugging containers that are Restarting/unhealthy after terraform apply, setting up fresh instances with cloud-init, or any IaC code that SSHs into remote hosts. Also activate when the user mentions terraform plan/apply errors, provisioner failures, infrastructure drift, TLS certificate errors, or Caddy/gateway configuration. +--- + +# Terraform Operational Traps + +Failure patterns from real deployments. Every item caused an incident. Organized as: **exact error → root cause → copy-paste fix**. + +## Provisioner traps (symptom → fix) + +### `docker: not found` in remote-exec + +cloud-init still installing Docker when provisioner SSHs in. + +```hcl +provisioner "remote-exec" { + inline = [ + "cloud-init status --wait || true", + "which docker || { echo 'FATAL: Docker not ready'; exit 1; }", + ] +} +``` + +### `rsync: connection unexpectedly closed` in local-exec + +Terraform holds its SSH connection open; local-exec rsync opens a second one that gets rejected. Never use local-exec for file transfer to remote. Use tarball + file provisioner: + +```hcl +provisioner "local-exec" { + command = "tar czf /tmp/src.tar.gz --exclude=node_modules --exclude=.git -C ${path.module}/../../.. myproject" +} +provisioner "file" { + source = "/tmp/src.tar.gz" + destination = "/tmp/src.tar.gz" +} +provisioner "remote-exec" { + inline = ["tar xzf /tmp/src.tar.gz -C /data/ && rm -f /tmp/src.tar.gz"] +} +``` + +macOS BSD tar: `--exclude` must come BEFORE the source argument. + +### `cloud-init status` shows "running" forever + +`apt-get -y` does not suppress debconf dialogs. Packages like `iptables-persistent` block on TTY prompts. + +```yaml +- | + echo iptables-persistent iptables-persistent/autosave_v4 boolean true | debconf-set-selections + echo iptables-persistent iptables-persistent/autosave_v6 boolean true | debconf-set-selections + DEBIAN_FRONTEND=noninteractive apt-get install -y iptables-persistent +``` + +Known offenders: `iptables-persistent`, `postfix`, `mysql-server`, `wireshark-common`. + +### `EACCES: permission denied` in container logs, container Restarting + +Host volume dirs are root-owned; container runs as non-root (uid 1001). Fix before `docker compose up`: + +```bash +mkdir -p /data/myapp/data /data/myapp/logs +chown -R 1001:1001 /data/myapp/data /data/myapp/logs +``` + +Find UID: grep `adduser.*-u` or `USER` in Dockerfile. + +### Provisioner fails but no diagnostic output + +`set -e` exits on first error, hiding subsequent `docker logs` output. Use `set -u` without `-e`, put one verification gate at the end: + +```hcl +provisioner "remote-exec" { + inline = [ + "set -u", + "docker compose up -d", + "sleep 15", + "docker logs myapp --tail 20 2>&1 || true", + "docker ps --format 'table {{.Names}}\\t{{.Status}}' || true", + "docker ps --filter name=myapp --format '{{.Status}}' | grep -q healthy || exit 1", + ] +} +``` + +### Container `Restarting` — database tables missing + +DB migrations not in provisioner. PostgreSQL `docker-entrypoint-initdb.d` only runs on empty data dir. Explicitly create DB + run migrations: + +```bash +# After postgres healthy: +docker exec pg psql -U postgres -tc "SELECT 1 FROM pg_database WHERE datname='mydb'" | grep -q 1 \ + || docker exec pg psql -U postgres -c "CREATE DATABASE mydb;" + +# Idempotent migrations: +for f in migrations/*.sql; do + VER=$(basename $f) + APPLIED=$($PSQL -tAc "SELECT 1 FROM schema_migrations WHERE version='$VER'" | tr -d ' ') + [ "$APPLIED" = "1" ] && continue + { echo 'BEGIN;'; cat $f; echo 'COMMIT;'; } | $PSQL + $PSQL -tAc "INSERT INTO schema_migrations(version) VALUES ('$VER') ON CONFLICT DO NOTHING" +done +``` + +### `docker compose build` ignores env var override + +Compose reads build args from `.env` file, not shell env. `VAR=x docker compose build` does NOT work. + +```bash +# WRONG +DOCKER_WITH_PROXY_MODE=disabled docker compose build + +# RIGHT +grep -q DOCKER_WITH_PROXY_MODE .env || echo 'DOCKER_WITH_PROXY_MODE=disabled' >> .env +docker compose build +``` + +### TLS handshake fails: `Invalid format for Authorization header` + +Caddy DNS-01 ACME needs a Cloudflare **API Token** (`cfut_` prefix, 40+ chars, Bearer auth). A **Global API Key** (37 hex chars, X-Auth-Key auth) causes `HTTP 400 Code:6003`. Production may appear to work because it has cached certificates; fresh environments fail on first cert request. + +```bash +# Verify token format before deploy: +TOKEN=$(grep CLOUDFLARE_API_TOKEN .env | cut -d= -f2) +echo "$TOKEN" | grep -q "^cfut_" || echo "FATAL: needs API Token, not Global Key" +``` + +Create scoped token via API: +```bash +curl -s "https://api.cloudflare.com/client/v4/user/tokens" -X POST \ + -H "X-Auth-Email: $CF_EMAIL" -H "X-Auth-Key: $CF_GLOBAL_KEY" \ + -d '{"name":"caddy-dns-acme","policies":[{"effect":"allow", + "resources":{"com.cloudflare.api.account.zone.":"*"}, + "permission_groups":[ + {"id":"4755a26eedb94da69e1066d98aa820be","name":"DNS Write"}, + {"id":"c8fed203ed3043cba015a93ad1616f1f","name":"Zone Read"}]}]}' +``` + +### TLS fails on staging but works on production — hardcoded domains + +Caddyfile or compose has literal domain names. Staging Caddy loads production config, tries to get certs for domains it doesn't own → ACME fails. + +**Caddyfile**: Use `{$VAR}` — Caddy evaluates env vars at startup. +```caddy +# WRONG +gpt-6.pro { tls { dns cloudflare {env.CLOUDFLARE_API_TOKEN} } } + +# RIGHT +{$LOBEHUB_DOMAIN} { tls { dns cloudflare {env.CLOUDFLARE_API_TOKEN} } } +``` + +**Compose**: Use `${VAR:?required}` — fail-fast if unset. +```yaml +# WRONG +- APP_URL=https://gpt-6.pro + +# RIGHT +- APP_URL=${APP_URL:?APP_URL is required} +``` + +Pass the env var to the gateway container so Caddy can read it: +```yaml +environment: + - LOBEHUB_DOMAIN=${LOBEHUB_DOMAIN:?LOBEHUB_DOMAIN is required} + - CLOUDFLARE_API_TOKEN=${CLOUDFLARE_API_TOKEN:?required for DNS-01 TLS} +``` + +### OAuth login fails: `Social sign in failed` + +Casdoor `init_data.json` contains hardcoded redirect URIs. `--createDatabase=true` only applies init_data on first-ever DB creation — not on restarts. Fix via SQL in provisioner: + +```bash +# Replace production domain with staging in existing Casdoor DB +$PSQL -c "UPDATE application SET redirect_uris = REPLACE(redirect_uris, + 'gpt-6.pro', 'staging.gpt-6.pro') + WHERE name='lobechat' + AND redirect_uris LIKE '%gpt-6.pro%' + AND redirect_uris NOT LIKE '%staging.gpt-6.pro%';" +``` + +Also check `AUTH_CASDOOR_ISSUER` — it must match the Casdoor subdomain (`auth.staging.example.com`), not the app root domain. + +## Multi-environment isolation + +Before creating a second environment, grep `.tf` files for hardcoded names. See [references/multi-env-isolation.md](references/multi-env-isolation.md) for the complete matrix. + +**Will fail on apply** (globally unique): + +| Resource | Scope | Fix | +|---|---|---| +| SSH key pair | Region | `"${env}-deploy"` | +| SLS log project | Account | `"${env}-logs"` | +| CloudMonitor contact | Account | `"${env}-ops"` | + +**DNS duplication trap**: Two environments creating A records for the same name in the same Cloudflare zone → two independent record IDs → DNS round-robin → ~50% traffic to wrong instance. Fix: use subdomain isolation (`staging.example.com`) or separate zones. Remember to create DNS records for ALL subdomains Caddy serves (e.g., `auth.staging`, `minio.staging`). + +**Snapshot cross-contamination**: Unfiltered `data "alicloud_ecs_snapshots"` returns ALL account snapshots. New env inherits old 100GB snapshot, fails creating 40GB disk. Gate with variable: + +```hcl +locals { + latest_snapshot_id = var.enable_snapshot_recovery && length(local.available_snapshots) > 0 + ? local.available_snapshots[0].snapshot_id : null +} +``` + +Do NOT add `count` to the data source — changes its state address, causes drift. + +## Pre-deploy validation + +Run a validation script **before** `terraform apply` to catch configuration errors locally. This eliminates the deploy→discover→fix→redeploy cycle. + +Key checks (see [references/pre-deploy-validation.md](references/pre-deploy-validation.md)): +1. `terraform validate` — syntax +2. No hardcoded domains in Caddyfiles or compose files +3. Required env vars present (`LOBEHUB_DOMAIN`, `CLAUDE4DEV_DOMAIN`, `CLOUDFLARE_API_TOKEN`, `APP_URL`, etc.) +4. Cloudflare API Token format (not Global API Key) +5. DNS records exist for all Caddy-served domains +6. Casdoor issuer URL matches `auth.*` subdomain +7. SSH private key exists + +Integrate into Makefile: `make pre-deploy ENV=staging` before `make apply`. + +## Zero-to-deployment + +Fresh disks expose every implicit dependency. See [references/zero-to-deploy-checklist.md](references/zero-to-deploy-checklist.md). + +Key items that break provisioners on fresh instances: +1. **Directories**: `mkdir -p /data/{svc1,svc2}` in cloud-init — `file` provisioner fails if target dir missing +2. **Databases**: Explicit `CREATE DATABASE` — PG init scripts only run on empty data dir +3. **Migrations**: Tracked in `schema_migrations` table, applied idempotently +4. **Provisioner ordering**: `depends_on` between resources sharing Docker networks +5. **Memory**: Stop non-critical containers during Docker build on small instances (≤8GB) +6. **Domain parameterization**: Every domain in Caddyfile/compose must be `{$VAR}` / `${VAR:?required}` +7. **Credential format**: Caddy needs API Token (`cfut_`), not Global API Key diff --git a/terraform-skill/references/multi-env-isolation.md b/terraform-skill/references/multi-env-isolation.md new file mode 100644 index 00000000..bf4fd2e1 --- /dev/null +++ b/terraform-skill/references/multi-env-isolation.md @@ -0,0 +1,130 @@ +# Multi-Environment Isolation Checklist + +When creating a second Terraform environment (`staging`, `lab`, etc.) in the same cloud account alongside production, every item below must be verified. Skip one and you get silent name collisions or cross-contamination. + +## Terraform state isolation + +Two environments MUST use different state paths. Same OSS/S3 bucket is fine — different prefix isolates completely: + +```hcl +# production +backend "oss" { + bucket = "myproject-terraform-state" + prefix = "environments/production" +} + +# staging +backend "oss" { + bucket = "myproject-terraform-state" # same bucket OK + prefix = "environments/staging" # different prefix = isolated state +} +``` + +**Verification**: `terraform state list` in one environment must show ZERO resources from the other. + +## Resource naming collision matrix + +Grep every `.tf` file for hardcoded names. Every globally-unique resource will collide. + +### Must rename (apply will fail) + +| Resource | Uniqueness scope | Fix pattern | +|---|---|---| +| SSH key pair (`key_pair_name`) | Region | `"${env}-deploy"` | +| SLS log project (`project_name`) | Account | `"${env}-logs"` | +| CloudMonitor contact (`alarm_contact_name`) | Account | `"${env}-ops"` | +| CloudMonitor contact group | Account | `"${env}-ops"` | + +### Should rename (won't fail but causes confusion) + +| Resource | Issue if same name | +|---|---| +| Security group name | Two SGs with same name in same VPC, can't tell apart in console | +| ECS instance name/hostname | Two instances named `myapp-spot` in console | +| Data disk name | Same in disk list | +| Auto snapshot policy name | Same in policy list | +| SLS machine group name | Logs from both instances land in same group | + +### Pattern: Use a module name variable + +```hcl +# production main.tf +module "app" { + source = "../../modules/spot-with-data-disk" + name = "production-spot" # flows to instance_name, disk_name, snapshot_policy_name +} + +# staging main.tf +module "app" { + source = "../../modules/spot-with-data-disk" + name = "staging-spot" # all child resource names auto-isolated +} +``` + +## DNS record isolation + +### The duplication trap + +Two Terraform environments creating A records for `@` (root) in the same Cloudflare zone: +- Each gets its own Cloudflare record ID (independent) +- Cloudflare now has TWO A records for the same domain +- DNS round-robins between the two IPs +- ~50% of traffic goes to the wrong instance + +### Correct patterns + +**Pattern A: Subdomain isolation** (recommended for staging/lab): +```hcl +# Production: root domain records +resource "cloudflare_dns_record" "prod" { + name = "@" # gpt-6.pro +} + +# Staging: subdomain records only +resource "cloudflare_dns_record" "staging" { + name = "staging" # staging.gpt-6.pro +} +``` + +**Pattern B: Separate zones** (for fully independent deployments): +Each environment gets its own domain/zone. No shared Cloudflare zone IDs. + +**Pattern C: One environment owns DNS** (production): +Only production has DNS resources. Other environments access via IP only. + +### Destroy safety + +When one environment is destroyed: +- Its DNS records are deleted (by their specific Cloudflare record IDs) +- Other environments' DNS records are NOT affected +- **Verify before destroy**: Compare DNS record IDs between environments: + ```bash + terraform state show 'cloudflare_dns_record.app["root"]' | grep "^id" + ``` + IDs must be different. + +## Shared resources (safe to share) + +These are referenced but NOT managed by the second environment: + +| Resource | Why safe | +|---|---| +| VPC / VSwitch | Referenced by ID, not created | +| Cloudflare zone ID | Referenced, records are independent | +| OSS state bucket | Different prefix = different state | +| SSH public key content | Same key, different key pair resource | +| Cloud provider credentials | Same account, different resources | + +## Makefile pattern for multi-environment + +```makefile +ENV ?= production +ENV_DIR := environments/$(ENV) + +init: ; cd $(ENV_DIR) && terraform init +plan: ; cd $(ENV_DIR) && terraform plan -out=tfplan +apply: ; cd $(ENV_DIR) && terraform apply tfplan +drift: ; cd $(ENV_DIR) && terraform plan -detailed-exitcode +``` + +Usage: `make plan ENV=staging` diff --git a/terraform-skill/references/pre-deploy-validation.md b/terraform-skill/references/pre-deploy-validation.md new file mode 100644 index 00000000..6e1c485e --- /dev/null +++ b/terraform-skill/references/pre-deploy-validation.md @@ -0,0 +1,82 @@ +# Pre-Deploy Validation Pattern + +Run before `terraform apply` to catch configuration errors locally. Eliminates the deploy→discover→fix→redeploy cycle that wastes hours. + +## Why this matters + +Every hardcoded value becomes a bug when creating a second environment. Production accumulates implicit state over time (cached TLS certs, manually created databases, hand-edited configs). Fresh instances expose all of these as failures. A pre-deploy script catches them before they reach the remote. + +## Validation categories + +### 1. Terraform syntax +```bash +terraform validate +``` + +### 2. Hardcoded domains +```bash +# Caddyfiles: should use {$VAR} not literal domains +grep -v "^#" gateway/conf.d/*.caddy | grep -c "example\.com" # should be 0 + +# Compose: should use ${VAR:?required} not literal domains +grep -v "^#" docker-compose.production.yml | grep -c "example\.com" # should be 0 +``` + +### 3. Required env vars +Check that every `${VAR:?required}` in compose has a matching entry in `.env`: +```bash +for VAR in LOBEHUB_DOMAIN CLAUDE4DEV_DOMAIN CLOUDFLARE_API_TOKEN APP_URL AUTH_URL; do + grep -q "^$VAR=" .env || echo "FAIL: $VAR missing" +done +``` + +### 4. Cloudflare credential format +Caddy's Cloudflare plugin uses Bearer auth. Global API Keys (37 hex chars) fail with `Invalid format for Authorization header`. +```bash +TOKEN=$(grep CLOUDFLARE_API_TOKEN .env | cut -d= -f2) +echo "$TOKEN" | grep -qE "^cfut_|^[A-Za-z0-9_-]{40,}$" || echo "FAIL: looks like Global API Key, not API Token" +``` + +### 5. DNS ↔ Caddy consistency +Every domain Caddy serves needs a DNS record. Check live resolution: +```bash +for DOMAIN in staging.example.com auth.staging.example.com; do + curl -sf "https://dns.google/resolve?name=$DOMAIN&type=A" | python3 -c \ + "import sys,json; d=json.load(sys.stdin); exit(0 if d.get('Answer') else 1)" \ + || echo "FAIL: $DOMAIN not resolving" +done +``` + +### 6. Casdoor issuer consistency +`AUTH_CASDOOR_ISSUER` must point to `auth.`, not the app's root domain: +```bash +ISSUER=$(grep AUTH_CASDOOR_ISSUER .env | cut -d= -f2) +DOMAIN=$(grep LOBEHUB_DOMAIN .env | cut -d= -f2) +[ "$ISSUER" = "https://auth.$DOMAIN" ] || echo "FAIL: issuer should be https://auth.$DOMAIN" +``` + +### 7. SSH key exists +```bash +[ -f ~/.ssh/id_ed25519 ] || echo "FAIL: SSH key not found" +``` + +## Makefile integration + +```makefile +pre-deploy: + @./scripts/validate-env.sh $(ENV) + +# Enforce: plan requires pre-deploy to pass +plan: pre-deploy + cd $(ENV_DIR) && terraform plan -out=tfplan +``` + +## Anti-pattern: deploy-and-pray + +The opposite of pre-deploy validation is the "deploy and see what breaks" cycle: +1. `terraform apply` → fails +2. SSH in to debug → discover error +3. Fix locally → commit → re-apply → fails differently +4. Repeat 5-10 times + +Each cycle takes 3-5 minutes (plan + apply + provisioner). Pre-deploy catches 80% of issues in <5 seconds locally. diff --git a/terraform-skill/references/zero-to-deploy-checklist.md b/terraform-skill/references/zero-to-deploy-checklist.md new file mode 100644 index 00000000..a975c268 --- /dev/null +++ b/terraform-skill/references/zero-to-deploy-checklist.md @@ -0,0 +1,168 @@ +# Zero-to-Deployment Checklist + +A fresh instance with an empty data disk exposes every implicit dependency that production silently relies on. This checklist covers everything that must be explicitly created before services will start. + +## Pre-flight: cloud-init must handle + +These run at OS boot, before Terraform provisioners: + +- [ ] **Mount data disk**: Format if new (`blkid` check), mount to `/data`, add to fstab +- [ ] **Create service directories**: `mkdir -p /data/{service1,service2,...}` — file provisioners fail if target dir doesn't exist +- [ ] **Install Docker + Compose**: Curl installer, enable systemd service +- [ ] **Configure swap**: `fallocate` on data disk (NOT system disk) +- [ ] **SSH hardening**: key-only auth, no password root login +- [ ] **Firewall**: UFW + DOCKER-USER iptables chain +- [ ] **Debconf preseed**: For any package with interactive prompts (iptables-persistent, etc.) +- [ ] **Signal readiness**: Write timestamp to `/data/cloud-init.log` + +## Provisioner ordering + +Terraform provisioners execute in declaration order within a resource, but resources execute in parallel unless `depends_on` is set. + +``` +lobehub_deploy ──────────────────→ channel_sync (depends_on lobehub) + → casdoor_sync (depends_on lobehub) + → minio_sync (depends_on lobehub) + +claude4dev_deploy (depends_on lobehub_deploy) + ├─ wait for cloud-init + ├─ upload source (tarball via file provisioner) + ├─ upload .env (staging variant) + ├─ start stateful (postgres, redis) --no-recreate + ├─ run DB migrations + ├─ build stateless images + ├─ fix volume permissions + ├─ start stateless (relay, api, frontend, gateway) + └─ verify health +``` + +## Database bootstrap + +### PostgreSQL databases + +PostgreSQL `docker-entrypoint-initdb.d` scripts only run when the data directory is empty (first-ever start). On subsequent starts — even if a database doesn't exist — init scripts are skipped. + +**Fix**: Explicitly create databases in provisioner: +```bash +# Wait for postgres healthy +sleep 10 +# Create database if missing (idempotent) +docker exec my-postgres psql -U postgres -tc \ + "SELECT 1 FROM pg_database WHERE datname='mydb'" | grep -q 1 \ + || docker exec my-postgres psql -U postgres -c "CREATE DATABASE mydb;" +``` + +### Schema migrations + +Migrations must be idempotent. Track applied versions: +```bash +PSQL='docker compose exec -T postgres psql -v ON_ERROR_STOP=1 -U myuser -d mydb' + +# Create tracking table +$PSQL -tAc "CREATE TABLE IF NOT EXISTS schema_migrations ( + version TEXT PRIMARY KEY, + applied_at TIMESTAMPTZ DEFAULT now() +)" + +# Apply each migration file in order +for f in migrations/*.sql; do + VER=$(basename $f) + APPLIED=$($PSQL -tAc "SELECT 1 FROM schema_migrations WHERE version='$VER'" | tr -d ' ') + if [ "$APPLIED" = "1" ]; then + echo "Skip: $VER" + else + echo "Apply: $VER" + { echo 'BEGIN;'; cat $f; echo 'COMMIT;'; } | $PSQL + $PSQL -tAc "INSERT INTO schema_migrations(version) VALUES ('$VER') ON CONFLICT DO NOTHING" + fi +done +``` + +## Docker build on remote + +### Proxy mode + +Docker Compose reads build args from `.env` via `${VAR:-default}`. Command-line env vars do NOT override `.env` values for compose interpolation. + +```bash +# WRONG: compose still reads DOCKER_WITH_PROXY_MODE from .env +DOCKER_WITH_PROXY_MODE=disabled docker compose build myapp + +# RIGHT: modify .env so compose reads the correct value +grep -q DOCKER_WITH_PROXY_MODE .env || echo 'DOCKER_WITH_PROXY_MODE=disabled' >> .env +docker compose build myapp +``` + +### Memory management + +Building Docker images while 10+ containers run can OOM on small instances (8GB). Strategy: + +```bash +# Stop non-critical containers to free RAM +cd /data/other-project && docker compose stop search-engine analytics-db || true + +# Build (memory-intensive) +cd /data/myproject && docker compose build myapp + +# Restart stopped containers +cd /data/other-project && docker compose up -d search-engine analytics-db || true +``` + +## Volume permissions + +Containers running as non-root need writable volume directories: + +```bash +# Before docker compose up: +mkdir -p data-dir logs-dir +chown -R 1001:1001 data-dir logs-dir # match container UID +``` + +Find the UID from the Dockerfile: +```dockerfile +RUN adduser -S myuser -u 1001 -G mygroup +USER myuser # runs as uid 1001 +``` + +## Environment-specific .env files + +Production `.env` contains production URLs. Staging needs its own `.env` with: + +| Variable | Production | Staging | +|---|---|---| +| `FRONTEND_URL` | `https://myapp.com` | `https://staging.myapp.com` | +| `CORS_ORIGIN` | `https://myapp.com` | `https://staging.myapp.com` | +| `NEW_API_URL` | `http://api-container:3000` | Same (internal Docker network) | +| `DOCKER_WITH_PROXY_MODE` | `required` (if behind proxy) | `disabled` (direct internet) | + +**Pattern**: Create `.env.staging` alongside `.env`. In Terraform: +```hcl +locals { + env_src = "${local.repo}/.env.staging" # staging-specific +} + +provisioner "file" { + source = local.env_src + destination = "${local.deploy_dir}/.env" +} +``` + +Rsync must exclude `.env` files (otherwise production .env overwrites staging .env): +``` +--exclude=.env --exclude='.env.*' +``` + +## Verification template + +After all services start, verify in the provisioner (not ad-hoc SSH): + +```bash +sleep 20 +echo '=== Service logs ===' +docker logs my-critical-service --tail 20 2>&1 || true +echo '=== All containers ===' +docker ps --format 'table {{.Names}}\t{{.Status}}' 2>&1 || true +# Final gate (only line that can fail) +docker ps --filter name=my-critical-service --format '{{.Status}}' | grep -q healthy \ + || { echo 'FATAL: service unhealthy'; exit 1; } +``` diff --git a/transcript-fixer/SKILL.md b/transcript-fixer/SKILL.md index 4830456c..55e4043a 100644 --- a/transcript-fixer/SKILL.md +++ b/transcript-fixer/SKILL.md @@ -1,150 +1,176 @@ --- name: transcript-fixer -description: Corrects speech-to-text transcription errors in meeting notes, lectures, and interviews using dictionary rules and AI. Learns patterns to build personalized correction databases. Use when working with transcripts containing ASR/STT errors, homophones, or Chinese/English mixed content requiring cleanup. +description: Corrects speech-to-text transcription errors using dictionary rules and AI-powered analysis. Builds personalized correction databases that learn from each fix. Triggers when working with ASR/STT output containing recognition errors, homophones, garbled technical terms, or Chinese/English mixed content. Also triggers on requests to clean up meeting notes, lecture transcripts, interview recordings, or any text produced by speech recognition. Use this skill even when the user just says "fix this transcript" or "clean up these meeting notes" without mentioning ASR specifically. --- # Transcript Fixer -Correct speech-to-text transcription errors through dictionary-based rules, AI-powered corrections, and automatic pattern detection. Build a personalized knowledge base that learns from each correction. - -## When to Use This Skill - -- Correcting ASR/STT errors in meeting notes, lectures, or interviews -- Building domain-specific correction dictionaries -- Fixing Chinese/English homophone errors or technical terminology -- Collaborating on shared correction knowledge bases +Two-phase correction pipeline: deterministic dictionary rules (instant, free) followed by AI-powered error detection. Corrections accumulate in `~/.transcript-fixer/corrections.db`, improving accuracy over time. ## Prerequisites -**Python execution must use `uv`** - never use system Python directly. - -If `uv` is not installed: -```bash -# macOS/Linux -curl -LsSf https://astral.sh/uv/install.sh | sh - -# Windows PowerShell -powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" -``` +All scripts use PEP 723 inline metadata — `uv run` auto-installs dependencies. Requires `uv` ([install guide](https://docs.astral.sh/uv/getting-started/installation/)). ## Quick Start -**Recommended: Use Enhanced Wrapper** (auto-detects API key, opens HTML diff): - ```bash # First time: Initialize database uv run scripts/fix_transcription.py --init -# Process transcript with enhanced UX -uv run scripts/fix_transcript_enhanced.py input.md --output ./corrected +# Single file +uv run scripts/fix_transcription.py --input meeting.md --stage 1 + +# Batch: multiple files in parallel (use shell loop) +for f in /path/to/*.txt; do + uv run scripts/fix_transcription.py --input "$f" --stage 1 +done ``` -The enhanced wrapper automatically: -- Detects GLM API key from shell configs (checks lines near `ANTHROPIC_BASE_URL`) -- Moves output files to specified directory -- Opens HTML visual diff in browser for immediate feedback +After Stage 1, Claude reads the output and fixes remaining ASR errors natively (no API key needed): +1. Read all Stage 1 outputs — read **entire** transcript before proposing corrections (later context disambiguates earlier errors) +2. Identify ASR errors — compile all corrections across files +3. Apply fixes with sed in batch, verify each with diff +4. Finalize: rename `_stage1.md` → `.md`, delete original `.txt` +5. Save stable patterns to dictionary for future reuse -**Alternative: Use Core Script Directly**: +See `references/example_session.md` for a concrete input/output walkthrough. +**Alternative: API batch processing** (for automation without Claude Code): ```bash -# 1. Set API key (if not auto-detected) export GLM_API_KEY="" # From https://open.bigmodel.cn/ +uv run scripts/fix_transcript_enhanced.py input.md --output ./corrected +``` + +## Core Workflow -# 2. Add common corrections (5-10 terms) -uv run scripts/fix_transcription.py --add "错误词" "正确词" --domain general +Two-phase pipeline with persistent learning: -# 3. Run full correction pipeline -uv run scripts/fix_transcription.py --input meeting.md --stage 3 +1. **Initialize** (once): `uv run scripts/fix_transcription.py --init` +2. **Add domain corrections**: `--add "错误词" "正确词" --domain ` +3. **Phase 1 — Dictionary**: `--input file.md --stage 1` (instant, free) +4. **Phase 2 — AI Correction**: Claude reads output and fixes errors natively, or `--stage 3` with `GLM_API_KEY` for API mode +5. **Save stable patterns**: `--add "错误词" "正确词"` after each session +6. **Review learned patterns**: `--review-learned` and `--approve` high-confidence suggestions -# 4. Review learned patterns after 3-5 runs -uv run scripts/fix_transcription.py --review-learned -``` +**Domains**: `general`, `embodied_ai`, `finance`, `medical`, or custom (e.g., `火星加速器`) +**Learning**: Patterns appearing ≥3 times at ≥80% confidence auto-promote from AI to dictionary + +**After fixing, always save reusable corrections to dictionary.** This is the skill's core value — see `references/iteration_workflow.md` for the complete checklist. + +### Dictionary Addition After Fixing + +After native AI correction, review all applied fixes and decide which to save. Use this decision matrix: -**Output files**: -- `*_stage1.md` - Dictionary corrections applied -- `*_stage2.md` - AI corrections applied (final version) -- `*_对比.html` - Visual diff (open in browser for best experience) +| Pattern type | Example | Action | +|-------------|---------|--------| +| Non-word → correct term | 克劳锐→Claude, cloucode→Claude Code | ✅ Add (zero false positive risk) | +| Rare word → correct term | 潜彩→前采, 维星→韦青 | ✅ Add (verify it's not a real word first) | +| Person/company name ASR error | 宋天航→宋天生, 策马攀山→策马看山 | ✅ Add (stable, unique) | +| Common word → context word | 争→蒸, 钱财→前采, 报纸→标品 | ❌ Skip (high false positive risk) | +| Real brand → different brand | Xcode→Claude Code, Clover→Claude | ❌ Skip (real words in other contexts) | -**Generate word-level diff** (recommended for reviewing corrections): +Batch add multiple corrections in one session: ```bash -uv run scripts/generate_word_diff.py original.md corrected.md output.html +uv run scripts/fix_transcription.py --add "错误1" "正确1" --domain tech +uv run scripts/fix_transcription.py --add "错误2" "正确2" --domain business +# Chain with && for efficiency ``` -This creates an HTML file showing word-by-word differences with clear highlighting: -- 🔴 `japanese 3 pro` → 🟢 `Gemini 3 Pro` (complete word replacements) -- Easy to spot exactly what changed without character-level noise +## False Positive Prevention -## Example Session +Adding wrong dictionary rules silently corrupts future transcripts. **Read `references/false_positive_guide.md` before adding any correction rule**, especially for short words (≤2 chars) or common Chinese words that appear correctly in normal text. -**Input transcript** (`meeting.md`): -``` -今天我们讨论了巨升智能的最新进展。 -股价系统需要优化，目前性能不够好。 -``` +## Native AI Correction (Default Mode) -**After Stage 1** (`meeting_stage1.md`): -``` -今天我们讨论了具身智能的最新进展。 ← "巨升"→"具身" corrected -股价系统需要优化,目前性能不够好。 ← Unchanged (not in dictionary) -``` +When running inside Claude Code, use Claude's own language understanding for Phase 2: -**After Stage 2** (`meeting_stage2.md`): -``` -今天我们讨论了具身智能的最新进展。 -框架系统需要优化，目前性能不够好。 ← "股价"→"框架" corrected by AI -``` +1. Run Stage 1 (dictionary) on all files (parallel if multiple) +2. Verify Stage 1 — diff original vs output. If dictionary introduced false positives, work from the **original** file +3. Read **all** Stage 1 outputs fully before proposing any corrections — later context often disambiguates earlier errors. For large files (>10k tokens), read in chunks but finish the entire file before identifying errors +4. Identify ASR errors per file — classify by confidence: + - **High confidence** (apply directly): non-words, obvious garbling, product name variants + - **Medium confidence** (present for review): context-dependent homophones, person names +5. Apply fixes efficiently: + - **Global replacements** (unique non-words like "克劳锐"→"Claude"): use `sed -i ''` with `-e` flags, multiple patterns in one command + - **Context-dependent** (common words like "争"→"蒸" only in distillation context): use sed with longer context phrases for uniqueness, or Edit tool +6. Verify with diff: `diff original.txt corrected_stage1.md` +7. Finalize files: rename `*_stage1.md` → `*.md`, delete original `.txt` +8. Save stable patterns to dictionary (see "Dictionary Addition" below) +9. Remove false positives if Stage 1 had any -**Learned pattern detected:** -``` -✓ Detected: "股价" → "框架" (confidence: 85%, count: 1) - Run --review-learned after 2 more occurrences to approve -``` +### Common ASR Error Patterns -## Core Workflow +AI product names are frequently garbled. These patterns recur across transcripts: -Three-stage pipeline stores corrections in `~/.transcript-fixer/corrections.db`: +| Correct term | Common ASR variants | +|-------------|-------------------| +| Claude | cloud, Clou, calloc, 克劳锐, Clover, color | +| Claude Code | cloud code, Xcode, call code, cloucode, cloudcode, color code | +| Claude Agent SDK | cloud agent SDK | +| Opus | Opaas | +| Vibe Coding | web coding, Web coding | +| GitHub | get Hub, Git Hub | +| prototype | Pre top | -1. **Initialize** (first time): `uv run scripts/fix_transcription.py --init` -2. **Add domain corrections**: `--add "错误词" "正确词" --domain ` -3. **Process transcript**: `--input file.md --stage 3` -4. **Review learned patterns**: `--review-learned` and `--approve` high-confidence suggestions +Person names and company names also produce consistent ASR errors across sessions — always add confirmed name corrections to the dictionary. + +### Efficient Batch Fix Strategy -**Stages**: Dictionary (instant, free) → AI via GLM API (parallel) → Full pipeline -**Domains**: `general`, `embodied_ai`, `finance`, `medical`, or custom names including Chinese (e.g., `火星加速器`, `具身智能`) -**Learning**: Patterns appearing ≥3 times at ≥80% confidence move from AI to dictionary +When fixing multiple files (e.g., 5 transcripts from one day): -See `references/workflow_guide.md` for detailed workflows, `references/script_parameters.md` for complete CLI reference, and `references/team_collaboration.md` for collaboration patterns. +1. **Stage 1 in parallel**: run all files through dictionary at once +2. **Read all files first**: build a mental model of speakers, topics, and recurring terms before fixing anything +3. **Compile a global correction list**: many errors repeat across files from the same session (same speakers, same topics) +4. **Apply global corrections first** (sed with multiple `-e` flags), then per-file context-dependent fixes +5. **Verify all diffs**, finalize all files, then do one dictionary addition pass -## Critical Workflow: Dictionary Iteration +### Enhanced Capabilities (Native Mode Only) -**MUST save corrections after each fix.** This is the skill's core value. +- **Intelligent paragraph breaks**: Add `\n\n` at logical topic transitions +- **Filler word reduction**: "这个这个这个" → "这个" +- **Interactive review**: Corrections confirmed before applying +- **Context-aware judgment**: Full document context resolves ambiguous errors -After fixing errors manually, immediately save to dictionary: +### When to Use API Mode Instead + +Use `GLM_API_KEY` + Stage 3 for batch processing, standalone usage without Claude Code, or reproducible automated processing. + +### Legacy Fallback + +When the script outputs `[CLAUDE_FALLBACK]` (GLM API error), switch to native mode automatically. + +## Utility Scripts + +**Timestamp repair**: ```bash -uv run scripts/fix_transcription.py --add "错误词" "正确词" --domain general +uv run scripts/fix_transcript_timestamps.py meeting.txt --in-place ``` -See `references/iteration_workflow.md` for complete iteration guide with checklist. +**Split transcript into sections** (rebase each to `00:00:00`): +```bash +uv run scripts/split_transcript_sections.py meeting.txt \ + --first-section-name "课前聊天" \ + --section "正式上课::好，无缝切换嘛。" \ + --rebase-to-zero +``` -## AI Fallback Strategy +**Word-level diff** (recommended for reviewing corrections): +```bash +uv run scripts/generate_word_diff.py original.md corrected.md output.html +``` -When GLM API is unavailable (503, network issues), the script outputs `[CLAUDE_FALLBACK]` marker. +## Output Files -Claude Code should then: -1. Analyze the text directly for ASR errors -2. Fix using Edit tool -3. **MUST save corrections to dictionary** with `--add` +- `*_stage1.md` — Dictionary corrections applied +- `*_corrected.txt` — Final version (native mode) or `*_stage2.md` (API mode) +- `*_对比.html` — Visual diff (open in browser) ## Database Operations -**MUST read `references/database_schema.md` before any database operations.** +**Read `references/database_schema.md` before any database operations.** -Quick reference: ```bash -# View all corrections sqlite3 ~/.transcript-fixer/corrections.db "SELECT * FROM active_corrections;" - -# Check schema version sqlite3 ~/.transcript-fixer/corrections.db "SELECT value FROM system_config WHERE key='schema_version';" ``` @@ -153,30 +179,40 @@ sqlite3 ~/.transcript-fixer/corrections.db "SELECT value FROM system_config WHER | Stage | Description | Speed | Cost | |-------|-------------|-------|------| | 1 | Dictionary only | Instant | Free | -| 2 | AI only | ~10s | API calls | -| 3 | Full pipeline | ~10s | API calls | +| 1 + Native | Dictionary + Claude AI (default) | ~1min | Free | +| 3 | Dictionary + API AI + diff report | ~10s | API calls | ## Bundled Resources **Scripts:** -- `ensure_deps.py` - Initialize shared virtual environment (run once, optional) -- `fix_transcript_enhanced.py` - Enhanced wrapper (recommended for interactive use) -- `fix_transcription.py` - Core CLI (for automation) -- `generate_word_diff.py` - Generate word-level diff HTML for reviewing corrections -- `examples/bulk_import.py` - Bulk import example +- `fix_transcription.py` — Core CLI (dictionary, add, audit, learning) +- `fix_transcript_enhanced.py` — Enhanced wrapper for interactive use +- `fix_transcript_timestamps.py` — Timestamp normalization and repair +- `generate_word_diff.py` — Word-level diff HTML generation +- `split_transcript_sections.py` — Split transcript by marker phrases **References** (load as needed): -- **Critical**: `database_schema.md` (read before DB operations), `iteration_workflow.md` (dictionary iteration best practices) -- Getting started: `installation_setup.md`, `glm_api_setup.md`, `workflow_guide.md` -- Daily use: `quick_reference.md`, `script_parameters.md`, `dictionary_guide.md` -- Advanced: `sql_queries.md`, `file_formats.md`, `architecture.md`, `best_practices.md` -- Operations: `troubleshooting.md`, `team_collaboration.md` +- **Safety**: `false_positive_guide.md` (read before adding rules), `database_schema.md` (read before DB ops) +- **Workflow**: `iteration_workflow.md`, `workflow_guide.md`, `example_session.md` +- **CLI**: `quick_reference.md`, `script_parameters.md` +- **Advanced**: `dictionary_guide.md`, `sql_queries.md`, `architecture.md`, `best_practices.md` +- **Operations**: `troubleshooting.md`, `installation_setup.md`, `glm_api_setup.md`, `team_collaboration.md` ## Troubleshooting -Verify setup health with `uv run scripts/fix_transcription.py --validate`. Common issues: -- Missing database → Run `--init` -- Missing API key → `export GLM_API_KEY=""` (obtain from https://open.bigmodel.cn/) -- Permission errors → Check `~/.transcript-fixer/` ownership +`uv run scripts/fix_transcription.py --validate` checks setup health. See `references/troubleshooting.md` for detailed resolution. + +## Next Step: Structure into Meeting Minutes -See `references/troubleshooting.md` for detailed error resolution and `references/glm_api_setup.md` for API configuration. +After correcting a transcript, if the content is from a meeting, lecture, or interview, suggest structuring it: + +``` +Transcript corrected: [N] errors fixed, saved to [output_path]. + +Want to turn this into structured meeting minutes with decisions and action items? + +Options: +A) Yes — run /meeting-minutes-taker (Recommended for meetings/lectures) +B) Export as PDF — run /pdf-creator on the corrected text +C) No thanks — the corrected transcript is all I need +``` diff --git a/transcript-fixer/references/example_session.md b/transcript-fixer/references/example_session.md new file mode 100644 index 00000000..59017217 --- /dev/null +++ b/transcript-fixer/references/example_session.md @@ -0,0 +1,25 @@ +# Example Session + +## Input transcript (`meeting.md`) +``` +今天我们讨论了巨升智能的最新进展。 +股价系统需要优化，目前性能不够好。 +``` + +## After Stage 1 (`meeting_stage1.md`) +``` +今天我们讨论了具身智能的最新进展。 ← "巨升"→"具身" corrected +股价系统需要优化，目前性能不够好。 ← Unchanged (not in dictionary) +``` + +## After Stage 2 (`meeting_stage2.md`) +``` +今天我们讨论了具身智能的最新进展。 +框架系统需要优化，目前性能不够好。 ← "股价"→"框架" corrected by AI +``` + +## Learned pattern detected +``` +✓ Detected: "股价" → "框架" (confidence: 85%, count: 1) + Run --review-learned after 2 more occurrences to approve +``` diff --git a/transcript-fixer/references/false_positive_guide.md b/transcript-fixer/references/false_positive_guide.md new file mode 100644 index 00000000..04ab1615 --- /dev/null +++ b/transcript-fixer/references/false_positive_guide.md @@ -0,0 +1,39 @@ +# False Positive Prevention Guide + +Dictionary-based corrections are powerful but dangerous. Adding the wrong rule silently corrupts every future transcript. The `--add` command runs safety checks automatically, but you must understand the risks. + +## What is safe to add + +- **ASR-specific gibberish**: "巨升智能" -> "具身智能" (no real word sounds like "巨升智能") +- **Long compound errors**: "语音是别" -> "语音识别" (4+ chars, unlikely to collide) +- **English transliteration errors**: "japanese 3 pro" -> "Gemini 3 Pro" + +## What is NEVER safe to add + +- **Common Chinese words**: "仿佛", "正面", "犹豫", "传说", "增加", "教育" -- these appear correctly in normal text. Replacing them corrupts transcripts from better ASR models. +- **Words <=2 characters**: Almost any 2-char Chinese string is a valid word or part of one. "线数" inside "产线数据" becomes "产线束据". +- **Both sides are real words**: "仿佛->反复", "犹豫->抑郁" -- both forms are valid Chinese. The "error" is only an error for one specific ASR model. + +## When in doubt, use a context rule instead + +Context rules use regex patterns that match only in specific surroundings, avoiding false positives: +```bash +# Instead of: --add "线数" "线束" +# Use a context rule in the database: +sqlite3 ~/.transcript-fixer/corrections.db "INSERT INTO context_rules (pattern, replacement, description, priority) VALUES ('(?线束 (not inside 产线数据)', 10);" +``` + +## Auditing the dictionary + +Run `--audit` periodically to scan all rules for false positive risks: +```bash +uv run scripts/fix_transcription.py --audit +uv run scripts/fix_transcription.py --audit --domain manufacturing +``` + +## Forcing a risky addition + +If you understand the risks and still want to add a flagged rule: +```bash +uv run scripts/fix_transcription.py --add "仿佛" "反复" --domain general --force +``` diff --git a/transcript-fixer/references/iteration_workflow.md b/transcript-fixer/references/iteration_workflow.md index ba94a462..17383118 100644 --- a/transcript-fixer/references/iteration_workflow.md +++ b/transcript-fixer/references/iteration_workflow.md @@ -16,7 +16,7 @@ The core value of transcript-fixer is building a personalized correction diction └─────────────────────────────────────────────────┘ ``` -**Key principle**: Every correction you make should be saved to the dictionary. This transforms one-time work into permanent value. +**Key principle**: Every stable, reusable ASR correction you make should be saved to the dictionary. This transforms one-time work into permanent value without polluting the database. ## Workflow Checklist @@ -34,7 +34,7 @@ Correction Progress: ## Save Corrections Immediately -After fixing any transcript, save corrections: +After fixing any transcript, save stable corrections: ```bash # Single correction @@ -122,3 +122,12 @@ Patterns appearing ≥3 times at ≥80% confidence are suggested for review. 3. **Use domains**: Organize corrections by topic for better precision 4. **Verify**: Always run --list to confirm saves 5. **Review suggestions**: Periodically check --review-learned for auto-detected patterns + +## What NOT to Save to Dictionary + +Do **not** save these as reusable dictionary entries: + +- Full-sentence deletions +- One-off section headers or meeting-specific boilerplate +- Context-only disambiguations such as `cloud -> Claude` when `cloud` can also be legitimate +- File-local cleanup after section splitting or timestamp rebasing diff --git a/transcript-fixer/references/script_parameters.md b/transcript-fixer/references/script_parameters.md index a5537abc..6cbeab3d 100644 --- a/transcript-fixer/references/script_parameters.md +++ b/transcript-fixer/references/script_parameters.md @@ -9,6 +9,8 @@ Detailed command-line parameters and usage examples for transcript-fixer Python - [Correction Management](#correction-management) - [Correction Workflow](#correction-workflow) - [Learning Commands](#learning-commands) +- [fix_transcript_timestamps.py](#fix_transcript_timestampspy) - Normalize/repair speaker timestamps +- [split_transcript_sections.py](#split_transcript_sectionspy) - Split transcript into named sections - [diff_generator.py](#diffgeneratorpy) - Generate comparison reports - [Common Workflows](#common-workflows) - [Exit Codes](#exit-codes) @@ -74,6 +76,59 @@ python scripts/fix_transcription.py --input meeting.md --stage 3 --output ./corr - `2` - GLM_API_KEY environment variable not set (Stage 2 or 3 only) - `3` - API request failed +## fix_transcript_timestamps.py + +Normalize speaker timestamp lines such as `天生 00:21` or `Speaker 7 01:31:10`. + +### Syntax + +```bash +python scripts/fix_transcript_timestamps.py [--output FILE | --in-place | --check] +``` + +### Key Parameters + +- `--format {hhmmss,preserve}`: output timestamp style +- `--rebase-to-zero`: reset the first detected speaker timestamp to `00:00:00` +- `--rollover-backjump-seconds`: threshold for treating `59:58 -> 00:05` as a new hour +- `--jitter-seconds`: tolerated small backward jitter before flagging anomaly + +### Usage Examples + +```bash +# Normalize mixed MM:SS / HH:MM:SS +python scripts/fix_transcript_timestamps.py meeting.txt --in-place + +# Rebase a split transcript so it starts at 00:00:00 +python scripts/fix_transcript_timestamps.py workshop-class.txt --in-place --rebase-to-zero + +# Only inspect anomalies, do not write +python scripts/fix_transcript_timestamps.py meeting.txt --check +``` + +## split_transcript_sections.py + +Split a transcript into named sections using marker phrases. Useful for workshop transcripts that include setup chat, class, and debrief in one file. + +### Syntax + +```bash +python scripts/split_transcript_sections.py \ + --first-section-name \ + --section "Name::Marker" \ + --section "Name::Marker" +``` + +### Usage Example + +```bash +python scripts/split_transcript_sections.py workshop.txt \ + --first-section-name "课前聊天" \ + --section "正式上课::好，无缝切换嘛。对。那个曹总连上了吗？那个网页。" \ + --section "课后复盘::我们复盘一下。" \ + --rebase-to-zero +``` + ## generate_diff_report.py Multi-format diff report generator for comparing correction stages. diff --git a/transcript-fixer/references/workflow_guide.md b/transcript-fixer/references/workflow_guide.md index b6132b6d..5dbc7dad 100644 --- a/transcript-fixer/references/workflow_guide.md +++ b/transcript-fixer/references/workflow_guide.md @@ -17,6 +17,7 @@ Detailed step-by-step workflows for transcript correction and management. - [5. Stage-by-Stage Execution](#5-stage-by-stage-execution) - [6. Context-Aware Rules](#6-context-aware-rules) - [7. Diff Report Generation](#7-diff-report-generation) + - [8. Workshop Transcript Split + Timestamp Rebase](#8-workshop-transcript-split--timestamp-rebase) - [Batch Processing](#batch-processing) - [Process Multiple Files](#process-multiple-files) - [Parallel Processing](#parallel-processing) @@ -400,6 +401,30 @@ See `file_formats.md` for context_rules schema. See `script_parameters.md` for advanced diff options. +### 8. Workshop Transcript Split + Timestamp Rebase + +**Goal**: Split a long workshop transcript into sections such as setup chat, class, and debrief, then make each section start from `00:00:00`. + +**Steps**: + +1. **Correct transcript text first** (dictionary + AI/manual review) +2. **Pick marker phrases** for each section boundary +3. **Split and rebase**: + +```bash +uv run scripts/split_transcript_sections.py workshop.txt \ + --first-section-name "课前聊天" \ + --section "正式上课::好，无缝切换嘛。对。那个曹总连上了吗？那个网页。" \ + --section "课后复盘::我们复盘一下。" \ + --rebase-to-zero +``` + +4. **If you already split the files**, rebase a single file directly: + +```bash +uv run scripts/fix_transcript_timestamps.py class.txt --in-place --rebase-to-zero +``` + ## Batch Processing ### Process Multiple Files diff --git a/transcript-fixer/scripts/cli/__init__.py b/transcript-fixer/scripts/cli/__init__.py index 6e6e5b0c..ca8d6207 100644 --- a/transcript-fixer/scripts/cli/__init__.py +++ b/transcript-fixer/scripts/cli/__init__.py @@ -9,6 +9,7 @@ from .commands import ( cmd_init, cmd_add_correction, + cmd_audit, cmd_list_corrections, cmd_run_correction, cmd_review_learned, @@ -25,6 +26,7 @@ __all__ = [ 'cmd_init', 'cmd_add_correction', + 'cmd_audit', 'cmd_list_corrections', 'cmd_run_correction', 'cmd_review_learned', diff --git a/transcript-fixer/scripts/cli/argument_parser.py b/transcript-fixer/scripts/cli/argument_parser.py index 086d67f9..a418fd08 100644 --- a/transcript-fixer/scripts/cli/argument_parser.py +++ b/transcript-fixer/scripts/cli/argument_parser.py @@ -37,12 +37,24 @@ def create_argument_parser() -> argparse.ArgumentParser: dest="add_correction", help="Add correction" ) + parser.add_argument( + "--force", + action="store_true", + default=False, + help="Force --add even when safety checks detect risks (common word, substring collision)" + ) parser.add_argument( "--list", action="store_true", dest="list_corrections", help="List all corrections" ) + parser.add_argument( + "--audit", + action="store_true", + dest="audit_dictionary", + help="Audit all active corrections for false positive risks (common words, short text, substring collisions)" + ) # Correction workflow parser.add_argument( @@ -62,8 +74,8 @@ def create_argument_parser() -> argparse.ArgumentParser: ) parser.add_argument( "--domain", "-d", - default="general", - help="Correction domain" + default=None, + help="Correction domain (default: all domains)" ) # Learning commands diff --git a/transcript-fixer/scripts/cli/commands.py b/transcript-fixer/scripts/cli/commands.py index 4caa832c..dbac6553 100644 --- a/transcript-fixer/scripts/cli/commands.py +++ b/transcript-fixer/scripts/cli/commands.py @@ -43,26 +43,106 @@ def cmd_init(args: argparse.Namespace) -> None: def cmd_add_correction(args: argparse.Namespace) -> None: - """Add a single correction""" + """Add a single correction with safety checks""" service = _get_service() + force = getattr(args, 'force', False) try: - service.add_correction(args.from_text, args.to_text, args.domain) - print(f"✅ Added: '{args.from_text}' → '{args.to_text}' (domain: {args.domain})") + service.add_correction( + args.from_text, args.to_text, args.domain, force=force, + ) + print(f"Added: '{args.from_text}' -> '{args.to_text}' (domain: {args.domain})") except Exception as e: - print(f"❌ Error: {e}") + print(f"Error: {e}", file=sys.stderr) sys.exit(1) +def cmd_audit(args: argparse.Namespace) -> None: + """Audit all active corrections for false positive risks""" + service = _get_service() + domain = getattr(args, 'domain', None) + + print(f"\nAuditing corrections" + (f" (domain: {domain})" if domain else " (all domains)") + "...") + print("=" * 70) + + issues = service.audit_dictionary(domain) + + if not issues: + corrections = service.get_corrections(domain) + print(f"\nAll {len(corrections)} corrections passed safety checks.") + return + + # Categorize + error_count = 0 + warning_count = 0 + for from_text, warnings in issues.items(): + for w in warnings: + if w.level == "error": + error_count += 1 + else: + warning_count += 1 + + corrections = service.get_corrections(domain) + print(f"\nScanned {len(corrections)} corrections. " + f"Found issues in {len(issues)} rules:") + print(f" Errors: {error_count} (should be removed or converted to context rules)") + print(f" Warnings: {warning_count} (review recommended)") + print() + + # Print details grouped by severity + for severity in ["error", "warning"]: + label = "ERRORS" if severity == "error" else "WARNINGS" + relevant = { + ft: [w for w in ws if w.level == severity] + for ft, ws in issues.items() + } + relevant = {ft: ws for ft, ws in relevant.items() if ws} + + if not relevant: + continue + + print(f"--- {label} ({len(relevant)} rules) ---") + for from_text, warnings in sorted(relevant.items()): + to_text = corrections.get(from_text, "?") + print(f"\n '{from_text}' -> '{to_text}'") + for w in warnings: + print(f" [{w.category}] {w.message}") + print(f" Suggestion: {w.suggestion}") + print() + + if error_count > 0: + print( + f"ACTION REQUIRED: {error_count} error(s) found. These rules are " + f"actively causing false positives and should be removed or " + f"converted to context rules." + ) + print( + f"To remove a rule: " + f"sqlite3 ~/.transcript-fixer/corrections.db " + f"\"UPDATE corrections SET is_active=0 WHERE from_text='...';\"" + ) + + def cmd_list_corrections(args: argparse.Namespace) -> None: """List all corrections""" service = _get_service() corrections = service.get_corrections(args.domain) - print(f"\n📋 Corrections (domain: {args.domain})") + if args.domain: + header = f"domain: {args.domain}, {len(corrections)} total" + else: + header = f"all domains, {len(corrections)} total" + + print(f"\n📋 Corrections ({header})") print("=" * 60) - for wrong, correct in sorted(corrections.items()): - print(f" '{wrong}' → '{correct}'") - print(f"\nTotal: {len(corrections)} corrections\n") + + if args.domain: + for wrong, correct in sorted(corrections.items()): + print(f" '{wrong}' → '{correct}'") + else: + all_corrections = service.repository.get_all_corrections(active_only=True) + for c in all_corrections: + print(f" [{c.domain}] '{c.from_text}' → '{c.to_text}'") + print() def cmd_run_correction(args: argparse.Namespace) -> None: @@ -83,12 +163,23 @@ def cmd_run_correction(args: argparse.Namespace) -> None: # Load corrections and rules corrections = service.get_corrections(args.domain) context_rules = service.load_context_rules() + domain_stats = service.get_domain_stats() # Read input file print(f"📖 Reading: {input_path.name}") with open(input_path, 'r', encoding='utf-8') as f: original_text = f.read() - print(f" File size: {len(original_text):,} characters\n") + print(f" File size: {len(original_text):,} characters") + + # Show domain loading info + if args.domain: + print(f"📚 Loaded {len(corrections)} corrections (domain: {args.domain})") + elif domain_stats: + parts = ", ".join(f"{d}: {n}" for d, n in sorted(domain_stats.items())) + print(f"📚 Loaded {len(corrections)} corrections ({parts})") + else: + print(f"📚 No corrections in database") + print() # Stage 1: Dictionary corrections stage1_changes = [] @@ -109,7 +200,17 @@ def cmd_run_correction(args: argparse.Namespace) -> None: stage1_file = output_dir / f"{input_path.stem}_stage1.md" with open(stage1_file, 'w', encoding='utf-8') as f: f.write(stage1_text) - print(f"💾 Saved: {stage1_file.name}\n") + print(f"💾 Saved: {stage1_file.name}") + + # Hint when 0 corrections and other domains have rules + if summary['total_changes'] == 0 and args.domain and domain_stats: + other = {d: n for d, n in domain_stats.items() if d != args.domain} + if other: + parts = ", ".join(f"{d} ({n})" for d, n in sorted(other.items())) + total = sum(other.values()) + print(f"hint: no rules in domain '{args.domain}'. Available: {parts}") + print(f"hint: run without --domain to use all {total} rules") + print() # Stage 2: AI corrections stage2_changes = [] diff --git a/transcript-fixer/scripts/core/correction_service.py b/transcript-fixer/scripts/core/correction_service.py index 4d53c50c..daf70cff 100644 --- a/transcript-fixer/scripts/core/correction_service.py +++ b/transcript-fixer/scripts/core/correction_service.py @@ -12,6 +12,7 @@ import re import os +import sys import logging from pathlib import Path from typing import Dict, List, Optional, Tuple @@ -23,6 +24,10 @@ DatabaseError ) +# Import safety check for common words +sys.path.insert(0, str(Path(__file__).parent.parent)) +from utils.common_words import check_correction_safety, audit_corrections, SafetyWarning + logger = logging.getLogger(__name__) @@ -178,10 +183,15 @@ def add_correction( domain: str = "general", source: str = "manual", confidence: float = 1.0, - notes: Optional[str] = None + notes: Optional[str] = None, + force: bool = False, ) -> int: """ - Add a correction with full validation. + Add a correction with full validation and safety checks. + + Safety checks detect common Chinese words and substring collision + risks that would cause false positives. Pass force=True to bypass + (errors become warnings printed to stderr). Args: from_text: Original (incorrect) text @@ -190,12 +200,13 @@ def add_correction( source: Origin of correction confidence: Confidence score notes: Optional notes + force: If True, downgrade safety errors to warnings Returns: ID of inserted correction Raises: - ValidationError: If validation fails + ValidationError: If validation or safety check fails """ # Comprehensive validation self.validate_correction_text(from_text, "from_text") @@ -210,6 +221,34 @@ def add_correction( f"from_text and to_text are identical: '{from_text}'" ) + # Safety check: detect common words and substring collisions + safety_warnings = check_correction_safety(from_text, to_text, strict=True) + + if safety_warnings: + errors = [w for w in safety_warnings if w.level == "error"] + warns = [w for w in safety_warnings if w.level == "warning"] + + if errors and not force: + # Block the addition + msg_parts = [] + for w in errors: + msg_parts.append(f"[{w.category}] {w.message}") + msg_parts.append(f" Suggestion: {w.suggestion}") + raise ValidationError( + f"Safety check BLOCKED adding '{from_text}' -> '{to_text}':\n" + + "\n".join(msg_parts) + + "\n\nUse --force to override (at your own risk)." + ) + + # Print warnings (errors downgraded by --force, or genuine warnings) + all_to_print = errors + warns if force else warns + if all_to_print: + for w in all_to_print: + prefix = "FORCED" if w.level == "error" else "WARNING" + logger.warning( + f"[{prefix}] [{w.category}] {w.message} | {w.suggestion}" + ) + # Get current user added_by = os.getenv("USER") or os.getenv("USERNAME") or "unknown" @@ -382,6 +421,14 @@ def export_corrections(self, domain: str = "general") -> Dict[str, str]: # ==================== Statistics and Reporting ==================== + def get_domain_stats(self) -> Dict[str, int]: + """Get count of active corrections per domain.""" + all_corrections = self.repository.get_all_corrections(active_only=True) + stats: Dict[str, int] = {} + for c in all_corrections: + stats[c.domain] = stats.get(c.domain, 0) + 1 + return stats + def get_statistics(self, domain: Optional[str] = None) -> Dict[str, any]: """ Get correction statistics. @@ -423,6 +470,31 @@ def get_statistics(self, domain: Optional[str] = None) -> Dict[str, any]: return stats + # ==================== Audit Operations ==================== + + def audit_dictionary( + self, + domain: Optional[str] = None, + ) -> Dict[str, List[SafetyWarning]]: + """ + Audit all active corrections for safety issues. + + Scans every rule and flags: + - from_text that is a common Chinese word (false positive risk) + - from_text that is <= 2 characters (high collision risk) + - from_text that appears as substring in common words (collateral damage) + - Both from_text and to_text being common words (bidirectional risk) + + Args: + domain: Optional domain filter (None = all domains) + + Returns: + Dict mapping from_text to list of SafetyWarnings. + Only entries with issues are included. + """ + corrections = self.get_corrections(domain) + return audit_corrections(corrections) + # ==================== Helper Methods ==================== def _detect_conflicts( diff --git a/transcript-fixer/scripts/core/dictionary_processor.py b/transcript-fixer/scripts/core/dictionary_processor.py index 15a95863..6a1907e7 100644 --- a/transcript-fixer/scripts/core/dictionary_processor.py +++ b/transcript-fixer/scripts/core/dictionary_processor.py @@ -14,9 +14,17 @@ from __future__ import annotations import re +import sys +import logging +from pathlib import Path from typing import Dict, List, Tuple from dataclasses import dataclass +sys.path.insert(0, str(Path(__file__).parent.parent)) +from utils.common_words import ALL_COMMON_WORDS + +logger = logging.getLogger(__name__) + @dataclass class Change: @@ -96,7 +104,16 @@ def _apply_context_rules(self, text: str) -> Tuple[str, List[Change]]: return corrected, changes def _apply_dictionary(self, text: str) -> Tuple[str, List[Change]]: - """Apply simple dictionary replacements""" + """ + Apply dictionary replacements with substring safety checks. + + Safety layers (applied in order at each match site): + 1. Superset check: if to_text already exists at the match position, + skip to prevent duplication (e.g., "金流"→"现金流" inside "现金流"). + This applies to ALL rules regardless of length. + 2. Boundary check (short rules only, <=3 chars): if the match is inside + a longer common word, skip to prevent collateral damage. + """ changes = [] corrected = text @@ -104,32 +121,167 @@ def _apply_dictionary(self, text: str) -> Tuple[str, List[Change]]: if wrong not in corrected: continue - # Find all occurrences - occurrences = [] - start = 0 - while True: - pos = corrected.find(wrong, start) - if pos == -1: - break - line_num = corrected[:pos].count('\n') + 1 - occurrences.append(line_num) - start = pos + len(wrong) - - # Track changes - for line_num in occurrences: - changes.append(Change( - line_number=line_num, - from_text=wrong, - to_text=correct, - rule_type="dictionary", - rule_name="corrections_dict" - )) - - # Apply replacement - corrected = corrected.replace(wrong, correct) + # All rules go through position-aware replacement to get + # the superset check. Short rules additionally get the + # boundary check against common words. + needs_boundary_check = len(wrong) <= 3 + corrected, new_changes = self._apply_with_safety_checks( + corrected, wrong, correct, needs_boundary_check, + ) + changes.extend(new_changes) return corrected, changes + def _find_occurrences(self, text: str, target: str) -> List[int]: + """Find all line numbers where target appears in text.""" + occurrences = [] + start = 0 + while True: + pos = text.find(target, start) + if pos == -1: + break + line_num = text[:pos].count('\n') + 1 + occurrences.append(line_num) + start = pos + len(target) + return occurrences + + def _apply_with_safety_checks( + self, + text: str, + wrong: str, + correct: str, + check_boundaries: bool, + ) -> Tuple[str, List[Change]]: + """ + Apply replacement at each match position with two safety layers: + + 1. Superset check (all rules): When to_text contains from_text + (e.g., "金流"→"现金流"), check if the surrounding text already + forms to_text. If so, skip — the text is already correct. + + 2. Boundary check (short rules only): Check if the match is inside + a longer common word (e.g., "天差" inside "天差地别"). + """ + changes = [] + result_parts = [] + search_start = 0 + + while search_start < len(text): + pos = text.find(wrong, search_start) + if pos == -1: + result_parts.append(text[search_start:]) + break + + # Safety layer 1: superset check. + # If to_text contains from_text, the replacement could create + # duplication. Check if to_text already exists at this position. + if self._already_corrected(text, pos, wrong, correct): + result_parts.append(text[search_start:pos + len(wrong)]) + search_start = pos + len(wrong) + logger.debug( + f"Skipped '{wrong}' at pos {pos}: " + f"already corrected ('{correct}' present)" + ) + continue + + # Safety layer 2: boundary check (short rules only). + if check_boundaries and self._is_inside_longer_word( + text, pos, wrong + ): + result_parts.append(text[search_start:pos + len(wrong)]) + search_start = pos + len(wrong) + logger.debug( + f"Skipped '{wrong}' at pos {pos}: part of longer word" + ) + continue + + # Safe to replace + line_num = text[:pos].count('\n') + 1 + changes.append(Change( + line_number=line_num, + from_text=wrong, + to_text=correct, + rule_type="dictionary", + rule_name="corrections_dict" + )) + + result_parts.append(text[search_start:pos]) + result_parts.append(correct) + search_start = pos + len(wrong) + + return "".join(result_parts), changes + + @staticmethod + def _already_corrected( + text: str, pos: int, from_text: str, to_text: str + ) -> bool: + """ + Check if to_text already exists at the match position, meaning + the text is already in the corrected form. + + This catches the case where from_text is a substring of to_text + (e.g., "金流" is inside "现金流"). If the surrounding text already + forms "现金流", replacing "金流" would produce "现现金流". + + Returns True if the replacement should be skipped. + """ + if from_text not in to_text: + # to_text doesn't contain from_text, so no superset risk. + return False + + to_len = len(to_text) + from_len = len(from_text) + + # Find all positions where from_text appears inside to_text. + # For each, check if the surrounding text matches to_text. + offset = 0 + while True: + idx = to_text.find(from_text, offset) + if idx == -1: + break + + # If to_text were at text position (pos - idx), from_text at pos + # would be the substring starting at idx within to_text. + candidate_start = pos - idx + candidate_end = candidate_start + to_len + + if (candidate_start >= 0 + and candidate_end <= len(text) + and text[candidate_start:candidate_end] == to_text): + return True + + offset = idx + 1 + + return False + + @staticmethod + def _is_inside_longer_word(text: str, pos: int, match: str) -> bool: + """ + Check if the match at `pos` is embedded inside a longer common word. + + Looks at a window around the match and checks all possible substrings + of that window against the common words set. + """ + match_len = len(match) + # Check windows of 2 to 5 characters that overlap with the match + max_word_len = 5 + window_start = max(0, pos - (max_word_len - 1)) + window_end = min(len(text), pos + match_len + (max_word_len - 1)) + window = text[window_start:window_end] + + # Position of the match within the window + match_offset = pos - window_start + + # Check all substrings that contain the match position + for length in range(match_len + 1, min(max_word_len + 1, len(window) + 1)): + for start in range(max(0, match_offset + match_len - length), + min(match_offset + 1, len(window) - length + 1)): + substr = window[start:start + length] + if substr != match and substr in ALL_COMMON_WORDS: + return True + + return False + def get_summary(self, changes: List[Change]) -> Dict[str, int]: """Generate summary statistics""" summary = { diff --git a/transcript-fixer/scripts/fix_transcript_enhanced.py b/transcript-fixer/scripts/fix_transcript_enhanced.py index b3ca4582..594baf30 100755 --- a/transcript-fixer/scripts/fix_transcript_enhanced.py +++ b/transcript-fixer/scripts/fix_transcript_enhanced.py @@ -1,4 +1,11 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "httpx>=0.24.0", +# "filelock>=3.13.0", +# ] +# /// """ Enhanced transcript fixer wrapper with improved user experience. diff --git a/transcript-fixer/scripts/fix_transcript_timestamps.py b/transcript-fixer/scripts/fix_transcript_timestamps.py new file mode 100644 index 00000000..19664e26 --- /dev/null +++ b/transcript-fixer/scripts/fix_transcript_timestamps.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +"""Normalize and repair speaker timestamp lines in ASR transcripts. + +This script targets transcript lines shaped like: + + 天生 00:21 + Speaker 11 01:31:10 + +Common fixes: +- Normalize mixed `MM:SS` / `HH:MM:SS` into one format +- Repair hour rollovers for `MM:SS` timestamps after long recordings +- Optionally rebase a split transcript so it starts at 00:00:00 +- Report suspicious backward jumps instead of silently rewriting them +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path + + +SPEAKER_TS_RE = re.compile(r"^(?P.+?)\s+(?P\d{1,2}:\d{2}(?::\d{2})?)\s*$") + + +@dataclass +class TimestampEvent: + line_no: int + original: str + fixed: str + reason: str + + +@dataclass +class RepairResult: + changed_lines: list[TimestampEvent] + anomalies: list[TimestampEvent] + repaired_text: str + matched_count: int + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Repair speaker timestamp lines in transcript files." + ) + parser.add_argument("input", help="Input transcript file") + parser.add_argument( + "--output", + help="Output path. Defaults to _timestamps_fixed when not using --check.", + ) + parser.add_argument( + "--in-place", + action="store_true", + help="Overwrite the input file", + ) + parser.add_argument( + "--check", + action="store_true", + help="Only analyze and report issues without writing a file", + ) + parser.add_argument( + "--format", + choices=("hhmmss", "preserve"), + default="hhmmss", + help="Timestamp output format. Default: hhmmss", + ) + parser.add_argument( + "--rollover-backjump-seconds", + type=int, + default=15 * 60, + help=( + "Backward jump threshold for treating MM:SS as a new hour rollover. " + "Default: 900" + ), + ) + parser.add_argument( + "--jitter-seconds", + type=int, + default=5, + help="Allowed small backward jitter before flagging an anomaly. Default: 5", + ) + parser.add_argument( + "--rebase-to-zero", + action="store_true", + help="Rebase the first detected speaker timestamp in the file to 00:00:00.", + ) + return parser.parse_args() + + +def to_seconds(parts: list[int]) -> int: + if len(parts) == 2: + minutes, seconds = parts + return minutes * 60 + seconds + hours, minutes, seconds = parts + return hours * 3600 + minutes * 60 + seconds + + +def format_timestamp(total_seconds: int, output_format: str) -> str: + hours = total_seconds // 3600 + minutes = (total_seconds % 3600) // 60 + seconds = total_seconds % 60 + + if output_format == "preserve" and hours == 0: + return f"{minutes:02d}:{seconds:02d}" + return f"{hours:02d}:{minutes:02d}:{seconds:02d}" + + +def repair_timestamps( + text: str, + *, + output_format: str, + rollover_backjump_seconds: int, + jitter_seconds: int, + rebase_to_zero: bool, +) -> RepairResult: + lines = text.splitlines(keepends=True) + repaired_lines: list[str] = [] + changed_lines: list[TimestampEvent] = [] + anomalies: list[TimestampEvent] = [] + + current_hour = 0 + last_abs_seconds: int | None = None + first_abs_seconds: int | None = None + matched_count = 0 + + for line_no, line in enumerate(lines, start=1): + stripped = line.rstrip("\r\n") + newline = line[len(stripped) :] + match = SPEAKER_TS_RE.match(stripped) + + if not match: + repaired_lines.append(line) + continue + + matched_count += 1 + prefix = match.group("prefix") + raw_ts = match.group("ts") + parts = [int(part) for part in raw_ts.split(":")] + + if len(parts) == 3: + abs_seconds = to_seconds(parts) + current_hour = parts[0] + + if ( + last_abs_seconds is not None + and abs_seconds + jitter_seconds < last_abs_seconds + ): + anomalies.append( + TimestampEvent( + line_no=line_no, + original=raw_ts, + fixed=format_timestamp(abs_seconds, output_format), + reason="explicit_hhmmss_backwards", + ) + ) + else: + minutes, seconds = parts + candidate = current_hour * 3600 + minutes * 60 + seconds + + if last_abs_seconds is not None: + while candidate + rollover_backjump_seconds < last_abs_seconds: + current_hour += 1 + candidate = current_hour * 3600 + minutes * 60 + seconds + + if candidate + jitter_seconds < last_abs_seconds: + anomalies.append( + TimestampEvent( + line_no=line_no, + original=raw_ts, + fixed=format_timestamp(candidate, output_format), + reason="small_backjump_after_rollover_check", + ) + ) + + abs_seconds = candidate + + if first_abs_seconds is None: + first_abs_seconds = abs_seconds + + display_seconds = ( + abs_seconds - first_abs_seconds + if rebase_to_zero and first_abs_seconds is not None + else abs_seconds + ) + + fixed_ts = format_timestamp(display_seconds, output_format) + repaired_lines.append(f"{prefix} {fixed_ts}{newline}") + + if fixed_ts != raw_ts: + changed_lines.append( + TimestampEvent( + line_no=line_no, + original=raw_ts, + fixed=fixed_ts, + reason="normalized_or_rollover_fixed", + ) + ) + + last_abs_seconds = abs_seconds + + return RepairResult( + changed_lines=changed_lines, + anomalies=anomalies, + repaired_text="".join(repaired_lines), + matched_count=matched_count, + ) + + +def default_output_path(input_path: Path) -> Path: + return input_path.with_name( + f"{input_path.stem}_timestamps_fixed{input_path.suffix}" + ) + + +def print_summary(result: RepairResult) -> None: + print(f"Matched speaker timestamp lines: {result.matched_count}") + print(f"Rewritten timestamp lines: {len(result.changed_lines)}") + print(f"Anomalies flagged: {len(result.anomalies)}") + + if result.changed_lines: + print("\nChanged lines:") + for event in result.changed_lines[:20]: + print( + f" L{event.line_no}: {event.original} -> {event.fixed} " + f"({event.reason})" + ) + if len(result.changed_lines) > 20: + print(f" ... {len(result.changed_lines) - 20} more") + + if result.anomalies: + print("\nAnomalies:") + for event in result.anomalies[:20]: + print( + f" L{event.line_no}: {event.original} -> {event.fixed} " + f"({event.reason})" + ) + if len(result.anomalies) > 20: + print(f" ... {len(result.anomalies) - 20} more") + + +def main() -> int: + args = parse_args() + input_path = Path(args.input).expanduser().resolve() + + if not input_path.exists(): + print(f"Input file not found: {input_path}", file=sys.stderr) + return 1 + + if args.check and args.in_place: + print("--check and --in-place cannot be used together", file=sys.stderr) + return 1 + + text = input_path.read_text(encoding="utf-8") + result = repair_timestamps( + text, + output_format=args.format, + rollover_backjump_seconds=args.rollover_backjump_seconds, + jitter_seconds=args.jitter_seconds, + rebase_to_zero=args.rebase_to_zero, + ) + + print_summary(result) + + if args.check: + return 0 if not result.anomalies else 2 + + if args.in_place: + output_path = input_path + elif args.output: + output_path = Path(args.output).expanduser().resolve() + else: + output_path = default_output_path(input_path) + + output_path.write_text(result.repaired_text, encoding="utf-8") + print(f"\nWrote repaired transcript: {output_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/transcript-fixer/scripts/fix_transcription.py b/transcript-fixer/scripts/fix_transcription.py index 6e63379b..8815eacd 100755 --- a/transcript-fixer/scripts/fix_transcription.py +++ b/transcript-fixer/scripts/fix_transcription.py @@ -1,4 +1,11 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [ +# "httpx>=0.24.0", +# "filelock>=3.13.0", +# ] +# /// """ Transcript Fixer - Main Entry Point @@ -31,6 +38,7 @@ from cli import ( cmd_init, cmd_add_correction, + cmd_audit, cmd_list_corrections, cmd_run_correction, cmd_review_learned, @@ -89,6 +97,8 @@ def main() -> None: elif args.add_correction: args.from_text, args.to_text = args.add_correction cmd_add_correction(args) + elif getattr(args, 'audit_dictionary', False): + cmd_audit(args) elif args.list_corrections: cmd_list_corrections(args) elif args.review_learned: diff --git a/transcript-fixer/scripts/generate_word_diff.py b/transcript-fixer/scripts/generate_word_diff.py index a36f78d5..8c17360a 100755 --- a/transcript-fixer/scripts/generate_word_diff.py +++ b/transcript-fixer/scripts/generate_word_diff.py @@ -1,4 +1,8 @@ #!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// """ Generate Word-Level Diff HTML Comparison diff --git a/transcript-fixer/scripts/split_transcript_sections.py b/transcript-fixer/scripts/split_transcript_sections.py new file mode 100644 index 00000000..cd0e6481 --- /dev/null +++ b/transcript-fixer/scripts/split_transcript_sections.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# /// script +# requires-python = ">=3.10" +# dependencies = [] +# /// +"""Split a transcript into named sections and optionally rebase timestamps. + +Example: + uv run scripts/split_transcript_sections.py meeting.txt \ + --first-section-name "课前聊天" \ + --section "正式上课::好，无缝切换嘛。对。那个曹总连上了吗？那个网页。" \ + --section "课后复盘::我们复盘一下。" \ + --rebase-to-zero +""" + +from __future__ import annotations + +import argparse +import re +import sys +from dataclasses import dataclass +from pathlib import Path + +from fix_transcript_timestamps import repair_timestamps + + +INVALID_FILENAME_CHARS = re.compile(r'[\\/:*?"<>|]+') + + +@dataclass +class SectionSpec: + name: str + marker: str + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Split a transcript into named sections using marker phrases." + ) + parser.add_argument("input", help="Input transcript file") + parser.add_argument( + "--first-section-name", + default="part-1", + help="Name for the section before the first marker. Default: part-1", + ) + parser.add_argument( + "--section", + action="append", + required=True, + metavar="NAME::MARKER", + help=( + "Section boundary definition. Split starts at the first occurrence of MARKER " + "and names that segment NAME." + ), + ) + parser.add_argument( + "--output-dir", + help="Directory for output files. Defaults to the input file directory.", + ) + parser.add_argument( + "--rebase-to-zero", + action="store_true", + help="Rebase each output section so its first speaker timestamp becomes 00:00:00.", + ) + parser.add_argument( + "--format", + choices=("hhmmss", "preserve"), + default="hhmmss", + help="Timestamp output format when rebasing. Default: hhmmss", + ) + parser.add_argument( + "--rollover-backjump-seconds", + type=int, + default=15 * 60, + help="Backward jump threshold for MM:SS hour rollover repair. Default: 900", + ) + parser.add_argument( + "--jitter-seconds", + type=int, + default=5, + help="Allowed small backward jitter before flagging an anomaly. Default: 5", + ) + return parser.parse_args() + + +def parse_section_arg(value: str) -> SectionSpec: + if "::" not in value: + raise ValueError(f"Invalid --section value: {value!r}") + name, marker = value.split("::", 1) + name = name.strip() + marker = marker.strip() + if not name or not marker: + raise ValueError(f"Invalid --section value: {value!r}") + return SectionSpec(name=name, marker=marker) + + +def sanitize_filename_component(name: str) -> str: + return INVALID_FILENAME_CHARS.sub("-", name).strip().replace(" ", "-") + + +def split_text_by_markers( + text: str, + *, + first_section_name: str, + sections: list[SectionSpec], +) -> list[tuple[str, str]]: + boundaries: list[tuple[int, SectionSpec]] = [] + search_from = 0 + + for spec in sections: + idx = text.find(spec.marker, search_from) + if idx == -1: + raise ValueError(f"Marker not found for section {spec.name!r}: {spec.marker!r}") + boundaries.append((idx, spec)) + search_from = idx + 1 + + result: list[tuple[str, str]] = [] + prev_idx = 0 + prev_name = first_section_name + + for idx, spec in boundaries: + result.append((prev_name, text[prev_idx:idx].rstrip() + "\n")) + prev_idx = idx + prev_name = spec.name + + result.append((prev_name, text[prev_idx:].rstrip() + "\n")) + return result + + +def maybe_rebase( + text: str, + *, + rebase_to_zero: bool, + output_format: str, + rollover_backjump_seconds: int, + jitter_seconds: int, +) -> str: + if not rebase_to_zero: + return text + result = repair_timestamps( + text, + output_format=output_format, + rollover_backjump_seconds=rollover_backjump_seconds, + jitter_seconds=jitter_seconds, + rebase_to_zero=True, + ) + return result.repaired_text + + +def main() -> int: + args = parse_args() + input_path = Path(args.input).expanduser().resolve() + + if not input_path.exists(): + print(f"Input file not found: {input_path}", file=sys.stderr) + return 1 + + try: + sections = [parse_section_arg(value) for value in args.section] + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + + text = input_path.read_text(encoding="utf-8") + + try: + parts = split_text_by_markers( + text, + first_section_name=args.first_section_name, + sections=sections, + ) + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 1 + + output_dir = ( + Path(args.output_dir).expanduser().resolve() + if args.output_dir + else input_path.parent + ) + output_dir.mkdir(parents=True, exist_ok=True) + + for idx, (name, content) in enumerate(parts, start=1): + content = maybe_rebase( + content, + rebase_to_zero=args.rebase_to_zero, + output_format=args.format, + rollover_backjump_seconds=args.rollover_backjump_seconds, + jitter_seconds=args.jitter_seconds, + ) + safe_name = sanitize_filename_component(name) + output_path = output_dir / f"{input_path.stem}-{idx:02d}-{safe_name}{input_path.suffix}" + output_path.write_text(content, encoding="utf-8") + print(f"{output_path}\t{len(content.splitlines())} lines") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/transcript-fixer/scripts/tests/test_common_words_safety.py b/transcript-fixer/scripts/tests/test_common_words_safety.py new file mode 100644 index 00000000..3d4e205c --- /dev/null +++ b/transcript-fixer/scripts/tests/test_common_words_safety.py @@ -0,0 +1,675 @@ +#!/usr/bin/env python3 +""" +Tests for common word safety checks and boundary-aware replacement. + +Covers the three classes of production bugs: +1. Common words added as corrections cause false positives +2. Substring matching causes collateral damage +3. Short common words should never be dictionary entries +""" + +import unittest +import tempfile +import shutil +from pathlib import Path +import sys + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from utils.common_words import ( + check_correction_safety, + audit_corrections, + SafetyWarning, + ALL_COMMON_WORDS, + COMMON_WORDS_2CHAR, + SUBSTRING_COLLISION_MAP, +) +from core.dictionary_processor import DictionaryProcessor +from core.correction_repository import CorrectionRepository +from core.correction_service import CorrectionService, ValidationError + + +class TestSafetyChecks(unittest.TestCase): + """Test the check_correction_safety function.""" + + def test_common_word_blocked_strict(self): + """Adding a common word like '仿佛' should produce an error in strict mode.""" + warnings = check_correction_safety("仿佛", "反复", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "Expected at least one error for '仿佛'") + self.assertTrue( + any(w.category == "common_word" for w in errors), + "Expected 'common_word' category", + ) + + def test_common_word_warning_nonstrict(self): + """In non-strict mode, common words produce warnings, not errors.""" + warnings = check_correction_safety("仿佛", "反复", strict=False) + errors = [w for w in warnings if w.level == "error"] + self.assertEqual(len(errors), 0, "Non-strict mode should have no errors") + warns = [w for w in warnings if w.level == "warning"] + self.assertTrue(len(warns) > 0, "Expected at least one warning") + + def test_both_common_words_flagged(self): + """When both from_text and to_text are common words, flag with 'both_common'.""" + warnings = check_correction_safety("正面", "正念", strict=True) + both = [w for w in warnings if w.category == "both_common"] + # "正面" is common, "正念" may or may not be -- but at least common_word should fire + common = [w for w in warnings if w.category == "common_word"] + self.assertTrue(len(common) > 0) + + def test_short_text_warning(self): + """2-char text not in common words list still gets a short_text warning.""" + # Use something unlikely to be in the common words list + warnings = check_correction_safety("zz", "xx", strict=True) + short_warns = [w for w in warnings if w.category == "short_text"] + self.assertTrue(len(short_warns) > 0, "Expected short_text warning for 2-char text") + + def test_known_substring_collision(self): + """'线数' is in SUBSTRING_COLLISION_MAP and should trigger collision warning.""" + warnings = check_correction_safety("线数", "线束", strict=True) + collisions = [w for w in warnings if w.category == "substring_collision"] + self.assertTrue(len(collisions) > 0, "Expected substring_collision for '线数'") + + def test_safe_correction_no_warnings(self): + """A safe, domain-specific correction should produce no warnings.""" + # "巨升智能" -> "具身智能" is a genuine ASR error, not a common word + warnings = check_correction_safety("巨升智能", "具身智能", strict=True) + self.assertEqual(len(warnings), 0, f"Expected no warnings, got: {warnings}") + + def test_long_from_text_safe(self): + """Long from_text (>4 chars) should not trigger short text or collision warnings.""" + warnings = check_correction_safety("语音识别错误", "语音识别模型", strict=True) + short_warns = [w for w in warnings if w.category == "short_text"] + self.assertEqual(len(short_warns), 0) + + # --- Production false positives from the bug report --- + + def test_production_false_positive_fangfu(self): + """'仿佛→反复' was a real production false positive.""" + warnings = check_correction_safety("仿佛", "反复", strict=True) + self.assertTrue(len(warnings) > 0) + + def test_production_false_positive_zhengmian(self): + """'正面→正念' was a real production false positive.""" + warnings = check_correction_safety("正面", "正念", strict=True) + self.assertTrue(len(warnings) > 0) + + def test_production_false_positive_youyu(self): + """'犹豫→抑郁' was a real production false positive.""" + warnings = check_correction_safety("犹豫", "抑郁", strict=True) + self.assertTrue(len(warnings) > 0) + + def test_production_false_positive_chuanshuo(self): + """'传说→穿梭' was a real production false positive.""" + warnings = check_correction_safety("传说", "穿梭", strict=True) + self.assertTrue(len(warnings) > 0) + + def test_production_false_positive_yanji(self): + """'演技→眼界' was a real production false positive.""" + warnings = check_correction_safety("演技", "眼界", strict=True) + self.assertTrue(len(warnings) > 0) + + def test_production_false_positive_zengjia(self): + """'增加→工站/环节' was a real production false positive.""" + warnings = check_correction_safety("增加", "工站", strict=True) + self.assertTrue(len(warnings) > 0) + + +class TestAuditCorrections(unittest.TestCase): + """Test the audit_corrections function.""" + + def test_audit_finds_known_bad_rules(self): + """Audit should flag the production false positives.""" + corrections = { + "仿佛": "反复", + "正面": "正念", + "线数": "线束", + "巨升智能": "具身智能", # This one is fine + } + issues = audit_corrections(corrections) + + self.assertIn("仿佛", issues) + self.assertIn("正面", issues) + self.assertIn("线数", issues) + self.assertNotIn("巨升智能", issues) + + def test_audit_empty_dict(self): + """Audit of empty dict returns empty.""" + issues = audit_corrections({}) + self.assertEqual(len(issues), 0) + + +class TestBoundaryAwareReplacement(unittest.TestCase): + """Test DictionaryProcessor's boundary-aware replacement logic.""" + + def test_substring_collision_prevented(self): + """'线数→线束' should NOT match inside '产线数据'.""" + processor = DictionaryProcessor({"线数": "线束"}, []) + text = "这条产线数据很重要" + result, changes = processor.process(text) + self.assertEqual(result, "这条产线数据很重要", + "Should NOT replace '线数' inside '产线数据'") + self.assertEqual(len(changes), 0) + + def test_standalone_match_replaced(self): + """'线数→线束' SHOULD match when it's standalone (not inside a longer word).""" + processor = DictionaryProcessor({"线数": "线束"}, []) + text = "检查线数是否正确" + result, changes = processor.process(text) + # "线数" here is standalone (not inside a common word), + # so it should be replaced + self.assertEqual(result, "检查线束是否正确") + self.assertEqual(len(changes), 1) + + def test_long_correction_not_affected(self): + """Corrections longer than 3 chars use standard replacement.""" + processor = DictionaryProcessor({"巨升智能": "具身智能"}, []) + text = "今天讨论巨升智能的进展" + result, changes = processor.process(text) + self.assertEqual(result, "今天讨论具身智能的进展") + self.assertEqual(len(changes), 1) + + def test_multiple_replacements_mixed(self): + """Mix of safe and unsafe positions should be handled correctly.""" + processor = DictionaryProcessor({"数据": "数据集"}, []) + text = "大数据分析和数据清洗" + result, changes = processor.process(text) + # "数据" inside "大数据" should be protected + # "数据" standalone should be replaced + # Both are common words, so boundary check applies + # The exact behavior depends on what's in ALL_COMMON_WORDS + # At minimum, the processor should not crash + self.assertIsInstance(result, str) + + def test_no_corrections_no_changes(self): + """Empty corrections dict produces no changes.""" + processor = DictionaryProcessor({}, []) + text = "原始文本" + result, changes = processor.process(text) + self.assertEqual(result, "原始文本") + self.assertEqual(len(changes), 0) + + def test_context_rules_still_work(self): + """Context rules (regex) are unaffected by boundary checks.""" + context_rules = [{ + "pattern": r"股价系统", + "replacement": "框架系统", + "description": "ASR error fix" + }] + processor = DictionaryProcessor({}, context_rules) + text = "股价系统需要优化" + result, changes = processor.process(text) + self.assertEqual(result, "框架系统需要优化") + self.assertEqual(len(changes), 1) + + +class TestSupersetReplacementBug(unittest.TestCase): + """ + Bug 1: When to_text contains from_text as a substring, and the + surrounding text already forms to_text, the replacement must be skipped. + + Production example: rule "金流"→"现金流", input "现金流断了" + Without fix: "现现金流断了" (WRONG -- duplicated prefix) + With fix: "现金流断了" (correct -- already in target form) + + This check must work for ALL rule lengths, not just short rules. + """ + + def test_suffix_superset_skip(self): + """from_text is a suffix of to_text: 金流→现金流 inside 现金流.""" + processor = DictionaryProcessor({"金流": "现金流"}, []) + result, changes = processor.process("现金流断了") + self.assertEqual(result, "现金流断了") + self.assertEqual(len(changes), 0) + + def test_suffix_superset_standalone_replaced(self): + """Standalone from_text should still be replaced.""" + processor = DictionaryProcessor({"金流": "现金流"}, []) + result, changes = processor.process("金流断了") + self.assertEqual(result, "现金流断了") + self.assertEqual(len(changes), 1) + + def test_prefix_superset_skip(self): + """from_text is a prefix of to_text: 现金→现金流 inside 现金流.""" + processor = DictionaryProcessor({"现金": "现金流"}, []) + result, changes = processor.process("现金流断了") + self.assertEqual(result, "现金流断了") + self.assertEqual(len(changes), 0) + + def test_middle_superset_skip(self): + """from_text is in the middle of to_text.""" + processor = DictionaryProcessor({"金流": "现金流通"}, []) + result, changes = processor.process("现金流通畅") + self.assertEqual(result, "现金流通畅") + self.assertEqual(len(changes), 0) + + def test_long_rule_superset_skip(self): + """Superset check must also work for long rules (>3 chars).""" + processor = DictionaryProcessor({"金流断裂": "现金流断裂"}, []) + result, changes = processor.process("现金流断裂了") + self.assertEqual(result, "现金流断裂了") + self.assertEqual(len(changes), 0) + + def test_long_rule_superset_standalone_replaced(self): + """Long rule standalone should still be replaced.""" + processor = DictionaryProcessor({"金流断裂": "现金流断裂"}, []) + result, changes = processor.process("金流断裂了") + self.assertEqual(result, "现金流断裂了") + self.assertEqual(len(changes), 1) + + def test_superset_with_unknown_words(self): + """Superset check works regardless of common_words membership.""" + # Use words NOT in ALL_COMMON_WORDS + processor = DictionaryProcessor({"资流": "投资流"}, []) + result, changes = processor.process("投资流断了") + self.assertEqual(result, "投资流断了") + self.assertEqual(len(changes), 0) + + def test_superset_mixed_positions(self): + """One occurrence is already correct, another is standalone.""" + processor = DictionaryProcessor({"金流": "现金流"}, []) + result, changes = processor.process("现金流好，金流差") + self.assertEqual(result, "现金流好，现金流差") + self.assertEqual(len(changes), 1) + + def test_no_superset_normal_replacement(self): + """When to_text does NOT contain from_text, normal replacement.""" + processor = DictionaryProcessor({"金流": "资金链"}, []) + result, changes = processor.process("金流断了") + self.assertEqual(result, "资金链断了") + self.assertEqual(len(changes), 1) + + +class TestIdiomCompoundProtection(unittest.TestCase): + """ + Bug 2: Short rules must not corrupt idioms and compound words. + + Production examples: + - "天差"→"偏差" inside "天差地别" => "偏差地别" (broken idiom) + - "亮亮"→"亮哥" inside "漂漂亮亮" => "漂漂亮哥" (broken phrase) + + Defense: _is_inside_longer_word checks common_words set. + """ + + def test_tiancha_inside_idiom(self): + """天差→偏差 must not break 天差地别.""" + processor = DictionaryProcessor({"天差": "偏差"}, []) + result, changes = processor.process("天差地别") + self.assertEqual(result, "天差地别") + self.assertEqual(len(changes), 0) + + def test_liangliang_inside_compound(self): + """亮亮→亮哥 must not break 漂漂亮亮.""" + processor = DictionaryProcessor({"亮亮": "亮哥"}, []) + result, changes = processor.process("漂漂亮亮") + self.assertEqual(result, "漂漂亮亮") + self.assertEqual(len(changes), 0) + + def test_tiancha_standalone_replaced(self): + """Standalone 天差 (not inside idiom) should be replaced.""" + processor = DictionaryProcessor({"天差": "偏差"}, []) + # 天差 alone, not followed by 地别 or other idiom continuation + result, changes = processor.process("误差天差太大了") + # Whether this gets replaced depends on common_words; at minimum + # it should not crash. If 天差 is in common words, it stays. + self.assertIsInstance(result, str) + + +class TestValidPhraseProtection(unittest.TestCase): + """ + Bug 3: Short rules must not corrupt valid phrases where from_text + is a legitimate substring. + + Production example: + - "被看"→"被砍" inside "被看见" => "被砍见" + + Defense: _is_inside_longer_word checks common_words set. + """ + + def test_beikan_inside_beikanjian(self): + """被看→被砍 must not break 被看见.""" + processor = DictionaryProcessor({"被看": "被砍"}, []) + result, changes = processor.process("被看见") + self.assertEqual(result, "被看见") + self.assertEqual(len(changes), 0) + + def test_beikan_in_sentence(self): + """被看→被砍 must not break 被看见 in a full sentence.""" + processor = DictionaryProcessor({"被看": "被砍"}, []) + result, changes = processor.process("他被看见了") + self.assertEqual(result, "他被看见了") + self.assertEqual(len(changes), 0) + + +class TestServiceSafetyIntegration(unittest.TestCase): + """Integration tests: CorrectionService rejects unsafe corrections.""" + + def setUp(self): + self.test_dir = Path(tempfile.mkdtemp()) + self.db_path = self.test_dir / "test.db" + self.repository = CorrectionRepository(self.db_path) + self.service = CorrectionService(self.repository) + + def tearDown(self): + self.service.close() + shutil.rmtree(self.test_dir) + + def test_common_word_rejected(self): + """Adding a common word correction is blocked by default.""" + with self.assertRaises(ValidationError) as ctx: + self.service.add_correction("仿佛", "反复", "general") + self.assertIn("Safety check BLOCKED", str(ctx.exception)) + + def test_common_word_forced(self): + """Adding a common word with force=True succeeds.""" + correction_id = self.service.add_correction( + "仿佛", "反复", "general", force=True, + ) + self.assertIsInstance(correction_id, int) + self.assertGreater(correction_id, 0) + + def test_safe_correction_accepted(self): + """A genuine ASR correction is accepted without force.""" + correction_id = self.service.add_correction( + "巨升智能", "具身智能", "general", + ) + self.assertIsInstance(correction_id, int) + + def test_audit_on_service(self): + """audit_dictionary method returns issues for unsafe rules.""" + # Force-add some unsafe rules + self.service.add_correction("仿佛", "反复", "general", force=True) + self.service.add_correction("巨升智能", "具身智能", "general") + + issues = self.service.audit_dictionary("general") + self.assertIn("仿佛", issues) + self.assertNotIn("巨升智能", issues) + + +class TestProductionFalsePositivesCoverage(unittest.TestCase): + """ + Verify ALL production false positives from the 2026-03 manual review + are present in the safety system and correctly caught. + + Each test corresponds to a specific word that caused real damage in production. + If any of these tests fail, it means the safety net has a gap. + """ + + # --- Category 1: Lifestyle domain --- + + def test_baojian_blocked(self): + """'保健' (lifestyle/beauty) must be caught.""" + self.assertIn("保健", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("保健", "宝剑", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'保健' must produce an error") + + def test_neihan_blocked(self): + """'内涵' (lifestyle/beauty) must be caught.""" + self.assertIn("内涵", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("内涵", "内含", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'内涵' must produce an error") + + def test_zhengjing_blocked(self): + """'正经' (lifestyle) must be caught.""" + self.assertIn("正经", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("正经", "正劲", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'正经' must produce an error") + + # --- Category 1: Manufacturing domain --- + + def test_jingong_blocked(self): + """'仅供' (manufacturing) must be caught.""" + self.assertIn("仅供", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("仅供", "紧供", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'仅供' must produce an error") + + def test_gongqi_blocked(self): + """'供气' (manufacturing) must be caught.""" + self.assertIn("供气", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("供气", "工器", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'供气' must produce an error") + + def test_chutou_blocked(self): + """'出头' (manufacturing) must be caught.""" + self.assertIn("出头", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("出头", "初投", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'出头' must produce an error") + + def test_jikou_blocked(self): + """'几口' (manufacturing) must be caught.""" + self.assertIn("几口", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("几口", "集口", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'几口' must produce an error") + + # --- Category 1: Various domains --- + + def test_liangben_blocked(self): + """'两本' must be caught.""" + self.assertIn("两本", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("两本", "量本", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'两本' must produce an error") + + def test_chuwu_blocked(self): + """'初五' must be caught.""" + self.assertIn("初五", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("初五", "出误", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'初五' must produce an error") + + def test_lijie_blocked(self): + """'力竭' must be caught.""" + self.assertIn("力竭", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("力竭", "立杰", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'力竭' must produce an error") + + def test_chongyu_blocked(self): + """'充于' must be caught.""" + self.assertIn("充于", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("充于", "冲余", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'充于' must produce an error") + + def test_shuju_blocked(self): + """'数据' must be caught.""" + self.assertIn("数据", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("数据", "束据", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'数据' must produce an error") + + # --- Category 1: Substring collision sources --- + + def test_beikan_blocked(self): + """'被看' (general) must be caught.""" + self.assertIn("被看", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("被看", "被砍", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'被看' must produce an error") + + def test_tiancha_blocked(self): + """'天差' (education) must be caught.""" + self.assertIn("天差", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("天差", "偏差", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'天差' must produce an error") + + def test_liangliang_blocked(self): + """'亮亮' (manufacturing) must be caught.""" + self.assertIn("亮亮", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("亮亮", "亮哥", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'亮亮' must produce an error") + + def test_jinliu_blocked(self): + """'金流' (manufacturing) must be caught.""" + self.assertIn("金流", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("金流", "现金流", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'金流' must produce an error") + + # --- Category 1: Substring issue sources --- + + def test_kanjian_blocked(self): + """'看见' must be caught (caused substring issues).""" + self.assertIn("看见", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("看见", "砍件", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'看见' must produce an error") + + def test_fenzhong_blocked(self): + """'分钟' must be caught (caused substring issues).""" + self.assertIn("分钟", COMMON_WORDS_2CHAR) + warnings = check_correction_safety("分钟", "份种", strict=True) + errors = [w for w in warnings if w.level == "error"] + self.assertTrue(len(errors) > 0, "'分钟' must produce an error") + + +class TestSubstringCollisionMapCoverage(unittest.TestCase): + """ + Verify all production substring collision patterns are in the map. + + Each test reproduces a real corruption pattern from production: + a short word matched inside a longer valid phrase and corrupted it. + """ + + def test_xianshu_collision_exists(self): + """'线数' inside '产线数据' -> corrupts to '产线束据'.""" + self.assertIn("线数", SUBSTRING_COLLISION_MAP) + self.assertIn("产线数据", SUBSTRING_COLLISION_MAP["线数"]) + + def test_jinliu_collision_exists(self): + """'金流' inside '现金流' -> corrupts to '现现金流'.""" + self.assertIn("金流", SUBSTRING_COLLISION_MAP) + self.assertIn("现金流", SUBSTRING_COLLISION_MAP["金流"]) + + def test_beikan_collision_exists(self): + """'被看' inside '被看见' -> corrupts to '被砍见'.""" + self.assertIn("被看", SUBSTRING_COLLISION_MAP) + self.assertIn("被看见", SUBSTRING_COLLISION_MAP["被看"]) + + def test_liangliang_collision_exists(self): + """'亮亮' inside '漂漂亮亮' -> corrupts to '漂漂亮哥'.""" + self.assertIn("亮亮", SUBSTRING_COLLISION_MAP) + self.assertIn("漂漂亮亮", SUBSTRING_COLLISION_MAP["亮亮"]) + + def test_tiancha_collision_exists(self): + """'天差' inside '天差地别' -> corrupts idiom to '偏差地别'.""" + self.assertIn("天差", SUBSTRING_COLLISION_MAP) + self.assertIn("天差地别", SUBSTRING_COLLISION_MAP["天差"]) + + def test_collision_safety_check_fires(self): + """check_correction_safety must flag entries in SUBSTRING_COLLISION_MAP.""" + for short_word in ["金流", "被看", "亮亮", "天差"]: + warnings = check_correction_safety(short_word, "dummy", strict=True) + collision_warnings = [ + w for w in warnings if w.category == "substring_collision" + ] + self.assertTrue( + len(collision_warnings) > 0, + f"'{short_word}' must trigger substring_collision warning", + ) + + +class TestBoundaryAwareProductionCollisions(unittest.TestCase): + """ + End-to-end tests: verify DictionaryProcessor does NOT corrupt + longer valid phrases when a short correction matches inside them. + + Each test reproduces an exact production corruption scenario. + """ + + def test_jinliu_inside_xianjinliu(self): + """'金流→现金流' must NOT corrupt '现金流' to '现现金流'.""" + processor = DictionaryProcessor({"金流": "现金流"}, []) + text = "公司的现金流很健康" + result, changes = processor.process(text) + self.assertEqual(result, "公司的现金流很健康", + "Must NOT replace '金流' inside '现金流'") + self.assertEqual(len(changes), 0) + + def test_beikan_inside_beikanjian(self): + """'被看→被砍' must NOT corrupt '被看见' to '被砍见'.""" + processor = DictionaryProcessor({"被看": "被砍"}, []) + text = "他被看见了" + result, changes = processor.process(text) + self.assertEqual(result, "他被看见了", + "Must NOT replace '被看' inside '被看见'") + self.assertEqual(len(changes), 0) + + def test_liangliang_inside_piaopiaoliangliag(self): + """'亮亮→亮哥' must NOT corrupt '漂漂亮亮' to '漂漂亮哥'.""" + processor = DictionaryProcessor({"亮亮": "亮哥"}, []) + text = "打扮得漂漂亮亮的" + result, changes = processor.process(text) + self.assertEqual(result, "打扮得漂漂亮亮的", + "Must NOT replace '亮亮' inside '漂漂亮亮'") + self.assertEqual(len(changes), 0) + + def test_tiancha_inside_tianchadiebie(self): + """'天差→偏差' must NOT corrupt '天差地别' to '偏差地别'.""" + processor = DictionaryProcessor({"天差": "偏差"}, []) + text = "两者天差地别" + result, changes = processor.process(text) + self.assertEqual(result, "两者天差地别", + "Must NOT replace '天差' inside '天差地别'") + self.assertEqual(len(changes), 0) + + def test_kanjian_not_corrupted_by_beikan(self): + """'被看→被砍' must NOT corrupt '看见' if '被看见' is in text.""" + processor = DictionaryProcessor({"被看": "被砍"}, []) + text = "我被看见了，别人也看见了" + result, changes = processor.process(text) + # '被看见' contains '被看' -- boundary check must protect it + self.assertNotIn("被砍", result, + "Must NOT corrupt any instance of '被看' inside '被看见'") + + +class TestAuditCatchesAllProductionFalsePositives(unittest.TestCase): + """ + Verify audit_corrections flags every single production false positive + when they appear in a corrections dictionary. + """ + + def test_audit_catches_all_category1_words(self): + """Every Category 1 word must be flagged by audit_corrections.""" + all_false_positives = { + # lifestyle + "仿佛": "反复", "正面": "正念", "犹豫": "抑郁", + "传说": "穿梭", "演技": "眼界", "无果": "无过", + "旗号": "期号", "应急": "应集", "正经": "正劲", + # lifestyle/beauty + "保健": "宝剑", "内涵": "内含", + # manufacturing + "仅供": "紧供", "供气": "工器", "出头": "初投", "几口": "集口", + # lifestyle previously disabled + "增加": "工站", "教育": "叫于", "大一": "答疑", + "曲线": "去先", "分母": "份母", + # various domains + "两本": "量本", "初五": "出误", "数据": "束据", + "力竭": "立杰", "充于": "冲余", + # substring collision sources + "被看": "被砍", "天差": "偏差", "亮亮": "亮哥", "金流": "现金流", + # substring issue words + "看见": "砍件", "分钟": "份种", + } + + issues = audit_corrections(all_false_positives) + + for word in all_false_positives: + self.assertIn( + word, issues, + f"audit_corrections MUST flag '{word}' but did not" + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/transcript-fixer/scripts/tests/test_correction_service.py b/transcript-fixer/scripts/tests/test_correction_service.py index d81963db..a5521e54 100644 --- a/transcript-fixer/scripts/tests/test_correction_service.py +++ b/transcript-fixer/scripts/tests/test_correction_service.py @@ -160,6 +160,31 @@ def test_remove_nonexistent_correction(self): success = self.service.remove_correction("nonexistent", "general") self.assertFalse(success) + # ==================== Domain Stats Tests ==================== + + def test_get_corrections_all_domains(self): + """domain=None loads all domains.""" + self.service.add_correction("a", "b", "general") + self.service.add_correction("c", "d", "finance") + all_corr = self.service.get_corrections(None) + self.assertEqual(len(all_corr), 2) + self.assertIn("a", all_corr) + self.assertIn("c", all_corr) + + def test_get_domain_stats(self): + """get_domain_stats returns per-domain counts.""" + self.service.add_correction("a", "b", "general") + self.service.add_correction("c", "d", "finance") + self.service.add_correction("e", "f", "finance") + stats = self.service.get_domain_stats() + self.assertEqual(stats["general"], 1) + self.assertEqual(stats["finance"], 2) + + def test_get_domain_stats_empty(self): + """get_domain_stats returns empty dict when no corrections.""" + stats = self.service.get_domain_stats() + self.assertEqual(stats, {}) + # ==================== Import/Export Tests ==================== def test_import_corrections(self): diff --git a/transcript-fixer/scripts/tests/test_fix_transcript_timestamps.py b/transcript-fixer/scripts/tests/test_fix_transcript_timestamps.py new file mode 100644 index 00000000..b7fd4579 --- /dev/null +++ b/transcript-fixer/scripts/tests/test_fix_transcript_timestamps.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 +"""Tests for transcript timestamp normalization and rebasing.""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from fix_transcript_timestamps import repair_timestamps + + +class TestFixTranscriptTimestamps(unittest.TestCase): + def test_rollover_fix(self): + text = ( + "甲 58:50\n" + "内容 A\n" + "乙 59:58\n" + "内容 B\n" + "丙 00:05\n" + "内容 C\n" + ) + result = repair_timestamps( + text, + output_format="hhmmss", + rollover_backjump_seconds=15 * 60, + jitter_seconds=5, + rebase_to_zero=False, + ) + self.assertIn("甲 00:58:50", result.repaired_text) + self.assertIn("乙 00:59:58", result.repaired_text) + self.assertIn("丙 01:00:05", result.repaired_text) + self.assertEqual(len(result.anomalies), 0) + + def test_rebase_to_zero(self): + text = ( + "甲 01:31:10\n" + "内容 A\n" + "乙 01:31:12\n" + "内容 B\n" + ) + result = repair_timestamps( + text, + output_format="hhmmss", + rollover_backjump_seconds=15 * 60, + jitter_seconds=5, + rebase_to_zero=True, + ) + self.assertIn("甲 00:00:00", result.repaired_text) + self.assertIn("乙 00:00:02", result.repaired_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/transcript-fixer/scripts/tests/test_split_transcript_sections.py b/transcript-fixer/scripts/tests/test_split_transcript_sections.py new file mode 100644 index 00000000..d896e2b6 --- /dev/null +++ b/transcript-fixer/scripts/tests/test_split_transcript_sections.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Tests for transcript section splitting.""" + +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +from split_transcript_sections import ( + SectionSpec, + sanitize_filename_component, + split_text_by_markers, +) + + +class TestSplitTranscriptSections(unittest.TestCase): + def test_split_text_by_markers(self): + text = ( + "预热内容\n" + "开始安装\n" + "装环境内容\n" + "我们复盘一下。\n" + "复盘内容\n" + ) + parts = split_text_by_markers( + text, + first_section_name="课前聊天", + sections=[ + SectionSpec(name="正式上课", marker="开始安装"), + SectionSpec(name="课后复盘", marker="我们复盘一下。"), + ], + ) + self.assertEqual(parts[0][0], "课前聊天") + self.assertEqual(parts[1][0], "正式上课") + self.assertEqual(parts[2][0], "课后复盘") + self.assertTrue(parts[1][1].startswith("开始安装")) + self.assertTrue(parts[2][1].startswith("我们复盘一下。")) + + def test_sanitize_filename_component(self): + self.assertEqual(sanitize_filename_component("课后/复盘"), "课后-复盘") + + +if __name__ == "__main__": + unittest.main() diff --git a/transcript-fixer/scripts/utils/common_words.py b/transcript-fixer/scripts/utils/common_words.py new file mode 100644 index 00000000..f9848a30 --- /dev/null +++ b/transcript-fixer/scripts/utils/common_words.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 +""" +Common Chinese Words Safety Check + +Detects when a correction's from_text is a common Chinese word, +which would cause false positive replacements across transcripts. + +This is the core defense against the "仿佛→反复" class of bugs: +valid corrections for one ASR model that corrupt correct text from better models. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import List, Set + + +# High-frequency Chinese words that should NEVER be dictionary correction sources. +# These are words that appear correctly in normal Chinese text and replacing them +# would cause widespread collateral damage. +# +# Organized by category for maintainability. Not exhaustive -- the heuristic +# checks below catch additional cases. +# +# IMPORTANT: This list was curated from actual production false positives found +# in a 187-video transcription run (2026-03). Each entry caused real damage. + +COMMON_WORDS_2CHAR: Set[str] = { + # --- Production false positives (confirmed damage, 2026-03 run) --- + # lifestyle domain + "仿佛", "正面", "犹豫", "传说", "演技", "无果", "旗号", "应急", "正经", + # lifestyle/beauty domain + "保健", "内涵", + # manufacturing domain + "仅供", "供气", "出头", "几口", + # lifestyle - previously disabled then re-confirmed + "增加", "教育", "大一", "曲线", "分母", + # various domains - discovered in manual review + "两本", "初五", "数据", "力竭", "充于", + # general/manufacturing/education - substring collision sources + "被看", "天差", "亮亮", "金流", + # caused substring issues in production + "看见", "分钟", + # --- High-frequency general vocabulary --- + "我们", "他们", "你们", "这个", "那个", "什么", "怎么", "为什么", + "可以", "因为", "所以", "但是", "虽然", "如果", "已经", "正在", + "需要", "应该", "可能", "一定", "非常", "比较", "特别", "一般", + "开始", "结束", "继续", "发展", "问题", "方法", "工作", "时间", + "学习", "研究", "分析", "讨论", "了解", "知道", "觉得", "认为", + "希望", "表示", "提出", "建议", "要求", "计划", "设计", "管理", + "技术", "系统", "数据", "网络", "平台", "产品", "服务", "市场", + "企业", "公司", "团队", "项目", "客户", "用户", "资源", "成本", + "效果", "质量", "安全", "标准", "流程", "模式", "策略", "方案", + "结构", "功能", "接口", "模块", "组件", "测试", "部署", "运维", + "目标", "任务", "进度", "优化", "调整", "更新", "升级", "维护", + "配置", "参数", "设置", "选项", "状态", "信息", "内容", "格式", + "教育", "培训", "实践", "经验", "能力", "水平", "素质", "思维", + "创新", "合作", "沟通", "交流", "反馈", "评估", "考核", "激励", + # --- Common verbs and adjectives --- + "实现", "完成", "处理", "解决", "执行", "操作", "运行", "启动", + "关闭", "打开", "保存", "删除", "修改", "添加", "移除", "查看", + "搜索", "过滤", "排序", "导入", "导出", "上传", "下载", "同步", + "重要", "关键", "核心", "基本", "主要", "次要", "简单", "复杂", + "明确", "清晰", "具体", "详细", "准确", "完整", "稳定", "灵活", + # --- Domain terms that look like ASR errors but are valid --- + "线数", "曲线", "分母", "正面", "旗号", "无果", "演技", + # --- Common verb+一 patterns (打一个/来一个/做一下 etc.) --- + # "打一" caused production false positive: "打一个锚" → "答疑个锚" (2026-04) + "打一", "来一", "做一", "写一", "给一", "拉一", "开一", "看一", + "跑一", "找一", "选一", "试一", "走一", "问一", "搞一", "聊一", +} + +# Common 3+ character words that should also be protected. +# These serve dual purpose: +# 1. Never used as correction sources (same as 2-char words) +# 2. Used by DictionaryProcessor._is_inside_longer_word() to detect +# when a short correction target is embedded inside a valid longer word +COMMON_WORDS_3PLUS: Set[str] = { + "自动化", "智能化", "数字化", "信息化", "标准化", "规范化", + "产线数", "服务器", "数据库", "操作系统", "人工智能", "机器学习", + "深度学习", "自然语言", "计算机视觉", "强化学习", + "区块链", "云计算", "大数据", "物联网", "互联网", + # --- Production collision targets (longer words containing short false positives) --- + # These must be here so _is_inside_longer_word() can detect them + "产线数据", "现金流", "资金流", "现金流量", "资金流向", + "被看见", "被看到", "被看作", "被看成", "被看好", + "漂漂亮亮", "亮亮堂堂", "明明亮亮", + "天差地别", "天差地远", + "被看见", "没看见", + "出头露面", "出头之日", + "正月初五", "大年初五", + "保健品", "保健操", "医疗保健", + "文化内涵", + "无果而终", + # --- Common verb+一+量词 patterns (防止"打一"→X 类误纠) --- + "打一个", "打一针", "打一下", "打一次", "打一把", + "来一个", "来一下", "来一次", "来一杯", + "做一个", "做一下", "做一次", + "写一个", "写一下", "写一篇", + "给一个", "看一下", "看一看", "看一遍", + "跑一下", "跑一遍", "跑一次", + "试一下", "试一试", "试一次", + # --- Common Chinese idioms/phrases containing short words --- + # These are needed to prevent idiom corruption + "正面临", "正面对", + "应急响应", "应急预案", "应急处理", + "仅供参考", "仅供参阅", +} + +# Words that commonly contain other words as substrings. +# Key: the short word, Value: common words containing it. +# Used to warn about substring collision risk. +SUBSTRING_COLLISION_MAP: dict[str, list[str]] = { + "线数": ["产线数据", "曲线数", "线数量"], + "增加": ["新增加", "增加值"], + "数据": ["大数据", "数据库", "数据集", "元数据"], + "服务": ["服务器", "服务端", "微服务", "云服务"], + "测试": ["单元测试", "集成测试", "压力测试", "测试用例"], + "模型": ["大模型", "模型训练", "预训练模型"], + "学习": ["学习率", "深度学习", "机器学习", "强化学习"], + "正面": ["正面临", "正面对"], + "应急": ["应急响应", "应急预案", "应急处理"], + "无果": ["无果而终", "毫无果断"], + # --- Production substring collision patterns (2026-03 manual review) --- + # "线数" inside "产线数据" → corrupts to "产线束据" + # (already covered above) + # "金流" inside "现金流" → corrupts to "现现金流" (replacement contains match) + "金流": ["现金流", "资金流", "资金流向", "现金流量"], + # "被看" inside "被看见" → corrupts to "被砍见" + "被看": ["被看见", "被看到", "被看作", "被看成", "被看好"], + # "亮亮" inside "漂漂亮亮" → corrupts to "漂漂亮哥" + "亮亮": ["漂漂亮亮", "亮亮堂堂", "明明亮亮"], + # "天差" inside "天差地别" → corrupts idiom to "偏差地别" + "天差": ["天差地别", "天差地远"], + # "看见" inside longer phrases → substring collision risk + "看见": ["被看见", "看见过", "没看见"], + # "分钟" inside longer phrases → substring collision risk + "分钟": ["几分钟", "十分钟", "三十分钟", "一分钟"], + # "出头" common in phrases + "出头": ["出头露面", "出头之日", "冒出头"], + # "初五" common in date phrases + "初五": ["正月初五", "大年初五"], + # "保健" common in compound words + "保健": ["保健品", "保健操", "医疗保健"], + # "内涵" common in compound words + "内涵": ["内涵段子", "文化内涵"], + # "打一" common in verb+一+量词 (2026-04 production false positive) + "打一": ["打一个", "打一针", "打一下", "打一次", "打一把"], +} + +ALL_COMMON_WORDS: Set[str] = COMMON_WORDS_2CHAR | COMMON_WORDS_3PLUS + + +@dataclass +class SafetyWarning: + """A warning about a potentially dangerous correction rule.""" + level: str # "error" (should block) or "warning" (should confirm) + category: str # "common_word", "short_text", "substring_collision" + message: str + suggestion: str # What to do instead + + +def check_correction_safety( + from_text: str, + to_text: str, + strict: bool = True, +) -> List[SafetyWarning]: + """ + Check if a correction rule is safe to add to the dictionary. + + This is the main entry point. Returns a list of warnings/errors. + Empty list = safe to add. + + Args: + from_text: The text to be replaced (the "wrong" text) + to_text: The replacement text (the "correct" text) + strict: If True, common word matches are errors; if False, warnings + + Returns: + List of SafetyWarning objects (empty = safe) + """ + warnings: List[SafetyWarning] = [] + + # Check 1: Is from_text a known common word? + if from_text in ALL_COMMON_WORDS: + level = "error" if strict else "warning" + warnings.append(SafetyWarning( + level=level, + category="common_word", + message=( + f"'{from_text}' is a common Chinese word that appears correctly " + f"in normal text. Replacing it with '{to_text}' will cause " + f"false positives across all transcripts." + ), + suggestion=( + f"Use a context rule instead: add a regex pattern that matches " + f"'{from_text}' only in the specific context where it's an ASR error. " + f"Example: match '{from_text}' only when preceded/followed by specific characters." + ), + )) + + # Check 2: Is from_text very short (<=2 chars)? + if len(from_text) <= 2: + # Even if not in our common words list, 2-char Chinese words are risky + if from_text not in ALL_COMMON_WORDS: + # Not already flagged above -- add a length warning + warnings.append(SafetyWarning( + level="warning", + category="short_text", + message=( + f"'{from_text}' is only {len(from_text)} character(s). " + f"Short corrections have high false positive risk in Chinese " + f"because they match as substrings inside longer words." + ), + suggestion=( + f"Verify '{from_text}' is never a valid word in any context. " + f"If unsure, use a context rule with surrounding text patterns instead." + ), + )) + + # Check 3: Could from_text match as a substring inside common words? + # This catches the "线数" matching inside "产线数据" bug. + if from_text in SUBSTRING_COLLISION_MAP: + collisions = SUBSTRING_COLLISION_MAP[from_text] + warnings.append(SafetyWarning( + level="error" if strict else "warning", + category="substring_collision", + message=( + f"'{from_text}' is a substring of common words: " + f"{', '.join(collisions)}. " + f"Replacing '{from_text}' with '{to_text}' will corrupt these words." + ), + suggestion=( + f"Use a context rule with negative lookahead/lookbehind to exclude " + f"matches inside these common words. Example regex: " + f"'(? None: + """ + Check if from_text appears as a substring in any common word, + where the common word is NOT the from_text itself. + """ + if len(from_text) > 4: + # Long enough that substring collisions are unlikely to be problematic + return + + collisions: List[str] = [] + for word in ALL_COMMON_WORDS: + if word == from_text: + continue + if from_text in word: + collisions.append(word) + + if collisions: + # Only show first 5 to avoid spam + shown = collisions[:5] + more = f" (and {len(collisions) - 5} more)" if len(collisions) > 5 else "" + warnings.append(SafetyWarning( + level="warning", + category="substring_collision", + message=( + f"'{from_text}' appears inside {len(collisions)} common word(s): " + f"{', '.join(shown)}{more}. " + f"This replacement may cause collateral damage." + ), + suggestion=( + f"Review whether '{from_text}→{to_text}' could corrupt any of " + f"these words. Consider using a context rule instead." + ), + )) + + +def audit_corrections( + corrections: dict[str, str], +) -> dict[str, List[SafetyWarning]]: + """ + Audit all corrections in a dictionary for safety issues. + + Used by the --audit command. + + Args: + corrections: Dict of {from_text: to_text} + + Returns: + Dict of {from_text: [warnings]} for entries with issues. + Entries with no issues are not included. + """ + results: dict[str, List[SafetyWarning]] = {} + + for from_text, to_text in corrections.items(): + warnings = check_correction_safety(from_text, to_text, strict=False) + if warnings: + results[from_text] = warnings + + return results diff --git a/transcript-fixer/scripts/utils/security.py b/transcript-fixer/scripts/utils/security.py index 69e6185c..b98cfb9b 100644 --- a/transcript-fixer/scripts/utils/security.py +++ b/transcript-fixer/scripts/utils/security.py @@ -43,7 +43,7 @@ def mask_secret(secret: str, visible_chars: int = 4) -> str: Masked string like "7fb3...DPRR" Examples: - >>> mask_secret("7fb3ab7b186242288fe93a27227b7149.bJCOEAsUfejvWDPR") + >>> mask_secret("example-fake-api-key-1234567890abcdef.test") '7fb3...DPRR' >>> mask_secret("short") @@ -248,7 +248,7 @@ class SecretStr: Wrapper for secrets that prevents accidental logging. Usage: - api_key = SecretStr("7fb3ab7b186242288fe93a27227b7149.bJCOEAsUfejvWDPR") + api_key = SecretStr("example-fake-api-key-1234567890abcdef.test") print(api_key) # Prints: SecretStr(7fb3...DPRR) print(api_key.get()) # Get actual value when needed diff --git a/tunnel-doctor/SKILL.md b/tunnel-doctor/SKILL.md index 865fbd17..84e9a581 100644 --- a/tunnel-doctor/SKILL.md +++ b/tunnel-doctor/SKILL.md @@ -27,15 +27,21 @@ Proxy/VPN tools on macOS create conflicts at five independent layers. Layers 1-3 Determine which scenario applies: - **Browser returns HTTP 503, but `curl` and SSH both work** → System proxy bypass conflict (Step 2C) +- **`local.` fails in browser/default `curl`, but direct/no-proxy request works** → Local vanity domain proxy interception (Step 2C-1) - **Tailscale ping works, SSH works, but curl/HTTP times out** → HTTP proxy env var conflict (Step 2A) - **Tailscale ping works, SSH/TCP times out** → Route conflict (Step 2B) - **Remote dev server auth redirects to `localhost` → browser can't follow** → SSH tunnel needed (Step 2D) - **`make status` / scripts curl to localhost fail with proxy** → localhost proxy interception (Step 2E) - **`git push/pull` fails with `FATAL: failed to begin relaying via HTTP`** → SSH double tunnel (Step 2F) -- **`docker pull` fails with `TLS handshake timeout` or `docker build` can't fetch base images** → VM/container proxy propagation (Step 2G) +- **`docker build` `RUN apk/apt` fails with `Connection refused` instantly** → OrbStack transparent proxy + TUN conflict (Step 2G-1, fix: `--network host`) +- **`docker pull` fails with `TLS handshake timeout`** → VM proxy misconfiguration (Step 2G-2, fix: `docker.json` with `host.internal`) +- **Container healthcheck `(unhealthy)` but app runs fine** → Lowercase proxy env var leak (Step 2G-4, fix: clear `http_proxy`+`HTTP_PROXY`) +- **`docker build` can't fetch base images** → VM/container proxy propagation (Step 2G) - **`git clone` fails with `Connection closed by 198.18.x.x`** → TUN DNS hijack for SSH (Step 2H) - **SSH connects but `operation not permitted`** → Tailscale SSH config issue (Step 4) - **SSH connects but `be-child ssh` exits code 1** → WSL snap sandbox issue (Step 5) +- **TCP port 22 reachable (`nc -z` succeeds) but SSH fails with `kex_exchange_identification: Connection closed`** → Tailscale SSH proxy intercept on WSL (Step 5A) +- **`tailscale ssh` returns "not available on App Store builds"** → Wrong Tailscale distribution on macOS (Step 5B) **Key distinctions**: - SSH does NOT use `http_proxy`/`NO_PROXY` env vars. If SSH works but HTTP doesn't → Layer 2. @@ -43,7 +49,30 @@ Determine which scenario applies: - If `tailscale ping` works but regular `ping` doesn't → Layer 1 (route table corrupted). - If `ssh -T git@github.com` works but `git push` fails intermittently → Layer 4 (double tunnel). - If host `curl https://...` works but `docker pull` times out → Layer 5 (VM proxy propagation). +- If `docker pull` works but `docker build` `RUN apk add` fails instantly with `Connection refused` → OrbStack transparent proxy broken by TUN (Step 2G-1). +- If container healthcheck shows `(unhealthy)` but app works → lowercase `http_proxy` leaked into container (Step 2G-4). - If DNS resolves to `198.18.x.x` virtual IPs → TUN DNS hijack (Step 2H). +- If `nc -z` succeeds on port 22 but SSH gets no banner (`kex_exchange_identification`) → Tailscale SSH proxy intercept (Step 5A). Confirm with `tcpdump -i any port 22` on the remote — 0 packets means Tailscale intercepts above the kernel. +- If `tailscale ssh` fails with "not available on App Store builds" → install Standalone Tailscale (Step 5B). + +### Fast Path: Run Automated Checks + +For common macOS conflicts (env proxy, system proxy exceptions, direct/proxy path split, local TLS trust), run: + +```bash +python3 scripts/quick_diagnose.py --host local.claude4.dev --url https://local.claude4.dev/health +``` + +Optional route ownership check for a Tailscale destination: + +```bash +python3 scripts/quick_diagnose.py --host --url http://:/health --tailscale-ip <100.x.x.x> +``` + +Interpretation: +- `direct=PASS` + `forced_proxy=FAIL` = host must bypass proxy (`skip-proxy` + `NO_PROXY`). +- `strict_tls=FAIL` + `direct=PASS` = path is reachable; trust issue only (install/trust local CA). +- `host in scutil exceptions: no` = browser/system clients still likely proxied. ### Step 2A: Fix HTTP Proxy Environment Variables @@ -76,6 +105,18 @@ export NO_PROXY=localhost,127.0.0.1,.ts.net,100.64.0.0/10,192.168.*,10.*,172.16. **NO_PROXY syntax pitfalls** — see [references/proxy_conflict_reference.md](references/proxy_conflict_reference.md) for the compatibility matrix. +**Go `net/http` CIDR caveat**: Go's standard `net/http` does NOT support CIDR notation in `NO_PROXY`. Setting `NO_PROXY=100.64.0.0/10` works for curl and Python, but Go programs (including Tailscale-adjacent tooling) will still send traffic through the proxy. The fix is to use MagicDNS hostnames (e.g., `workstation-4090-wsl`) instead of raw IPs, or add explicit hostnames to `NO_PROXY`: + +```bash +# WRONG for Go programs — CIDR is silently ignored +NO_PROXY=100.64.0.0/10 go-program http://100.101.102.103:8002/health # → goes through proxy + +# CORRECT — use hostname (matched as suffix) or explicit IP +export NO_PROXY=localhost,127.0.0.1,.ts.net,workstation-4090-wsl,100.101.102.103,192.168.*,10.*,172.16.* +``` + +This is especially relevant when accessing Tailscale services from Go-based tools (e.g., custom CLIs, Go test suites hitting remote APIs). + Verify the fix: ```bash @@ -107,6 +148,19 @@ gateway: 192.168.x.1 # Default gateway interface: en0 # Physical interface, NOT Tailscale ``` +**Important**: Not all `utun` interfaces are Tailscale's. Verify which utun belongs to Tailscale before concluding the route is correct: + +```bash +# Find Tailscale's utun interface (has a 100.x.x.x IP) +ifconfig | grep -A2 'inet 100\.' +``` + +Quick indicators by MTU: +- **MTU 1280** → typically Tailscale +- **MTU 4064** → typically Shadowrocket TUN + +If `route -n get` shows traffic going to a utun with MTU 4064, it is hitting Shadowrocket's TUN, not Tailscale — this is still a route conflict even though the interface name starts with `utun`. + Confirm with full route table: ```bash @@ -150,6 +204,41 @@ skip-proxy = 192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12, 100.64.0.0/10, localhost - `skip-proxy`: Bypasses the HTTP proxy layer only. Traffic still flows through the TUN interface and Tailscale utun handles it. Safe. - `tun-excluded-routes`: Removes the CIDR from the TUN routing entirely. This creates a competing `en0` route that overrides Tailscale. Breaks everything. +#### Step 2C-1: Fix Local Vanity Domain Interception (`local.`) + +**Symptom**: `https://local.` fails in browser or default `curl`, but succeeds with direct/no-proxy command: + +```bash +env -u http_proxy -u https_proxy curl -k -I https://local./health +# -> 200 +curl -I https://local./health +# -> proxy CONNECT then TLS reset/failure +``` + +**Root cause**: The domain is routed through system/shell proxy instead of local direct path. + +**Fix**: +1. Add domain to proxy app bypass list (`skip-proxy` for Shadowrocket). +2. Add domain to shell bypass list (`NO_PROXY`/`no_proxy`). +3. If local TLS uses internal CA, trust the local root certificate. + +```bash +# ~/.zshrc +export NO_PROXY=localhost,127.0.0.1,.ts.net,100.64.0.0/10,192.168.*,10.*,172.16.*,local.,www.local. +export no_proxy="$NO_PROXY" +``` + +**Verification**: + +```bash +python3 scripts/quick_diagnose.py --host local. --url https://local./health +``` + +Expected: +- `host in NO_PROXY: yes` +- `host in scutil exceptions: yes` +- `ambient=PASS` and `direct=PASS` + ### Step 2D: Fix Auth Redirect for Remote Dev (SSH Tunnel) **Symptom**: Dev server runs on a remote machine (e.g., Mac Mini via Tailscale). You access `http://:3010` in the browser. Login/signup works, but after auth, the app redirects to `http://localhost:3010/` which fails — `localhost` on your machine isn't running the dev server. @@ -234,7 +323,7 @@ GIT_SSH_COMMAND="ssh -o ProxyCommand=none" git push origin main ### Step 2G: Fix VM/Container Runtime Proxy Propagation (Docker pull/build failures) -**Symptom**: `docker pull` or `docker build` fails with `net/http: TLS handshake timeout` or `Internal Server Error` from `auth.docker.io`, while host `curl` to the same URLs works fine. +**Symptom**: `docker pull` or `docker build` fails with `net/http: TLS handshake timeout`, `Connection refused` from Alpine/Debian repos, or `Internal Server Error` from `auth.docker.io`, while host `curl` to the same URLs works fine. **Applies to**: OrbStack, Docker Desktop, or any VM-based Docker runtime on macOS with Shadowrocket/Clash TUN active. @@ -247,66 +336,160 @@ VM process (Docker): Docker daemon → VM bridge → host network → TUN → The TUN handles host-originated traffic correctly but may drop or delay VM-bridged traffic (different TCP stack, MTU, keepalive behavior). -**Three sub-problems and their fixes**: +**Critical distinction: `docker pull` vs `docker build` use different proxy paths**: + +| Operation | Proxy source | What controls it | +|-----------|-------------|------------------| +| `docker pull` | Docker daemon config | `~/.orbstack/config/docker.json` or `docker info` | +| `docker build` (`RUN apt/apk`) | Build container env | `--build-arg http_proxy=...` or `--network host` | +| `docker run` | Container env | `-e http_proxy=...` or inherited from daemon | + +Fixing `docker.json` alone will NOT fix `docker build` — the `RUN` commands inside the build container don't inherit daemon proxy settings. + +**Diagnosis** — identify which sub-problem: + +```bash +# 1. Can the Docker daemon pull images? +docker pull --quiet alpine:latest 2>&1 + +# 2. Can a RUN command inside a build reach the internet? +docker build --no-cache - <<'EOF' 2>&1 +FROM alpine:latest +RUN apk update && echo "APK OK" +EOF + +# 3. Can a running container reach the internet? +docker run --rm alpine:latest sh -c "apk update 2>&1 | head -3" +``` + +**Four sub-problems and their fixes**: + +#### 2G-1: `docker build` fails but host works (most common with OrbStack + Shadowrocket) + +**Symptom**: `RUN apk add` or `RUN apt-get install` inside `docker build` fails with `Connection refused` instantly (< 0.2s), even though host `curl` to the same URL works. + +**Root cause**: OrbStack's `network_proxy: auto` creates a transparent proxy inside the VM that intercepts all HTTPS traffic. When Shadowrocket TUN is also active, the transparent proxy's upstream connection breaks — it redirects HTTPS to `127.0.0.1` inside the VM, which has nothing listening. + +**Diagnosis**: + +```bash +# Verify: inside the container, HTTPS goes to 127.0.0.1 (broken transparent proxy) +docker run --rm alpine:latest sh -c "wget -q --timeout=5 -O /dev/null https://dl-cdn.alpinelinux.org/ 2>&1" +# → "wget: can't connect to remote host (127.0.0.1): Connection refused" +# ^^^^^^^^^^^^ This is the smoking gun + +# Verify: --network host bypasses the VM bridge and works +docker run --rm --network host alpine:latest sh -c "apk update 2>&1 | head -3" +# → "v3.23.x ... OK: 27431 distinct packages available" ← Works! +``` + +**Fix** — use `--network host` for docker build: + +```bash +docker build --network host -f Dockerfile -t myimage . +``` + +This bypasses OrbStack's VM network bridge entirely. The build container uses the host's network stack directly, where Shadowrocket TUN correctly handles traffic. + +**Trade-off**: `--network host` disables build-time network isolation. For CI/CD, prefer fixing the proxy config (2G-2). For local development, `--network host` is the pragmatic fix. + +**Permanent fix** — if all your builds need this, add to `~/.docker/daemon.json` or use a shell alias: + +```bash +# Shell alias (add to ~/.zshrc) +alias docker-build='docker build --network host' +``` + +#### 2G-2: OrbStack auto-detects and caches proxy config -#### 2G-1: OrbStack auto-detects and caches proxy (most common) +OrbStack's `network_proxy: auto` reads `http_proxy` from the shell environment and configures the Docker daemon. The config is stored in `~/.orbstack/config/docker.json`. -OrbStack's `network_proxy: auto` reads `http_proxy` from the shell environment and writes it to `~/.orbstack/config/docker.json`. **Crucially**, `orbctl config set network_proxy none` does NOT clean up `docker.json` — the cached proxy persists. +**Key behaviors**: +- `network_proxy: auto` — OrbStack reads host env, creates transparent proxy in VM +- `network_proxy: none` — Disables transparent proxy, but VM bridge traffic still routes through TUN (may timeout) +- `docker.json` — Controls `docker pull` proxy, NOT `docker build` RUN commands **Diagnosis**: ```bash -# OrbStack config says "none" but Docker still shows proxy -orbctl config get network_proxy # → "none" -docker info | grep -i proxy # → HTTP Proxy: http://127.0.0.1:1082 ← stale! +# Check all three layers +echo "=== OrbStack config ===" +orbctl config get network_proxy -# The real source of truth: +echo "=== docker.json (daemon proxy) ===" cat ~/.orbstack/config/docker.json -# → {"proxies": {"http-proxy": "http://127.0.0.1:1082", ...}} ← cached! + +echo "=== Docker info (effective proxy) ===" +docker info | grep -iE "proxy|No Proxy" ``` -**Fix** — DON'T remove the proxy. Instead, add precise `no-proxy` to prevent localhost interception while keeping the proxy as the VM's outbound channel: +**Fix** — configure `docker.json` with `host.internal` (OrbStack resolves this to the host IP): ```bash -# Write corrected config (keeps proxy, adds no-proxy for local traffic) python3 -c " -import json +import json, os config = { 'proxies': { - 'http-proxy': 'http://127.0.0.1:1082', - 'https-proxy': 'http://127.0.0.1:1082', + 'http-proxy': 'http://host.internal:1082', + 'https-proxy': 'http://host.internal:1082', 'no-proxy': 'localhost,127.0.0.1,::1,192.168.128.0/24,100.64.0.0/10,host.internal,*.local' } } -json.dump(config, open('$HOME/.orbstack/config/docker.json', 'w'), indent=2) +path = os.path.expanduser('~/.orbstack/config/docker.json') +json.dump(config, open(path, 'w'), indent=2) +print('Written:', path) " -# Full restart (not just docker engine) +# Full restart required orbctl stop && sleep 3 && orbctl start ``` -**Why NOT remove the proxy**: When TUN is active, removing the Docker proxy means VM traffic goes directly through the bridge → TUN path, which causes TLS handshake timeouts. The proxy provides a working outbound channel because OrbStack maps host `127.0.0.1` into the VM. +**Important**: Use `host.internal` (OrbStack-specific), NOT `127.0.0.1` (points to VM loopback) and NOT `host.docker.internal` (may not resolve in all contexts). -#### 2G-2: Removing proxy makes Docker worse (counter-intuitive) +**Why NOT remove the proxy**: When TUN is active, removing the Docker proxy means VM traffic goes directly through the bridge → TUN path, which causes TLS handshake timeouts. The proxy provides a working outbound channel. + +#### 2G-3: Removing proxy makes Docker worse (counter-intuitive) | Docker config | Traffic path | Result | |---------------|-------------|--------| -| Proxy ON, no `no-proxy` | Docker → proxy → TUN → internet | Docker Hub ✅, localhost probes ❌ | -| Proxy OFF | Docker → VM bridge → host → TUN → internet | TLS timeout ❌ | -| **Proxy ON + `no-proxy`** | **External: Docker → proxy → internet ✅; Local: Docker → direct ✅** | **Both work ✅** | +| Proxy ON (`127.0.0.1`), no `no-proxy` | Docker → VM proxy → ??? | `docker pull` may work, localhost probes ❌ | +| Proxy ON (`host.internal`), + `no-proxy` | External: Docker → host proxy → internet; Local: direct | **Both work ✅** | +| Proxy OFF (`network_proxy: none`) | Docker → VM bridge → host → TUN → internet | TLS timeout ❌ | +| **`--network host` (build only)** | **Build container → host network → TUN → internet** | **Build works ✅** | + +**Decision tree**: +- `docker pull` broken → Fix `docker.json` with `host.internal` proxy (2G-2) +- `docker build` broken → Use `--network host` (2G-1) OR pass `--build-arg http_proxy=http://host.internal:1082` +- Both broken → Fix both: `docker.json` + `--network host` + +#### 2G-4: Deploy scripts and container healthchecks probe localhost through proxy + +Deploy scripts that `curl localhost` inside containers or Docker healthchecks that use `wget http://localhost` will route through the proxy if env vars leak into the container. + +**Common symptoms**: +- Container healthcheck shows `(unhealthy)` but the app inside is running fine +- `wget: can't connect to remote host (127.0.0.1): Connection refused` in healthcheck logs (proxy port, not app port) -#### 2G-3: Deploy scripts probe localhost through proxy +**Root cause**: Docker inherits uppercase AND lowercase proxy env vars from the host. Many tools only clear uppercase (`HTTP_PROXY=`) but forget lowercase (`http_proxy=http://127.0.0.1:1082`). The healthcheck `wget` uses lowercase. -Deploy scripts that `curl localhost` inside the Docker environment will route through the proxy. Fix by adding `NO_PROXY` at the script level: +**Fix in docker-compose.yml** — clear BOTH cases: + +```yaml +environment: + # Must clear both uppercase and lowercase — wget/curl check different vars + - HTTP_PROXY= + - HTTPS_PROXY= + - http_proxy= + - https_proxy= + - NO_PROXY=* + - no_proxy=* +``` + +**Fix in deploy scripts**: ```bash -# In deploy.sh or similar scripts: _local_bypass="localhost,127.0.0.1,::1" -if [[ -n "${NO_PROXY:-}" ]]; then - export NO_PROXY="${_local_bypass},${NO_PROXY}" -else - export NO_PROXY="${_local_bypass}" -fi +export NO_PROXY="${_local_bypass}${NO_PROXY:+,${NO_PROXY}}" export no_proxy="$NO_PROXY" # Use 127.0.0.1 instead of localhost in probe URLs (some proxy implementations @@ -324,8 +507,15 @@ docker info | grep -iE "proxy|No Proxy" # Pull test docker pull --quiet hello-world -# Local probe test -curl -s http://127.0.0.1:3001/health +# Build test (the real verification) +docker build --network host --no-cache - <<'EOF' +FROM alpine:latest +RUN apk update && echo "BUILD OK" +EOF + +# Container env check (no proxy leak) +docker exec env | grep -i proxy +# Expected: all empty or not set ``` ### Step 2H: Fix TUN DNS Hijack for SSH/Git (198.18.x.x virtual IPs) @@ -453,13 +643,89 @@ sudo tailscale up --ssh **Important**: The new installation may assign a different Tailscale IP. Check with `tailscale status --self`. +### Step 5A: Fix Tailscale SSH Proxy Silent Failure on WSL + +**Symptom**: TCP port 22 is reachable (`nc -z -w 5 22` succeeds), but SSH fails immediately with: + +``` +kex_exchange_identification: Connection closed by remote host +``` + +No SSH banner is ever received. This happens even with apt-installed Tailscale (not snap). + +**Root cause**: When `tailscale up --ssh` is enabled on WSL, Tailscale intercepts port 22 connections at the application layer (above the kernel network stack). If Tailscale's built-in SSH proxy malfunctions, it accepts the TCP connection but immediately closes it before sending the SSH banner. + +**Key diagnostic** — on the WSL instance: + +```bash +# This will show 0 packets even during active SSH attempts +sudo tcpdump -i any port 22 -c 5 -w /dev/null 2>&1 +``` + +Zero packets means Tailscale is intercepting connections before they reach the kernel network stack. The kernel's `sshd` never sees the connection. + +**Distinction from Step 5**: Step 5 covers snap sandbox issues where `be-child ssh` fails. This is a different problem — Tailscale's SSH proxy itself silently fails, regardless of installation method. + +**Fix** — disable Tailscale's SSH proxy and use regular sshd: + +```bash +# On the WSL instance: +sudo tailscale up --ssh=false + +# Verify sshd is running +sudo service ssh status +# If not running: +sudo service ssh start + +# Verify from the client machine: +ssh -o ConnectTimeout=10 @ 'echo SSH_OK' +``` + +After disabling Tailscale SSH, connections go through the kernel network stack to `sshd` as normal. The Tailscale ACL `"action": "accept"` in Step 4 is no longer relevant — authentication is handled by `sshd` using SSH keys or passwords. + +**When to keep `--ssh` enabled**: Only if you specifically need Tailscale's SSH features (ACL-based access control, no SSH key management). If standard sshd works, prefer `--ssh=false` for reliability. + +### Step 5B: Fix App Store Tailscale on macOS (Missing `tailscale ssh`) + +**Symptom**: Running `tailscale ssh` returns: + +``` +The 'tailscale ssh' subcommand is not available on macOS builds +distributed through the App Store or TestFlight. +``` + +**Root cause**: The App Store version of Tailscale for macOS is sandboxed and does not include the `tailscale ssh` subcommand. + +**Fix** — install the Standalone version: + +1. Uninstall the App Store version (delete from /Applications) +2. Download the Standalone build from https://pkgs.tailscale.com/stable/#macos +3. Install to /Applications + +**Post-install CLI setup**: The standalone `tailscale` CLI binary is embedded inside the app bundle. Add an alias to your shell config: + +```bash +# ~/.zshrc +alias tailscale="/Applications/Tailscale.app/Contents/MacOS/Tailscale" +``` + +Verify: + +```bash +source ~/.zshrc +tailscale version +tailscale ssh @ # Should work now +``` + ### Step 6: Verify End-to-End Run a complete connectivity test: ```bash -# 1. Check route is correct +# 1. Check route is correct (must show Tailscale's utun, not en0 or Shadowrocket's utun) route -n get +# Also confirm which utun is Tailscale's: +ifconfig | grep -A2 'inet 100\.' # 2. Test TCP connectivity nc -z -w 5 22 @@ -468,7 +734,7 @@ nc -z -w 5 22 ssh -o ConnectTimeout=10 -o StrictHostKeyChecking=no @ 'echo SSH_OK && hostname && whoami' ``` -All three must pass. If step 1 fails, revisit Step 3. If step 2 fails, check WSL sshd or firewall. If step 3 fails, revisit Steps 4-5. +All three must pass. If step 1 fails, revisit Step 3. If step 1 shows wrong utun (e.g., Shadowrocket's utun with MTU 4064 instead of Tailscale's with MTU 1280), that is also a route conflict. If step 2 passes but step 3 fails with `kex_exchange_identification`, revisit Step 5A (Tailscale SSH proxy intercept). If step 2 fails, check WSL sshd or firewall. If step 3 fails with other errors, revisit Steps 4-5. ## SOP: Remote Development via Tailscale @@ -557,7 +823,13 @@ Each `-L` flag is independent. If one port is already bound locally, `ExitOnForw ### 4. SSH Non-Login Shell Setup -SSH non-login shells don't load `~/.zshrc`, so nvm/Homebrew tools and proxy env vars are unavailable. Prefix all remote commands with `source ~/.zshrc 2>/dev/null;`. See [references/proxy_conflict_reference.md § SSH Non-Login Shell Pitfall](references/proxy_conflict_reference.md) for details and examples. +**This is a frequent source of "it works interactively but fails in scripts" bugs.** SSH non-login shells don't load `~/.zshrc` (or `~/.bashrc` on Linux), so tools installed via nvm, Homebrew, uv, cargo, or any shell-level manager won't be in `$PATH`. Proxy env vars set in `~/.zshrc` also won't be loaded. + +This affects **all** remote commands run via `ssh user@host "command"`, including CI/CD pipelines, cron-triggered SSH, and Makefile remote targets. Prefix all remote commands with `source ~/.zshrc 2>/dev/null;` (macOS) or `source ~/.bashrc 2>/dev/null;` (Linux/WSL). + +**Common failure**: `ssh user@host "uv run ..."` or `ssh user@host "node ..."` returns `command not found` even though the command works in an interactive SSH session. + +See [references/proxy_conflict_reference.md § SSH Non-Login Shell Pitfall](references/proxy_conflict_reference.md) for details and examples. For Makefile targets that run remote commands: diff --git a/tunnel-doctor/references/proxy_conflict_reference.md b/tunnel-doctor/references/proxy_conflict_reference.md index 03f35e3b..34fed4cf 100644 --- a/tunnel-doctor/references/proxy_conflict_reference.md +++ b/tunnel-doctor/references/proxy_conflict_reference.md @@ -68,6 +68,14 @@ NO_PROXY="" curl -s -X POST "http://:8080/api/ curl --noproxy '*' -s --connect-timeout 2 "http://192.168.31.110:8080/api/read" | head -1 ``` +**Port conflict warning**: Shadowrocket's config API listens on port 8080 by default, which may conflict with other services (e.g., whisper.cpp server, development proxies). If the API returns unexpected content (HTML, JSON from another service), verify what is actually listening on the port: + +```bash +lsof -nP -iTCP:8080 | head -5 +``` + +If another service owns port 8080, you need to either stop that service or access the Shadowrocket API from a different device on the same network. + **Critical**: Use `--data-binary`, NOT `-d`. The `-d` flag URL-encodes the content, corrupting `#`, `=`, `&` and other characters in the config. This **destroys the entire configuration** — all rules, settings, and proxy groups are lost. The user must restore from backup. ```bash @@ -88,6 +96,31 @@ tun-excluded-routes = 10.0.0.0/8, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 19 Note: `100.64.0.0/10` is intentionally absent. +### Complete Working Reference Config for Tailscale Compatibility + +This is a validated reference showing the correct relationship between `skip-proxy`, `tun-excluded-routes`, and `[Rule]` for Tailscale coexistence: + +``` +[General] +# skip-proxy: bypass the HTTP proxy for these destinations (fixes browser 503) +# 100.64.0.0/10 MUST be here for browser access to Tailscale IPs +skip-proxy = 192.168.0.0/16, 10.0.0.0/8, 172.16.0.0/12, 100.64.0.0/10, localhost, *.local, captive.apple.com + +# tun-excluded-routes: CIDRs excluded from TUN routing (sent directly via physical interface) +# 100.64.0.0/10 must NOT be here — including it creates an en0 route that overrides Tailscale +tun-excluded-routes = 10.0.0.0/8, 127.0.0.0/8, 169.254.0.0/16, 172.16.0.0/12, 192.0.0.0/24, 192.0.2.0/24, 192.88.99.0/24, 192.168.0.0/16, 198.51.100.0/24, 203.0.113.0/24, 224.0.0.0/4, 255.255.255.255/32 + +[Rule] +# Tailscale traffic enters TUN but is passed through without proxying +IP-CIDR,100.64.0.0/10,DIRECT +# ... other rules ... +``` + +**Key points**: +- `skip-proxy` — YES, include `100.64.0.0/10` (browser bypass) +- `tun-excluded-routes` — NO, never include `100.64.0.0/10` (would hijack routing) +- `[Rule]` — YES, include `IP-CIDR,100.64.0.0/10,DIRECT` (TUN passthrough) + ## Clash / ClashX Pro ### The Fix @@ -151,12 +184,15 @@ export NO_PROXY=localhost,127.0.0.1,.ts.net,100.64.0.0/10,192.168.*,10.*,172.16. ### NO_PROXY Syntax Pitfalls -| Syntax | curl | Python requests | Node.js | Meaning | -|--------|------|-----------------|---------|---------| -| `.ts.net` | ✅ | ✅ | ✅ | Domain suffix match (correct) | -| `*.ts.net` | ❌ | ✅ | varies | Glob — curl does NOT support this | -| `100.64.0.0/10` | ✅ 7.86+ | ✅ 2.25+ | ❌ native | CIDR notation | -| `100.*` | ✅ | ✅ | ✅ | Too broad — covers public IPs `100.0-63.*` and `100.128-255.*` | +| Syntax | curl | Python requests | Go `net/http` | Node.js | Meaning | +|--------|------|-----------------|---------------|---------|---------| +| `.ts.net` | ✅ | ✅ | ✅ | ✅ | Domain suffix match (correct) | +| `*.ts.net` | ❌ | ✅ | ❌ | varies | Glob — curl and Go do NOT support this | +| `100.64.0.0/10` | ✅ 7.86+ | ✅ 2.25+ | ❌ | ❌ native | CIDR notation — Go silently ignores it | +| `100.*` | ✅ | ✅ | ❌ | ✅ | Too broad — covers public IPs `100.0-63.*` and `100.128-255.*` | +| `workstation-name` | ✅ | ✅ | ✅ | ✅ | Exact hostname match (safest for Go) | + +**Go `net/http` warning**: Go's proxy bypass logic (`httpproxy.Config.ProxyFunc`) does not implement CIDR matching. `NO_PROXY=100.64.0.0/10` is silently ignored — Go programs will still route traffic through the proxy. Use MagicDNS hostnames (e.g., `workstation-4090-wsl`) or explicit IPs (e.g., `100.101.102.103`) instead of CIDR ranges when Go programs need to bypass the proxy. **Key rule**: Always use `.ts.net` (leading dot, no asterisk) for domain suffix matching. This is the most portable syntax across all HTTP clients. @@ -282,9 +318,9 @@ Connection setup is slightly slower (~6s vs ~2s) because TUN routing has more ne ## General Principles -### Four Conflict Layers +### Five Conflict Layers -Proxy tools create conflicts at four independent layers on macOS. Layers 1-3 affect Tailscale connectivity; Layer 4 affects SSH git operations through the same proxy infrastructure: +Proxy tools create conflicts at five independent layers on macOS. Layers 1-3 affect Tailscale connectivity; Layer 4 affects SSH git operations; Layer 5 affects VM/container runtimes: | Layer | Setting | What it controls | Symptom when wrong | |-------|---------|------------------|--------------------| @@ -292,6 +328,7 @@ Proxy tools create conflicts at four independent layers on macOS. Layers 1-3 aff | 2. HTTP env vars | `http_proxy` / `NO_PROXY` | CLI tools (curl, wget, Python, Node.js) | `curl` times out, SSH works, browser works | | 3. System proxy | `skip-proxy` | Browser and system HTTP clients | Browser 503, `curl` works (both with/without proxy), SSH works | | 4. SSH ProxyCommand | `ProxyCommand connect -H` | SSH git operations (push/pull/clone) | `ssh -T` works, `git push` fails intermittently with `failed to begin relaying via HTTP` | +| 5. VM/Container proxy | Docker/OrbStack proxy config | `docker pull`, `docker build` | Host `curl` works, `docker pull` times out (TLS handshake timeout) | **Each layer is independent.** A fix at one layer doesn't help the others. You may need fixes at multiple layers simultaneously. @@ -398,4 +435,6 @@ If a proxy config change breaks Tailscale connectivity: sudo route delete -net 100.64.0.0/10 ``` +**Important**: Manually deleting a bad `en0` route with `sudo route delete` is only a temporary fix. Shadowrocket will re-add the route when the VPN connection is next reconnected or toggled. The only permanent fix is modifying the Shadowrocket configuration to remove `100.64.0.0/10` from `tun-excluded-routes` (it should never be there). + If `tun-excluded-routes` was modified, reverting it and restarting Shadowrocket will restore Tailscale's routing immediately. diff --git a/tunnel-doctor/scripts/quick_diagnose.py b/tunnel-doctor/scripts/quick_diagnose.py new file mode 100755 index 00000000..a53b833b --- /dev/null +++ b/tunnel-doctor/scripts/quick_diagnose.py @@ -0,0 +1,522 @@ +#!/usr/bin/env python3 +""" +Quick tunnel/proxy conflict diagnostics for macOS. + +This script detects the most common local and Tailscale networking conflicts: +1) Shell proxy env + NO_PROXY mismatch +2) System proxy exceptions mismatch +3) Proxy path failure vs direct path success +4) Local TLS trust issues +5) Route ownership conflicts for a Tailscale IP (optional) +""" + +import argparse +import json +import os +import re +import shlex +import socket +import subprocess +import sys +from typing import Dict, List, Optional, Tuple + + +def run(cmd: List[str], env: Optional[Dict[str, str]] = None) -> Tuple[int, str, str]: + proc = subprocess.run( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + ) + return proc.returncode, proc.stdout.strip(), proc.stderr.strip() + + +def split_csv(value: str) -> List[str]: + if not value: + return [] + return [item.strip() for item in value.split(",") if item.strip()] + + +def match_proxy_pattern(host: str, pattern: str) -> bool: + p = pattern.strip().lower() + h = host.strip().lower() + if not p or not h: + return False + + # CIDR / wildcard IP patterns are not domain checks. + if "/" in p and not p.startswith("*."): + return False + if re.match(r"^\d+\.\d+\.\d+\.\*$", p): + return False + + if p == h: + return True + if p.startswith("*."): + suffix = p[1:] # keep leading dot + return h.endswith(suffix) + if p.startswith("."): + return h.endswith(p) + return False + + +def has_host_bypass(host: str, patterns: List[str]) -> bool: + return any(match_proxy_pattern(host, item) for item in patterns) + + +def parse_scutil_proxy() -> Dict[str, object]: + code, stdout, _stderr = run(["scutil", "--proxy"]) + if code != 0: + return {"raw": "", "exceptions": [], "http_enabled": False, "https_enabled": False} + + raw = stdout + exceptions: List[str] = [] + for line in raw.splitlines(): + m = re.search(r"\d+\s*:\s*(.+)$", line) + if m and "ExceptionsList" not in line: + exceptions.append(m.group(1).strip()) + + http_enabled = bool(re.search(r"HTTPEnable\s*:\s*1", raw)) + https_enabled = bool(re.search(r"HTTPSEnable\s*:\s*1", raw)) + + return { + "raw": raw, + "exceptions": exceptions, + "http_enabled": http_enabled, + "https_enabled": https_enabled, + } + + +def resolve_host(host: str) -> List[str]: + ips = [] + try: + infos = socket.getaddrinfo(host, None) + for item in infos: + ip = item[4][0] + if ip not in ips: + ips.append(ip) + except socket.gaierror: + pass + return ips + + +def curl_status(url: str, timeout: int, mode: str, proxy_url: Optional[str] = None) -> Dict[str, object]: + cmd = [ + "curl", + "-k", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + str(timeout), + url, + ] + + env = os.environ.copy() + if mode == "direct": + for key in ( + "http_proxy", + "https_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "all_proxy", + "ALL_PROXY", + ): + env.pop(key, None) + elif mode == "forced_proxy" and proxy_url: + cmd.extend(["--proxy", proxy_url]) + + code, stdout, stderr = run(cmd, env=env) + http_code = stdout if stdout else "000" + ok = code == 0 and http_code.isdigit() and http_code != "000" + + return { + "ok": ok, + "http_code": http_code, + "exit_code": code, + "stderr": stderr, + "command": " ".join(shlex.quote(x) for x in cmd), + } + + +def strict_tls_check(url: str, timeout: int) -> Dict[str, object]: + cmd = [ + "curl", + "-sS", + "-o", + "/dev/null", + "-w", + "%{http_code}", + "--max-time", + str(timeout), + url, + ] + env = os.environ.copy() + for key in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY", "all_proxy", "ALL_PROXY"): + env.pop(key, None) + code, stdout, stderr = run(cmd, env=env) + cert_issue = "certificate" in stderr.lower() or "ssl" in stderr.lower() + return { + "ok": code == 0 and stdout != "000", + "http_code": stdout if stdout else "000", + "exit_code": code, + "stderr": stderr, + "cert_issue": cert_issue, + } + + +def find_tailscale_utun() -> Optional[str]: + """Find which utun interface belongs to Tailscale (has a 100.x.x.x IP).""" + code, stdout, _ = run(["ifconfig"]) + if code != 0: + return None + current_iface = "" + for line in stdout.splitlines(): + # Interface header line (e.g., "utun7: flags=...") + m = re.match(r"^(\w+):", line) + if m: + current_iface = m.group(1) + # Look for Tailscale CGNAT IP on a utun interface + if current_iface.startswith("utun") and "inet 100." in line: + return current_iface + return None + + +def get_iface_mtu(iface: str) -> Optional[int]: + """Get MTU of a network interface.""" + code, stdout, _ = run(["ifconfig", iface]) + if code != 0: + return None + m = re.search(r"mtu\s+(\d+)", stdout) + return int(m.group(1)) if m else None + + +def route_check(tailscale_ip: str) -> Dict[str, object]: + code, stdout, stderr = run(["route", "-n", "get", tailscale_ip]) + if code != 0: + return {"ok": False, "interface": "", "gateway": "", "raw": stderr or stdout} + + interface = "" + gateway = "" + for line in stdout.splitlines(): + line = line.strip() + if line.startswith("interface:"): + interface = line.split(":", 1)[1].strip() + if line.startswith("gateway:"): + gateway = line.split(":", 1)[1].strip() + + # Identify which utun is Tailscale's and whether the route points to it + tailscale_utun = find_tailscale_utun() + route_mtu = get_iface_mtu(interface) if interface else None + is_tailscale_iface = (interface == tailscale_utun) if tailscale_utun else None + wrong_utun = ( + interface.startswith("utun") + and tailscale_utun is not None + and interface != tailscale_utun + ) + + return { + "ok": True, + "interface": interface, + "gateway": gateway, + "tailscale_utun": tailscale_utun or "", + "route_iface_mtu": route_mtu, + "is_tailscale_iface": is_tailscale_iface, + "wrong_utun": wrong_utun, + "raw": stdout, + } + + +def pick_proxy_url() -> Optional[str]: + for key in ("http_proxy", "HTTP_PROXY", "https_proxy", "HTTPS_PROXY"): + value = os.environ.get(key) + if value: + return value + return None + + +def build_report( + host: str, + url: str, + timeout: int, + tailscale_ip: Optional[str], +) -> Dict[str, object]: + no_proxy = os.environ.get("NO_PROXY", "") + no_proxy_lc = os.environ.get("no_proxy", "") + no_proxy_entries = split_csv(no_proxy if no_proxy else no_proxy_lc) + + scutil_info = parse_scutil_proxy() + scutil_exceptions = scutil_info["exceptions"] + + proxy_url = pick_proxy_url() + direct = curl_status(url, timeout, mode="direct") + ambient = curl_status(url, timeout, mode="ambient") + forced_proxy = curl_status(url, timeout, mode="forced_proxy", proxy_url=proxy_url) if proxy_url else None + strict_tls = strict_tls_check(url, timeout) if url.startswith("https://") else None + + host_ips = resolve_host(host) + host_in_no_proxy = has_host_bypass(host, no_proxy_entries) + host_in_scutil_exceptions = has_host_bypass(host, scutil_exceptions) + + findings: List[Dict[str, str]] = [] + + if not host_ips: + findings.append( + { + "level": "error", + "title": "Host resolution failed", + "detail": f"{host} could not be resolved. Check DNS/hosts first.", + "fix": f"Add a hosts entry if this is local: 127.0.0.1 {host}", + } + ) + + if proxy_url and not host_in_no_proxy: + findings.append( + { + "level": "warn", + "title": "NO_PROXY missing target host", + "detail": f"Proxy is enabled ({proxy_url}) but NO_PROXY does not match {host}.", + "fix": ( + "Add host to NO_PROXY/no_proxy, e.g. " + f"NO_PROXY=...,{host}" + ), + } + ) + + if (scutil_info["http_enabled"] or scutil_info["https_enabled"]) and not host_in_scutil_exceptions: + findings.append( + { + "level": "warn", + "title": "System proxy exception missing target host", + "detail": f"scutil active exceptions do not include {host}.", + "fix": ( + "Add host to proxy app skip/bypass list (Shadowrocket/Clash/Surge), " + "then reload profile." + ), + } + ) + + if direct["ok"] and forced_proxy and not forced_proxy["ok"]: + findings.append( + { + "level": "error", + "title": "Proxy path is broken for target host", + "detail": ( + "Direct access works, but forced proxy tunnel fails. " + "Traffic must bypass proxy for this host." + ), + "fix": ( + f"Add {host} to both NO_PROXY and proxy app skip-proxy/DIRECT rules." + ), + } + ) + + if not ambient["ok"] and direct["ok"]: + findings.append( + { + "level": "error", + "title": "Ambient shell path fails while direct path works", + "detail": ( + "Current shell env/proxy settings break default access." + ), + "fix": ( + "Use NO_PROXY for this host, or temporarily unset proxy env for local verification." + ), + } + ) + + if strict_tls and not strict_tls["ok"] and direct["ok"] and strict_tls["cert_issue"]: + findings.append( + { + "level": "warn", + "title": "TLS trust issue detected", + "detail": "Network path is reachable, but strict TLS validation failed.", + "fix": "Trust local CA certificate (for local/internal TLS) or use a valid public cert.", + } + ) + + route_info = route_check(tailscale_ip) if tailscale_ip else None + if route_info and route_info["ok"]: + iface = str(route_info["interface"]) + ts_utun = str(route_info.get("tailscale_utun", "")) + route_mtu = route_info.get("route_iface_mtu") + wrong_utun = route_info.get("wrong_utun", False) + + if iface.startswith("en"): + findings.append( + { + "level": "error", + "title": "Possible route hijack for Tailscale destination", + "detail": f"route -n get {tailscale_ip} resolved to {iface}, not utun*.", + "fix": ( + "Check proxy TUN excluded-routes. Do not exclude 100.64.0.0/10 from TUN route table." + ), + } + ) + elif wrong_utun: + mtu_hint = f" (MTU {route_mtu})" if route_mtu else "" + findings.append( + { + "level": "error", + "title": "Route points to wrong utun interface", + "detail": ( + f"route -n get {tailscale_ip} resolved to {iface}{mtu_hint}, " + f"but Tailscale is on {ts_utun}. " + f"Likely hitting Shadowrocket/VPN TUN (MTU 4064) instead of Tailscale (MTU 1280)." + ), + "fix": ( + "Check proxy TUN excluded-routes and rule ordering. " + "Ensure IP-CIDR,100.64.0.0/10,DIRECT is in proxy rules." + ), + } + ) + + summary = { + "host": host, + "url": url, + "host_ips": host_ips, + "proxy_url": proxy_url or "", + "env_no_proxy": no_proxy if no_proxy else no_proxy_lc, + "host_in_no_proxy": host_in_no_proxy, + "scutil_http_enabled": scutil_info["http_enabled"], + "scutil_https_enabled": scutil_info["https_enabled"], + "host_in_scutil_exceptions": host_in_scutil_exceptions, + "connectivity": { + "ambient": ambient, + "direct": direct, + "forced_proxy": forced_proxy, + "strict_tls": strict_tls, + }, + "tailscale_route": route_info, + "findings": findings, + } + return summary + + +def print_human(report: Dict[str, object]) -> int: + print("=== Tunnel Doctor Quick Diagnose ===") + print(f"Host: {report['host']}") + print(f"URL: {report['url']}") + ips = report["host_ips"] + print(f"Resolved IPs: {', '.join(ips) if ips else 'N/A'}") + print("") + + print("Proxy Context") + print(f"- proxy env: {report['proxy_url'] or '(not set)'}") + print(f"- host in NO_PROXY: {'yes' if report['host_in_no_proxy'] else 'no'}") + print( + "- system proxy enabled: " + f"HTTP={'yes' if report['scutil_http_enabled'] else 'no'} " + f"HTTPS={'yes' if report['scutil_https_enabled'] else 'no'}" + ) + print(f"- host in scutil exceptions: {'yes' if report['host_in_scutil_exceptions'] else 'no'}") + print("") + + conn = report["connectivity"] + print("Connectivity Checks") + for key in ("ambient", "direct", "forced_proxy", "strict_tls"): + value = conn.get(key) + if not value: + continue + ok = "PASS" if value.get("ok") else "FAIL" + print( + f"- {key:12s}: {ok} " + f"(http={value.get('http_code', '000')}, exit={value.get('exit_code', 'n/a')})" + ) + stderr = value.get("stderr") + if stderr: + print(f" stderr: {stderr}") + print("") + + route = report.get("tailscale_route") + if route: + if route.get("ok"): + print("Tailscale Route Check") + print(f"- route interface: {route.get('interface') or 'N/A'}") + route_mtu = route.get("route_iface_mtu") + if route_mtu: + print(f" route iface MTU: {route_mtu}") + print(f"- gateway: {route.get('gateway') or 'N/A'}") + ts_utun = route.get("tailscale_utun") + if ts_utun: + print(f"- tailscale utun: {ts_utun}") + is_ts = route.get("is_tailscale_iface") + if is_ts is True: + print(" route → Tailscale utun: YES (correct)") + elif is_ts is False: + print(" route → Tailscale utun: NO (MISMATCH — see findings)") + else: + print("- tailscale utun: (not detected — is Tailscale running?)") + print("") + else: + print("Tailscale Route Check") + print(f"- failed: {route.get('raw', '')}") + print("") + + findings = report["findings"] + if not findings: + print("Result: no high-confidence conflict found.") + print("If browser still fails, verify proxy app profile mode/rule order and reload profile.") + return 0 + + print("Findings") + severity_order = {"error": 0, "warn": 1, "info": 2} + findings_sorted = sorted(findings, key=lambda x: severity_order.get(str(x.get("level")), 99)) + for idx, item in enumerate(findings_sorted, start=1): + print(f"{idx}. [{item['level'].upper()}] {item['title']}") + print(f" Detail: {item['detail']}") + print(f" Fix: {item['fix']}") + + return 1 if any(x["level"] == "error" for x in findings) else 0 + + +def main() -> int: + if sys.platform != "darwin": + print("This script is designed for macOS only.", file=sys.stderr) + return 2 + + parser = argparse.ArgumentParser( + description="Quick diagnostics for Tailscale + proxy conflicts on macOS." + ) + parser.add_argument("--host", default="local.claude4.dev", help="Target host to diagnose.") + parser.add_argument( + "--url", + default="", + help="Full URL to test. Default: https:///health", + ) + parser.add_argument( + "--timeout", + type=int, + default=8, + help="curl timeout seconds (default: 8).", + ) + parser.add_argument( + "--tailscale-ip", + default="", + help="Optional Tailscale IP for route ownership check (e.g. 100.101.102.103).", + ) + parser.add_argument( + "--json", + action="store_true", + help="Output JSON report.", + ) + args = parser.parse_args() + + url = args.url.strip() or f"https://{args.host}/health" + report = build_report( + host=args.host.strip(), + url=url, + timeout=max(args.timeout, 1), + tailscale_ip=args.tailscale_ip.strip() or None, + ) + + if args.json: + print(json.dumps(report, indent=2, ensure_ascii=False)) + return 0 + return print_human(report) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/twitter-reader/.security-scan-passed b/twitter-reader/.security-scan-passed index 9ed173a8..540e9d2e 100644 --- a/twitter-reader/.security-scan-passed +++ b/twitter-reader/.security-scan-passed @@ -1,4 +1,4 @@ Security scan passed -Scanned at: 2026-01-11T15:27:31.794239 +Scanned at: 2026-04-06T16:24:46.159857 Tool: gitleaks + pattern-based validation -Content hash: 2349813d82b8932f89fdc653c0dde1b2335483478227653554553b59007975c1 +Content hash: bfcc70ffa4bfda5e6308942605ad8dc3cf62e8aeade5b00c6d9b774bebb190fb diff --git a/twitter-reader/SKILL.md b/twitter-reader/SKILL.md index 7dd9f463..03d3000a 100644 --- a/twitter-reader/SKILL.md +++ b/twitter-reader/SKILL.md @@ -1,72 +1,156 @@ --- name: twitter-reader -description: Fetch Twitter/X post content by URL using jina.ai API to bypass JavaScript restrictions. Use when Claude needs to retrieve tweet content including author, timestamp, post text, images, and thread replies. Supports individual posts or batch fetching from x.com or twitter.com URLs. +description: Fetch Twitter/X post content including long-form Articles with full images and metadata. Use when Claude needs to retrieve tweet/article content, author info, engagement metrics, and embedded media. Supports individual posts and X Articles (long-form content). Automatically downloads all images to local attachments folder and generates complete Markdown with proper image references. Preferred over Jina for X Articles with images. --- # Twitter Reader -Fetch Twitter/X post content without needing JavaScript or authentication. +Fetch Twitter/X post and article content with full media support. -## Prerequisites +## Quick Start (Recommended) -You need a Jina API key to use this skill: +For X Articles with images, use the new fetch_article.py script: -1. Visit https://jina.ai/ to sign up (free tier available) -2. Get your API key from the dashboard -3. Set the environment variable: +```bash +uv run --with pyyaml python scripts/fetch_article.py [output_dir] +``` +Example: ```bash -export JINA_API_KEY="your_api_key_here" +uv run --with pyyaml python scripts/fetch_article.py \ + https://x.com/HiTw93/status/2040047268221608281 \ + ./Clippings +``` + +This will: +- Fetch structured data via `twitter-cli` (likes, retweets, bookmarks) +- Fetch content with images via `jina.ai` API +- Download all images to `attachments/YYYY-MM-DD-AUTHOR-TITLE/` +- Generate complete Markdown with embedded image references +- Include YAML frontmatter with metadata + +### Example Output + +``` +Fetching: https://x.com/HiTw93/status/2040047268221608281 +-------------------------------------------------- +Getting metadata... +Title: 你不知道的大模型训练：原理、路径与新实践 +Author: Tw93 +Likes: 1648 + +Getting content and images... +Images: 15 + +Downloading 15 images... + ✓ 01-image.jpg + ✓ 02-image.jpg + ... + +✓ Saved: ./Clippings/2026-04-03-文章标题.md +✓ Images: ./Clippings/attachments/2026-04-03-HiTw93-.../ (15 downloaded) ``` -## Quick Start +## Alternative: Jina API (Text-only) -For a single tweet, use curl directly: +For simple text-only fetching without authentication: ```bash +# Single tweet curl "https://r.jina.ai/https://x.com/USER/status/TWEET_ID" \ -H "Authorization: Bearer ${JINA_API_KEY}" + +# Batch fetching +scripts/fetch_tweets.sh url1 url2 url3 ``` -For multiple tweets, use the bundled script: +## Features + +### Full Article Mode (fetch_article.py) +- ✅ Structured metadata (author, date, engagement metrics) +- ✅ Automatic image download (all embedded media) +- ✅ Complete Markdown with local image references +- ✅ YAML frontmatter for PKM systems +- ✅ Handles X Articles (long-form content) + +### Simple Mode (Jina API) +- Text-only content +- No authentication required beyond Jina API key +- Good for quick text extraction + +## Prerequisites +### For Full Article Mode +- `uv` (Python package manager) +- No additional setup (twitter-cli auto-installed) + +### For Simple Mode (Jina) ```bash -scripts/fetch_tweets.sh url1 url2 url3 +export JINA_API_KEY="your_api_key_here" +# Get from https://jina.ai/ +``` + +## Output Structure + +``` +output_dir/ +├── YYYY-MM-DD-article-title.md # Main Markdown file +└── attachments/ + └── YYYY-MM-DD-author-title/ + ├── 01-image.jpg + ├── 02-image.jpg + └── ... ``` ## What Gets Returned +### Full Article Mode +- **YAML Frontmatter**: source, author, date, likes, retweets, bookmarks +- **Markdown Content**: Full article text with local image references +- **Attachments**: All downloaded images in dedicated folder + +### Simple Mode - **Title**: Post author and content preview - **URL Source**: Original tweet link - **Published Time**: GMT timestamp -- **Markdown Content**: Full post text with media descriptions +- **Markdown Content**: Text with remote media URLs -## Bundled Scripts +## URL Formats Supported -### fetch_tweet.py +- `https://x.com/USER/status/ID` (posts) +- `https://x.com/USER/article/ID` (long-form articles) +- `https://twitter.com/USER/status/ID` (legacy) -Python script for fetching individual tweets. +## Scripts +### fetch_article.py +Full-featured article fetcher with image download: ```bash -python scripts/fetch_tweet.py https://x.com/user/status/123 output.md +uv run --with pyyaml python scripts/fetch_article.py [output_dir] ``` -### fetch_tweets.sh - -Bash script for batch fetching multiple tweets. - +### fetch_tweet.py +Simple text-only fetcher using Jina API: ```bash -scripts/fetch_tweets.sh \ - "https://x.com/user/status/123" \ - "https://x.com/user/status/456" +python scripts/fetch_tweet.py [output_file] ``` -## URL Formats Supported +### fetch_tweets.sh +Batch fetch multiple tweets (Jina API): +```bash +scripts/fetch_tweets.sh ... +``` -- `https://x.com/USER/status/ID` -- `https://twitter.com/USER/status/ID` -- `https://x.com/...` (redirects work automatically) +## Migration from Jina API -## Environment Variables +Old workflow: +```bash +curl "https://r.jina.ai/https://x.com/..." +# Manual image extraction and download +``` -- `JINA_API_KEY`: Required. Your Jina.ai API key for accessing the reader API +New workflow: +```bash +uv run --with pyyaml python scripts/fetch_article.py +# Automatic image download, complete Markdown +``` diff --git a/twitter-reader/scripts/fetch_article.py b/twitter-reader/scripts/fetch_article.py new file mode 100644 index 00000000..bf99e32a --- /dev/null +++ b/twitter-reader/scripts/fetch_article.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python3 +""" +Fetch Twitter/X Article with images using twitter-cli. + +Usage: + python fetch_article.py [output_dir] + +Example: + python fetch_article.py https://x.com/HiTw93/status/2040047268221608281 ./Clippings + +Features: + - Fetches structured data via twitter-cli + - Downloads all images to attachments folder + - Generates Markdown with embedded image references +""" + +import sys +import os +import re +import subprocess +import argparse +from pathlib import Path +from datetime import datetime + + +def run_twitter_cli(url: str) -> dict: + """Fetch article data using twitter-cli via uv run.""" + cmd = ["uv", "run", "--with", "twitter-cli", "twitter", "article", url] + + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Error fetching article: {result.stderr}", file=sys.stderr) + sys.exit(1) + + return parse_yaml_output(result.stdout) + + +def run_jina_api(url: str) -> str: + """Fetch article text with images using Jina API.""" + api_key = os.getenv("JINA_API_KEY", "") + jina_url = f"https://r.jina.ai/{url}" + + cmd = ["curl", "-s", jina_url] + if api_key: + cmd.extend(["-H", f"Authorization: Bearer {api_key}"]) + + result = subprocess.run(cmd, capture_output=True, text=True) + + if result.returncode != 0: + print(f"Warning: Jina API failed: {result.stderr}", file=sys.stderr) + return "" + + return result.stdout + + +def parse_yaml_output(output: str) -> dict: + """Parse twitter-cli YAML output into dict.""" + try: + import yaml + data = yaml.safe_load(output) + if data.get("ok") and "data" in data: + return data["data"] + return data + except ImportError: + print("Error: PyYAML required. Install with: uv pip install pyyaml", file=sys.stderr) + sys.exit(1) + except Exception as e: + print(f"Error parsing YAML: {e}", file=sys.stderr) + sys.exit(1) + + +def extract_image_urls(text: str) -> list: + """Extract image URLs from markdown text.""" + # Extract all pbs.twimg.com URLs (note: twimg not twitter) + pattern = r'https://pbs\.twimg\.com/media/[^\s\)"\']+' + matches = re.findall(pattern, text) + + # Deduplicate and normalize to large size + seen = set() + urls = [] + for url in matches: + base_url = url.split('?')[0] + if base_url not in seen: + seen.add(base_url) + urls.append(f"{base_url}?format=jpg&name=large") + + return urls + + +def download_images(image_urls: list, attachments_dir: Path) -> list: + """Download images and return list of local paths.""" + attachments_dir.mkdir(parents=True, exist_ok=True) + local_paths = [] + + for i, url in enumerate(image_urls, 1): + filename = f"{i:02d}-image.jpg" + filepath = attachments_dir / filename + + cmd = ["curl", "-sL", url, "-o", str(filepath)] + result = subprocess.run(cmd, capture_output=True) + + if result.returncode == 0 and filepath.exists() and filepath.stat().st_size > 0: + local_paths.append(f"attachments/{attachments_dir.name}/{filename}") + print(f" ✓ {filename}") + else: + print(f" ✗ Failed: {filename}") + + return local_paths + + +def replace_image_urls(text: str, image_urls: list, local_paths: list) -> str: + """Replace remote image URLs with local paths in markdown text.""" + for remote_url, local_path in zip(image_urls, local_paths): + # Extract base URL pattern + base_url = remote_url.split('?')[0].replace('?format=jpg&name=large', '') + # Replace all variations of this URL + pattern = re.escape(base_url) + r'(\?[^\)]*)?' + text = re.sub(pattern, local_path, text) + return text + + +def sanitize_filename(name: str) -> str: + """Sanitize string for use in filename.""" + # Remove special chars, keep alphanumeric, CJK, and some safe chars + name = re.sub(r'[^\w\s\-\u4e00-\u9fff]', '', name) + name = re.sub(r'\s+', '-', name.strip()) + return name[:60] # Limit length + + +def generate_markdown(data: dict, text: str, image_urls: list, local_paths: list, source_url: str) -> str: + """Generate complete Markdown document.""" + # Parse date + created = data.get("createdAtLocal", "") + if created: + date_str = created[:10] + else: + date_str = datetime.now().strftime("%Y-%m-%d") + + author = data.get("author", {}) + metrics = data.get("metrics", {}) + title = data.get("articleTitle", "Untitled") + + # Build frontmatter + md = f"""--- +source: {source_url} +author: {author.get("name", "")} +date: {date_str} +likes: {metrics.get("likes", 0)} +retweets: {metrics.get("retweets", 0)} +bookmarks: {metrics.get("bookmarks", 0)} +--- + +# {title} + +""" + + # Replace image URLs with local paths + if image_urls and local_paths: + text = replace_image_urls(text, image_urls, local_paths) + + md += text + return md + + +def main(): + parser = argparse.ArgumentParser(description="Fetch Twitter/X Article with images") + parser.add_argument("url", help="Twitter/X article URL") + parser.add_argument("output_dir", nargs="?", default=".", help="Output directory (default: current)") + args = parser.parse_args() + + if not args.url.startswith(("https://x.com/", "https://twitter.com/")): + print("Error: URL must be from x.com or twitter.com", file=sys.stderr) + sys.exit(1) + + print(f"Fetching: {args.url}") + print("-" * 50) + + # Fetch metadata from twitter-cli + print("Getting metadata...") + data = run_twitter_cli(args.url) + + title = data.get("articleTitle", "") + if not title: + print("Error: Could not fetch article data", file=sys.stderr) + sys.exit(1) + + author = data.get("author", {}) + + print(f"Title: {title}") + print(f"Author: {author.get('name', 'Unknown')}") + print(f"Likes: {data.get('metrics', {}).get('likes', 0)}") + + # Fetch content with images from Jina API + print("\nGetting content and images...") + jina_content = run_jina_api(args.url) + + # Use Jina content if available, otherwise fall back to twitter-cli text + if jina_content: + text = jina_content + # Remove Jina header lines to get clean markdown + # Find "Markdown Content:" and keep everything after it + marker = "Markdown Content:" + idx = text.find(marker) + if idx != -1: + text = text[idx + len(marker):].lstrip() + else: + text = data.get("articleText", "") + + # Extract image URLs + image_urls = extract_image_urls(text) + print(f"Images: {len(image_urls)}") + + # Setup output paths + output_dir = Path(args.output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + # Create attachments folder + date_str = data.get("createdAtLocal", "")[:10] if data.get("createdAtLocal") else datetime.now().strftime("%Y-%m-%d") + safe_author = sanitize_filename(author.get("screenName", "unknown")) + safe_title = sanitize_filename(title) + attachments_name = f"{date_str}-{safe_author}-{safe_title[:30]}" + attachments_dir = output_dir / "attachments" / attachments_name + + # Download images + local_paths = [] + if image_urls: + print(f"\nDownloading {len(image_urls)} images...") + local_paths = download_images(image_urls, attachments_dir) + + # Generate Markdown + md_content = generate_markdown(data, text, image_urls, local_paths, args.url) + + # Save Markdown + md_filename = f"{date_str}-{safe_title}.md" + md_path = output_dir / md_filename + md_path.write_text(md_content, encoding="utf-8") + + print(f"\n✓ Saved: {md_path}") + if local_paths: + print(f"✓ Images: {attachments_dir} ({len(local_paths)} downloaded)") + + return md_path + + +if __name__ == "__main__": + main() diff --git a/youtube-downloader/SKILL.md b/youtube-downloader/SKILL.md index b0438cc3..dabbb639 100644 --- a/youtube-downloader/SKILL.md +++ b/youtube-downloader/SKILL.md @@ -492,3 +492,17 @@ ffmpeg -i video.mp4 -i audio.m4a -c copy merged.mp4 - **PO Token Setup**: See `references/po-token-setup.md` for detailed installation and troubleshooting - **yt-dlp Documentation**: https://github.com/yt-dlp/yt-dlp - **Format Selection Guide**: https://github.com/yt-dlp/yt-dlp#format-selection + +## Next Step: Transcribe Downloaded Audio/Video + +After downloading, if the user's goal involves getting text from the video (transcription, subtitles, meeting notes), proactively suggest: + +``` +Download complete: [filename] + +If you need the spoken content as text, I can transcribe it for you. + +Options: +A) Transcribe with /asr-transcribe-to-text (Recommended for speech-to-text) +B) No thanks — I just needed the video file +``` diff --git a/youtube-downloader/references/internal-sop.md b/youtube-downloader/references/internal-sop.md new file mode 100644 index 00000000..5fdd1be3 --- /dev/null +++ b/youtube-downloader/references/internal-sop.md @@ -0,0 +1,12 @@ +# YouTube Downloader Internal SOP + +Use this SOP to avoid common yt-dlp failures and confusion: + +1. Quote YouTube URLs in shell commands (zsh treats `?` as glob). Example: `'https://www.youtube.com/watch?v=VIDEO_ID'`. +2. Ensure proxy is active for both yt-dlp and PO Token providers (HTTP_PROXY/HTTPS_PROXY/ALL_PROXY). +3. If you see "Sign in to confirm you're not a bot", request cookie permission and use browser cookies. +4. Start the PO Token provider before downloading. Prefer Docker bgutil; fall back to browser-based WPC when Docker is unavailable or fails. +5. Use `web_safari` client when cookies are present; otherwise use `mweb` for PO tokens. +6. Keep the browser window open while WPC is minting tokens and make sure it can reach YouTube through the same proxy. +7. If you see "Only images are available" or "Requested format is not available", treat it as PO token failure and retry after fixing provider/browser state. +8. If you see SSL EOF or fragment errors, treat it as proxy instability. Retry with progressive formats or switch to a more stable proxy.