From 5d8c19a22490fdb6e15152b19d44e5f782f202fa Mon Sep 17 00:00:00 2001 From: ajaycodesitbetter Date: Fri, 5 Jun 2026 22:54:05 +0530 Subject: [PATCH 1/6] feat(skill): add geo-gap-fixer GTM Intelligence skill (LLM GEO gap audit) --- README.md | 5 + packages/cli/registry.json | 13 +- skills/geo-gap-fixer/.env.example | 14 + skills/geo-gap-fixer/.gitignore | 18 + skills/geo-gap-fixer/README.md | 143 +++++ skills/geo-gap-fixer/SKILL.md | 91 ++++ skills/geo-gap-fixer/config.example.json | 8 + skills/geo-gap-fixer/package.json | 16 + .../geo-gap-fixer/references/output_format.md | 116 +++++ .../references/prompt_templates.md | 102 ++++ .../references/scoring_rubric.md | 116 +++++ .../geo-gap-fixer/scripts/analyze_results.py | 418 +++++++++++++++ skills/geo-gap-fixer/scripts/build_report.py | 445 ++++++++++++++++ skills/geo-gap-fixer/scripts/probe_llms.py | 492 ++++++++++++++++++ 14 files changed, 1996 insertions(+), 1 deletion(-) create mode 100644 skills/geo-gap-fixer/.env.example create mode 100644 skills/geo-gap-fixer/.gitignore create mode 100644 skills/geo-gap-fixer/README.md create mode 100644 skills/geo-gap-fixer/SKILL.md create mode 100644 skills/geo-gap-fixer/config.example.json create mode 100644 skills/geo-gap-fixer/package.json create mode 100644 skills/geo-gap-fixer/references/output_format.md create mode 100644 skills/geo-gap-fixer/references/prompt_templates.md create mode 100644 skills/geo-gap-fixer/references/scoring_rubric.md create mode 100644 skills/geo-gap-fixer/scripts/analyze_results.py create mode 100644 skills/geo-gap-fixer/scripts/build_report.py create mode 100644 skills/geo-gap-fixer/scripts/probe_llms.py diff --git a/README.md b/README.md index e43ad9b..3eff1ba 100644 --- a/README.md +++ b/README.md @@ -401,6 +401,11 @@ Manus AI users can import a skill directly from its OpenDirectory skill page. Th Give it your product URL. It finds your top 5 competitors, researches every press mention, podcast appearance, and community post across all of them, and tells you exactly which channels to pitch -- with the journalist's name, the angle that got your competitors featured, and a ready-to-send cold pitch for your product. 0.0.1 + + geo-gap-fixer + Audit how often LLMs recommend your brand vs. competitors — then get a concrete action plan to fix the gaps. + 1.0.0 + gh-issue-to-demand-signal Give the skill a competitor's public GitHub repo URL. It fetches their open issues, filters noise locally, clusters into 6 demand categories using the AI already running the skill, scores by real engagement (reactions), detects ignored demand (high reactions + no response = your opportunity), and outputs a ranked demand gap report with a GTM messaging brief. diff --git a/packages/cli/registry.json b/packages/cli/registry.json index 5e226bd..702f13d 100644 --- a/packages/cli/registry.json +++ b/packages/cli/registry.json @@ -135,6 +135,17 @@ "version": "1.0.0", "path": "skills/explain-this-pr" }, + { + "name": "geo-gap-fixer", + "description": "Audit how often LLMs recommend your brand vs competitors and generate a GEO action plan.", + "tags": [ + "Branding", + "AI" + ], + "author": "ajaycodesitbetter", + "version": "1.0.0", + "path": "skills/geo-gap-fixer" + }, { "name": "gh-issue-to-demand-signal", "description": "Takes a competitor's public GitHub repo URL, fetches their open issues via the GitHub REST API, filters noise locally, clusters issues into 6 deman...", @@ -577,4 +588,4 @@ "version": "0.0.1", "path": "skills/yc-intent-radar-skill" } -] \ No newline at end of file +] diff --git a/skills/geo-gap-fixer/.env.example b/skills/geo-gap-fixer/.env.example new file mode 100644 index 0000000..911fb09 --- /dev/null +++ b/skills/geo-gap-fixer/.env.example @@ -0,0 +1,14 @@ +# At least 2 of 4 API keys required. +# Missing keys are skipped gracefully — the script never crashes. + +# OpenAI (gpt-4o) +OPENAI_API_KEY=sk-... + +# Anthropic (claude-sonnet-4-6) +ANTHROPIC_API_KEY=sk-ant-... + +# Google Gemini (gemini-2.5-flash) +GOOGLE_API_KEY=AI... + +# Perplexity (sonar-pro) +PERPLEXITY_API_KEY=pplx-... diff --git a/skills/geo-gap-fixer/.gitignore b/skills/geo-gap-fixer/.gitignore new file mode 100644 index 0000000..0b67f32 --- /dev/null +++ b/skills/geo-gap-fixer/.gitignore @@ -0,0 +1,18 @@ +# Runtime outputs — regenerated by the scripts +data/ +report/ + +# User config (contains brand-specific info) +config.json + +# Environment variables +.env + +# Python +__pycache__/ +*.pyc +*.pyo + +# OS +.DS_Store +Thumbs.db diff --git a/skills/geo-gap-fixer/README.md b/skills/geo-gap-fixer/README.md new file mode 100644 index 0000000..65b4fca --- /dev/null +++ b/skills/geo-gap-fixer/README.md @@ -0,0 +1,143 @@ +# geo-gap-fixer + +**Audit how often LLMs recommend your brand vs. competitors — then get a concrete action plan to fix the gaps.** + +## Why this skill exists +In 2026, buyers discover tools by asking ChatGPT, Claude, Gemini, and Perplexity *"what's the best X for Y?"* — not by Googling. If LLMs consistently recommend a competitor instead of you, your traditional SEO rank is irrelevant. + +**geo-gap-fixer** is a free, open-source agent skill for GTM Intelligence that audits your Generative Engine Optimization (GEO) share-of-voice. It probes the LLMs, analyzes who gets recommended, and outputs a prioritized content backlog to fix the gaps. + +> One skill run replaces a $200/month AI-monitoring SaaS subscription. + +--- + +## Quickstart + +**1. Clone and Configure** +```bash +cp .env.example .env +cp config.example.json config.json +``` +Edit `config.json` with your brand, competitors, and category. Edit `.env` with at least 2 API keys (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`, `PERPLEXITY_API_KEY`). + +**2. Run the Pipeline** +```bash +python scripts/probe_llms.py # Probes LLM APIs +python scripts/analyze_results.py # Analyzes mentions & sentiment +python scripts/build_report.py # Generates the GEO report +``` + +**3. Read the Report** +Open `report/geo_audit_report.md` for your action plan. + +--- + +## Sample Report Output + +```markdown +### 1. Share of Voice (LLM Mention Rate) +| Brand | Mention Rate | Avg Rank | Win Rate | Assessment | +|-------|-------------|----------|----------|------------| +| **Linear** | **18.4%** | **4.2** | **5.3%** | 🔴 Critical: Brand is rarely recommended | +| Jira | 94.7% | 1.4 | 82.1% | 🟢 Dominant: Default recommendation | + +### 5. GEO Action Plan +🔴 **Critical Priority: Fix Mention Rate (<30%)** +- Create dedicated "Best Tools in 2026" comparison content. +- Ensure your homepage explicitly answers: "Why use Linear for ?" +``` + +--- + +## What You Get + +| # | Output Section | Description | +|---|---------------|-------------| +| 1 | **Share-of-Voice Table** | Brand mention rate per LLM (% of prompts where you're mentioned), ranked | +| 2 | **Prompt-Level Loss Log** | Which exact prompts your brand lost and to whom | +| 3 | **Competitor Language Patterns** | What framing/keywords LLMs use for competitors that they don't use for you | +| 4 | **Citation Gap List** | Which domains get cited by LLMs instead of your site | +| 5 | **GEO Action Plan** | Prioritized fixes: FAQ pages, comparison pages, alternative pages, schema improvements, authority targets | + +--- + +## Prerequisites + +- **Python 3.10+** +- **At least 2 of 4 API keys** (missing providers are skipped gracefully) +- Dependencies: `pip install openai anthropic google-genai` + +| Provider | Package | Model | Env Variable | +|----------|---------|-------|-------------| +| OpenAI | `openai` | gpt-4o | `OPENAI_API_KEY` | +| Anthropic | `anthropic` | claude-sonnet-4-6 | `ANTHROPIC_API_KEY` | +| Google | `google-genai` | gemini-2.5-flash | `GOOGLE_API_KEY` | +| Perplexity | `openai` (reused) | sonar-pro | `PERPLEXITY_API_KEY` | + +> **Note**: Perplexity uses the OpenAI SDK with a different base URL, so only 3 packages are needed. + +--- + +## Configuration (`config.json`) + +| Field | Type | Required | Description | +|-------|------|----------|-------------| +| `brand_name` | string | ✅ | Your brand name (e.g., "Linear") | +| `competitors` | list | ✅ | 1–5 competitor brand names | +| `category` | string | ✅ | Product category (e.g., "project management tool for developers") | +| `buyer_intent_prompts` | list | ❌ | Custom prompts. Empty = auto-generate from templates | +| `target_llms` | list | ❌ | Which LLMs to probe. Default: all 4 | +| `website_url` | string | ❌ | Your website for citation gap analysis | + +--- + +## Limitations (What this skill does NOT do) + +- **Not a continuous monitor**: This is a point-in-time audit. Run it monthly to track progress. +- **Not for traditional SEO**: It does not track Google Blue Link rankings, domain authority, or keyword search volume. +- **Not deep NLP sentiment analysis**: Sentiment uses keyword proximity, not complex NLP models. +- **Not free to run**: While the tool is free, you pay the LLM API providers directly (typically ~$0.50–$2.00 per audit). + +--- + +## Error Handling + +All scripts are designed to fail clearly, not silently: + +| Scenario | Behavior | +|----------|----------| +| Invalid JSON config | Exits with parse error and line number | +| Missing required fields | Exits listing exactly which fields are missing | +| No API keys set | Exits listing all 4 env variable names | +| Transient API failure | Retries with exponential backoff (2 attempts, 2s → 4s) | +| Persistent API failure | Skips that provider, continues with others | +| Zero responses collected | Exits non-zero after saving empty results | +| Missing upstream data file | Exits with instructions to run the previous script | + +--- + +## File Structure + +``` +geo-gap-fixer/ +├── README.md ← You are here +├── SKILL.md ← Agent instruction flow +├── .env.example ← API key template +├── .gitignore ← Excludes data/, report/, config.json +├── package.json ← OpenDirectory metadata +├── config.example.json ← Sample configuration +├── scripts/ +│ ├── probe_llms.py ← Send prompts to LLM APIs +│ ├── analyze_results.py ← Extract mentions, rank, sentiment, citations +│ └── build_report.py ← Assemble the 5-section report +├── references/ +│ ├── prompt_templates.md ← 20 default buyer-intent prompts +│ ├── scoring_rubric.md ← How metrics are calculated +│ └── output_format.md ← Sample report with realistic data +├── data/ ← Generated at runtime (gitignored) +│ ├── raw_responses.json +│ └── analysis.json +└── report/ ← Generated at runtime (gitignored) + ├── geo_audit_report.md + └── geo_audit_report.json +``` diff --git a/skills/geo-gap-fixer/SKILL.md b/skills/geo-gap-fixer/SKILL.md new file mode 100644 index 0000000..058dad1 --- /dev/null +++ b/skills/geo-gap-fixer/SKILL.md @@ -0,0 +1,91 @@ +--- +name: geo-gap-fixer +description: "Audit how often LLMs recommend your brand vs competitors and generate a GEO action plan." +category: "GTM Intelligence" +version: "1.0.0" +--- + +# GEO Gap Fixer + +> Agent skill that audits LLM brand visibility and converts gaps into a +> concrete GEO content action plan. + +--- + +## When to Use + +Use this skill when a user wants to audit their Generative Engine Optimization (GEO) share-of-voice to know which LLM prompts their brand is losing, understand why competitors are recommended instead, and get a specific content fix plan. + +**Do NOT use this skill for**: general SEO audits, paid ad optimization, or continuous social media monitoring. This is a point-in-time LLM visibility audit. + +--- + +## Step 1: Inputs + +To run the audit, the user must provide API keys and a configuration file. Ensure the following are set up: + +1. **API Keys**: At least 2 of 4 keys must be set in the environment or `.env` file (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`, `PERPLEXITY_API_KEY`). +2. **Dependencies**: `pip install openai anthropic google-genai` +3. **Config File**: `config.json` (copied from `config.example.json`) must contain: + - `brand_name` (string, required) + - `competitors` (list of strings, required, 1-10 entries) + - `category` (string, required) + - `buyer_intent_prompts` (list of strings, optional. If empty, 20 prompts are auto-generated) + - `target_llms` (list of strings, optional) + - `website_url` (string, optional) + +--- + +## Step 2: Execution Pipeline + +Run the following scripts in order. Stop and ask for clarification if any script fails. + +1. **`python scripts/probe_llms.py`** (Optional: append `--dry-run` to test config without API calls) + - Sends buyer-intent prompts to the configured LLM APIs. + - Saves responses to `data/raw_responses.json`. + +2. **`python scripts/analyze_results.py`** + - Analyzes raw responses for brand mentions, ranking, sentiment, and cited domains. + - Saves structured analysis to `data/analysis.json`. + +3. **`python scripts/build_report.py`** + - Assembles the final 5-section GEO audit report. + - Saves to `report/geo_audit_report.md` and `report/geo_audit_report.json`. + +--- + +## Step 3: Outputs & Interpretation + +The primary output is `report/geo_audit_report.md`. Present its findings to the user. + +**Key Sections to Interpret:** +1. **Share-of-Voice Table**: A mention rate below 30% is critical. Mention rate is the % of prompts where the brand is recommended. +2. **Prompt-Level Loss Log**: Which exact prompts the brand lost and to whom. +3. **Competitor Language Patterns**: The specific adjectives LLMs use for competitors. +4. **Citation Gap List**: Domains LLMs cite that the brand is missing from. +5. **GEO Action Plan**: Prioritized fixes (🔴 Critical, 🟡 High Priority, 🟢 Growth Plays). + +Direct the user to the **GEO Action Plan** first, as it contains the concrete steps to fix the gaps identified in the audit. + +--- + +## Step 4: Error Handling + +If you encounter issues while executing the pipeline, follow these rules: + +| Condition | Agent Action | +|-----------|--------------| +| Missing `config.json` | Tell the user to copy `config.example.json` and fill it out. | +| Invalid JSON in config | Notify the user of the parse error location and ask them to fix it. | +| Missing required fields | List the exact missing fields (`brand_name`, `competitors`, `category`). | +| No API keys set | Ask the user to export at least 2 of the 4 supported API keys. | +| 1 API key only | Warn the user that results are less reliable, but proceed with the run. | +| Transient API failure | The script auto-retries. If it fails completely, it skips the provider. | +| Persistent API failure | The script skips the provider gracefully. Continue the pipeline. | +| Zero responses | The script exits non-zero. Notify the user to check API keys or config. | +| Missing upstream data file | Re-run the preceding script in the pipeline (e.g., probe before analyze). | + +**Limitations to keep in mind**: +- This is a point-in-time audit, not a background monitor. +- Sentiment analysis uses keyword proximity, not deep NLP. +- API costs apply for each run (typically ~$0.50–$2.00). diff --git a/skills/geo-gap-fixer/config.example.json b/skills/geo-gap-fixer/config.example.json new file mode 100644 index 0000000..6a478d2 --- /dev/null +++ b/skills/geo-gap-fixer/config.example.json @@ -0,0 +1,8 @@ +{ + "brand_name": "Linear", + "competitors": ["Jira", "Asana", "Height", "Monday"], + "category": "project management tool for developers", + "buyer_intent_prompts": [], + "target_llms": ["openai", "anthropic", "google", "perplexity"], + "website_url": "https://linear.app" +} diff --git a/skills/geo-gap-fixer/package.json b/skills/geo-gap-fixer/package.json new file mode 100644 index 0000000..056c06b --- /dev/null +++ b/skills/geo-gap-fixer/package.json @@ -0,0 +1,16 @@ +{ + "name": "geo-gap-fixer", + "version": "1.0.0", + "description": "Audit how often LLMs recommend your brand vs competitors and generate a GEO action plan", + "category": "GTM Intelligence", + "tags": [ + "geo", + "seo", + "llm", + "brand-visibility", + "competitive-analysis", + "content-strategy" + ], + "author": "ajaycodesitbetter", + "license": "MIT" +} diff --git a/skills/geo-gap-fixer/references/output_format.md b/skills/geo-gap-fixer/references/output_format.md new file mode 100644 index 0000000..d173375 --- /dev/null +++ b/skills/geo-gap-fixer/references/output_format.md @@ -0,0 +1,116 @@ +# Output Format — GEO Gap Fixer + +> Sample report showing all 5 output sections with realistic data. +> This is what `build_report.py` generates in `report/geo_audit_report.md`. + +--- + +# GEO Gap Audit Report — Acme PM + +> **Generated**: 2026-06-05T12:00:00Z +> **Category**: project management tool for developers +> **Providers**: openai, anthropic, google +> **Total responses analyzed**: 150 + +--- + +## 1. Share-of-Voice Table + +How often each brand is mentioned and recommended by LLMs. + +| Brand | Mention Rate | Avg Rank | Win Rate | Mentioned | Wins | +|-------|-------------|----------|----------|-----------|------| +| Jira | 87% | 1.3 | 52% | 130 | 78 | +| Asana | 73% | 1.8 | 23% | 110 | 35 | +| Monday | 45% | 2.4 | 12% | 68 | 18 | +| Acme PM **← YOU** | 28% | 2.7 | 5% | 42 | 8 | +| Height | 15% | 2.9 | 3% | 22 | 4 | + +**Overall Health**: 🔴 **Critical** — LLMs rarely mention you; major GEO overhaul needed + +--- + +## 2. Prompt-Level Loss Log + +Prompts where your brand was NOT mentioned first (or not at all). + +| # | Prompt | Provider | Winner | Your Rank | +|---|--------|----------|--------|-----------| +| 1 | What is the best project management tool? | openai | Jira | Not mentioned | +| 2 | What is the best project management tool? | anthropic | Jira | 3 | +| 3 | What project management tool do you recommend? | openai | Asana | Not mentioned | +| 4 | Top project management tool in 2026 | google | Jira | Not mentioned | +| 5 | Acme PM vs Jira: which is better? | openai | Jira | 2 | +| 6 | Compare Acme PM and Asana for project management | anthropic | Asana | 2 | +| 7 | Best alternatives to Jira | openai | Asana | Not mentioned | +| 8 | Best alternatives to Jira | google | Monday | 3 | + +**Total losses**: 112 out of 150 analyzed responses + +--- + +## 3. Competitor Language Patterns + +How LLMs describe your competitors — keywords and framing you may be missing. + +### Jira +**Framing used by LLMs**: `enterprise-grade`, `industry standard`, `powerful`, `widely adopted`, `highly configurable`, `robust` + +### Asana +**Framing used by LLMs**: `user-friendly`, `collaborative`, `modern`, `intuitive`, `clean interface` + +### Monday +**Framing used by LLMs**: `visual`, `customizable`, `easy to use`, `colorful`, `flexible` + +**💡 Insight**: If competitors are described with keywords your brand lacks, consider incorporating similar language into your website copy, comparison pages, and product descriptions. + +--- + +## 4. Citation Gap List + +Domains cited by LLMs in their responses. + +| Domain | Times Cited | Your Domain? | +|--------|-------------|-------------| +| atlassian.com | 42 | ❌ No | +| asana.com | 31 | ❌ No | +| g2.com | 18 | ❌ No | +| monday.com | 15 | ❌ No | +| pcmag.com | 12 | ❌ No | +| capterra.com | 9 | ❌ No | +| acmepm.com | 0 | ✅ Yes | + +⚠️ **Your domain (https://acmepm.com) was never cited by any LLM.** + +--- + +## 5. GEO Action Plan + +Prioritized content and positioning fixes based on the audit findings. + +### 🔴 Critical (Do First) + +- [ ] **Create FAQ page**: "What is the best project management tool for developers?" — your brand is mentioned in only 28% of responses +- [ ] **Create comparison page**: "Acme PM vs Jira" — Jira wins 52% of prompts vs your 5% +- [ ] **Optimize landing page** for "project management tool for developers" — you lost 8 direct recommendation prompts + +### 🟡 High Priority + +- [ ] **Create alternatives page**: "Best alternatives to Asana" — Asana has 73% mention rate vs your 28% +- [ ] **Create alternatives page**: "Best alternatives to Monday" — Monday has 45% mention rate vs your 28% +- [ ] **Add schema markup**: FAQPage, SoftwareApplication, Organization — your domain is not being cited by any LLM +- [ ] **Address negative framing**: Review and update product descriptions to counter negative keywords found in LLM responses + +### 🟢 Growth Plays + +- [ ] **Build authority on cited domains**: Get listed/mentioned on atlassian.com, g2.com, pcmag.com — these are domains LLMs trust and cite +- [ ] **Mirror competitor framing**: Incorporate language like "enterprise-grade", "user-friendly", "modern", "intuitive", "robust" into your product pages +- [ ] **Publish original data/statistics**: Add "according to" citations, benchmarks, and original research that LLMs can extract and cite +- [ ] **Schedule monthly re-audits**: Run this skill monthly to track whether your GEO improvements are working + +**Total actions**: 11 items in your GEO backlog +**Focus first on**: 🔴 Critical items (3 actions) + +--- + +*Report generated by [geo-gap-fixer](https://github.com/Varnan-Tech/opendirectory) — an OpenDirectory skill for GEO auditing.* diff --git a/skills/geo-gap-fixer/references/prompt_templates.md b/skills/geo-gap-fixer/references/prompt_templates.md new file mode 100644 index 0000000..b3d74e8 --- /dev/null +++ b/skills/geo-gap-fixer/references/prompt_templates.md @@ -0,0 +1,102 @@ +# Prompt Templates — GEO Gap Fixer + +> 20 default buyer-intent prompts organized into 4 categories. +> These are used by `probe_llms.py` when `buyer_intent_prompts` is empty in config.json. + +All templates use `{brand}`, `{competitor}`, and `{category}` variables +that are interpolated from your `config.json` values. + +Templates containing `{competitor}` generate one prompt per competitor. + +--- + +## 1. Direct Recommendation (5 prompts) + +These test whether the LLM mentions your brand unprompted when asked +for the best tool in your category. + +| # | Template | +|---|----------| +| 1 | "What is the best {category}?" | +| 2 | "What {category} do you recommend?" | +| 3 | "Top {category} in 2026" | +| 4 | "Best {category} for startups" | +| 5 | "Best {category} for teams" | + +--- + +## 2. Comparison (5 prompts × N competitors) + +These test how LLMs position your brand when directly compared +with a competitor. Generates one prompt per competitor. + +| # | Template | +|---|----------| +| 6 | "{brand} vs {competitor}: which is better?" | +| 7 | "Compare {brand} and {competitor} for {category}" | +| 8 | "Should I use {brand} or {competitor}?" | +| 9 | "{brand} vs {competitor} pros and cons" | +| 10 | "Differences between {brand} and {competitor}" | + +--- + +## 3. Problem-Solution (5 prompts) + +These test whether LLMs recommend your brand when users describe +a pain point or need without naming any specific tool. + +| # | Template | +|---|----------| +| 11 | "I need a {category} that is fast and simple" | +| 12 | "Best {category} for developer teams" | +| 13 | "What {category} has the best API?" | +| 14 | "Most affordable {category} for small teams" | +| 15 | "{category} with best integrations" | + +--- + +## 4. Alternative Seeking (5 prompts × N competitors) + +These test whether your brand appears when users are looking +to switch away from a competitor. Generates one prompt per competitor. + +| # | Template | +|---|----------| +| 16 | "Best alternatives to {competitor}" | +| 17 | "Cheaper alternatives to {competitor}" | +| 18 | "What to use instead of {competitor}" | +| 19 | "Moving away from {competitor}, what should I try?" | +| 20 | "{competitor} competitors worth trying" | + +--- + +## Prompt Coverage Matrix + +| Category | Without competitors | With 4 competitors | Total | +|----------|--------------------|--------------------|-------| +| Direct Recommendation | 5 | — | 5 | +| Comparison | — | 5 × 4 = 20 | 20 | +| Problem-Solution | 5 | — | 5 | +| Alternative Seeking | — | 5 × 4 = 20 | 20 | +| **Total** | **10** | **40** | **50** | + +> With 4 competitors, you get 50 prompts total. Each prompt is sent to +> each available LLM provider, so 50 prompts × 3 providers = 150 API calls. + +--- + +## Customization + +You can override these defaults entirely by setting `buyer_intent_prompts` +in your `config.json`: + +```json +{ + "buyer_intent_prompts": [ + "What project management tool is best for a 10-person startup?", + "I'm evaluating tools for sprint planning, what do you suggest?" + ] +} +``` + +When custom prompts are provided, templates in this file are ignored. diff --git a/skills/geo-gap-fixer/references/scoring_rubric.md b/skills/geo-gap-fixer/references/scoring_rubric.md new file mode 100644 index 0000000..8363cd4 --- /dev/null +++ b/skills/geo-gap-fixer/references/scoring_rubric.md @@ -0,0 +1,116 @@ +# Scoring Rubric — GEO Gap Fixer + +> How `analyze_results.py` scores brand visibility across LLM responses. + +--- + +## Metrics Overview + +| # | Metric | Range | What It Measures | +|---|--------|-------|-----------------| +| 1 | Mention Rate | 0–100% | How often the brand appears in LLM responses | +| 2 | Average Rank | 1.0–3.0+ | Where the brand is positioned when mentioned | +| 3 | Sentiment Score | -1.0 to +1.0 | How positively/negatively the brand is framed | +| 4 | Citation Score | 0–100% | How often the brand's domain is cited vs competitors | +| 5 | Win Rate | 0–100% | How often the brand is mentioned first (ranked #1) | + +--- + +## 1. Mention Rate + +**Formula**: `(responses with brand mention / total responses) × 100` + +| Range | Interpretation | +|-------|---------------| +| 80–100% | Excellent — LLMs consistently recommend you | +| 50–79% | Good — present but not dominant | +| 20–49% | Weak — significant gaps in visibility | +| 0–19% | Critical — LLMs rarely mention you | + +**Detection method**: Case-insensitive word-boundary match (`\b{brand}\b`). +This prevents false positives like "linear" matching "linear algebra". + +--- + +## 2. Average Rank + +**Formula**: Mean position across responses where brand IS mentioned. + +| Rank | Meaning | +|------|---------| +| 1.0 | Always mentioned first — you're the top recommendation | +| 1.5–2.0 | Usually in top 2 — strong but not dominant | +| 2.0–3.0 | Mid-pack — often mentioned after competitors | +| 3.0+ | Afterthought — rarely the primary recommendation | + +**Calculation**: First occurrence position compared to all tracked names. +If brand appears before all competitors → rank 1. +If one competitor appears before brand → rank 2. And so on. + +--- + +## 3. Sentiment Score + +**Formula**: `(positive_keywords − negative_keywords) / total_keywords` + +Scored within ±200 characters of each brand mention (proximity-based). + +### Positive Keywords (weight: +1 each) +`best`, `recommend`, `excellent`, `leading`, `top`, `popular`, +`powerful`, `intuitive`, `fast`, `modern`, `innovative`, `reliable`, +`robust`, `preferred`, `standout`, `impressive`, `superior`, +`seamless`, `elegant`, `efficient` + +### Negative Keywords (weight: -1 each) +`limited`, `lacks`, `however`, `downside`, `expensive`, `complex`, +`steep learning curve`, `missing`, `outdated`, `slow`, `clunky`, +`basic`, `restrictive`, `difficult`, `confusing`, `poor`, +`frustrating`, `buggy`, `unreliable`, `overpriced` + +### Labels + +| Score | Label | +|-------|-------| +| > 0.2 | Positive | +| -0.2 to 0.2 | Neutral | +| < -0.2 | Negative | + +--- + +## 4. Citation Score + +**Formula**: `(brand domain citations / total domain citations) × 100` + +| Range | Interpretation | +|-------|---------------| +| 20%+ | Strong — your domain is a trusted source | +| 5–19% | Moderate — some presence but competitors dominate | +| 0–4% | Weak — LLMs rarely cite your domain | + +**Detection method**: URL regex extraction from response text, +domain normalization (strip `www.`, lowercase). + +--- + +## 5. Win Rate + +**Formula**: `(responses where brand is ranked #1 / total responses) × 100` + +| Range | Interpretation | +|-------|---------------| +| 50%+ | Dominant — you're the primary recommendation | +| 25–49% | Competitive — winning roughly half the time | +| 10–24% | Weak — rarely the first recommendation | +| 0–9% | Critical — almost never recommended first | + +--- + +## Overall Health Assessment + +The report uses these combined thresholds: + +| Mention Rate | Win Rate | Assessment | +|-------------|----------|------------| +| ≥60% | ≥30% | 🟢 Healthy — maintain and optimize | +| 30–59% | 10–29% | 🟡 At Risk — targeted content fixes needed | +| <30% | <10% | 🔴 Critical — major GEO overhaul needed | diff --git a/skills/geo-gap-fixer/scripts/analyze_results.py b/skills/geo-gap-fixer/scripts/analyze_results.py new file mode 100644 index 0000000..cd6f002 --- /dev/null +++ b/skills/geo-gap-fixer/scripts/analyze_results.py @@ -0,0 +1,418 @@ +#!/usr/bin/env python3 +""" +analyze_results.py — Extract brand mentions, rank, sentiment, citations, and framing. + +Reads data/raw_responses.json (output of probe_llms.py) and produces +data/analysis.json with structured metrics for each prompt-provider pair. + +Usage: + python scripts/analyze_results.py +""" + +import json +import re +import sys +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path + +# Fix Windows console encoding +try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + sys.stderr.reconfigure(encoding="utf-8", errors="replace") +except AttributeError: + pass + +# ── Constants ─────────────────────────────────────────────────────────────── + +SCRIPT_DIR = Path(__file__).resolve().parent +SKILL_ROOT = SCRIPT_DIR.parent +DATA_DIR = SKILL_ROOT / "data" +RAW_RESPONSES_PATH = DATA_DIR / "raw_responses.json" +ANALYSIS_PATH = DATA_DIR / "analysis.json" + +# Sentiment keyword lists — matched with word boundaries to avoid false positives +# (e.g., "best" inside "asbest" won't match) +POSITIVE_KEYWORDS = [ + "best", "recommend", "excellent", "leading", "top", "popular", + "powerful", "intuitive", "fast", "modern", "innovative", "reliable", + "robust", "preferred", "standout", "impressive", "superior", + "seamless", "elegant", "efficient", +] + +NEGATIVE_KEYWORDS = [ + "limited", "lacks", "however", "downside", "expensive", "complex", + "steep learning curve", "missing", "outdated", "slow", "clunky", + "basic", "restrictive", "difficult", "confusing", "poor", + "frustrating", "buggy", "unreliable", "overpriced", +] + +# Pre-compile word-boundary patterns for sentiment keywords +_POS_PATTERNS = [re.compile(r'\b' + re.escape(kw) + r'\b') for kw in POSITIVE_KEYWORDS] +_NEG_PATTERNS = [re.compile(r'\b' + re.escape(kw) + r'\b') for kw in NEGATIVE_KEYWORDS] + +# URL extraction regex +URL_PATTERN = re.compile(r'https?://(?:[a-zA-Z0-9\-._~:/?#\[\]@!$&\'()*+,;=%])+') + + +# ── Core Analysis Functions ───────────────────────────────────────────────── + + +def find_mentions(text: str, names: list[str]) -> dict[str, list[int]]: + """ + Find all mentions of each name in the text. + Returns {name: [char_positions]} using word-boundary matching. + """ + mentions = {} + text_lower = text.lower() + for name in names: + pattern = re.compile(r'\b' + re.escape(name.lower()) + r'\b') + positions = [m.start() for m in pattern.finditer(text_lower)] + if positions: + mentions[name] = positions + return mentions + + +def compute_rank(mentions: dict[str, list[int]], brand: str) -> int | None: + """ + Compute mention rank for the brand. + Rank 1 = brand appears first among all tracked names. + Returns None if brand not mentioned. + """ + if brand not in mentions: + return None + + brand_first = mentions[brand][0] + rank = 1 + for name, positions in mentions.items(): + if name != brand and positions[0] < brand_first: + rank += 1 + return rank + + +def compute_sentiment(text: str, name: str) -> dict: + """ + Compute sentiment for a specific brand/name within the text. + Uses keyword proximity scoring: look within ±200 chars of each mention. + """ + text_lower = text.lower() + pattern = re.compile(r'\b' + re.escape(name.lower()) + r'\b') + match_positions = [m.start() for m in pattern.finditer(text_lower)] + + if not match_positions: + return {"score": 0.0, "label": "not_mentioned", "positive": [], "negative": []} + + # Collect context around all mentions + context_chars = [] + for pos in match_positions: + start = max(0, pos - 200) + end = min(len(text_lower), pos + len(name) + 200) + context_chars.append(text_lower[start:end]) + + context = " ".join(context_chars) + + pos_found = [POSITIVE_KEYWORDS[i] for i, p in enumerate(_POS_PATTERNS) if p.search(context)] + neg_found = [NEGATIVE_KEYWORDS[i] for i, p in enumerate(_NEG_PATTERNS) if p.search(context)] + + total = len(pos_found) + len(neg_found) + if total == 0: + score = 0.0 + label = "neutral" + else: + score = round((len(pos_found) - len(neg_found)) / total, 2) + if score > 0.2: + label = "positive" + elif score < -0.2: + label = "negative" + else: + label = "neutral" + + return { + "score": score, + "label": label, + "positive": pos_found, + "negative": neg_found, + } + + +def extract_citations(text: str) -> list[str]: + """Extract unique cited domains from URLs in the response text.""" + urls = URL_PATTERN.findall(text) + domains = set() + for url in urls: + # Extract domain from URL + match = re.match(r'https?://([^/\s?#]+)', url) + if match: + domain = match.group(1).lower() + # Remove common prefixes + domain = re.sub(r'^www\.', '', domain) + domains.add(domain) + return sorted(domains) + + +def extract_framing(text: str, name: str) -> list[str]: + """ + Extract descriptive phrases used near a brand mention. + Captures adjectives and short phrases within ±60 chars. + """ + text_lower = text.lower() + pattern = re.compile(r'\b' + re.escape(name.lower()) + r'\b') + phrases = set() + + for match in pattern.finditer(text_lower): + start = max(0, match.start() - 60) + end = min(len(text_lower), match.end() + 60) + context = text_lower[start:end] + + # Extract adjective-like words near the brand + # Look for common descriptive patterns + desc_patterns = [ + r'(?:is|are|was)\s+((?:a\s+)?(?:very\s+)?[\w-]+(?:\s+[\w-]+)?)', + r'([\w-]+(?:\s+[\w-]+)?)\s+(?:tool|platform|solution|app|software)', + r'(?:known for|best for|great for|ideal for)\s+([\w\s-]+?)(?:\.|,|$)', + ] + for dp in desc_patterns: + for dm in re.finditer(dp, context): + phrase = dm.group(1).strip() + if len(phrase) > 2 and phrase != name.lower(): + phrases.add(phrase) + + return sorted(phrases)[:10] # Cap at 10 phrases + + +def determine_winner(mentions: dict[str, list[int]]) -> str | None: + """Determine which brand/competitor was mentioned first (i.e., the 'winner').""" + if not mentions: + return None + + first_positions = {name: positions[0] for name, positions in mentions.items()} + return min(first_positions, key=first_positions.get) + + +# ── Main Analysis Pipeline ────────────────────────────────────────────────── + + +def analyze(raw_data: dict) -> dict: + """Run the full analysis pipeline on raw response data.""" + meta = raw_data["meta"] + responses = raw_data["responses"] + brand = meta["brand_name"] + competitors = meta["competitors"] + all_names = [brand] + competitors + website_url = meta.get("website_url", "") + + if not responses: + print(" [WARN] No responses to analyze (empty dataset).") + + # Per-prompt results + prompt_results = [] + # Aggregate trackers + mention_counts = defaultdict(int) + rank_sums = defaultdict(float) + rank_counts = defaultdict(int) + win_counts = defaultdict(int) + all_cited_domains = defaultdict(int) + brand_cited_count = 0 + competitor_language = defaultdict(list) + total_prompts_per_provider = defaultdict(int) + + for resp in responses: + text = resp["response"] + prompt_text = resp["prompt"] + provider = resp["provider"] + + total_prompts_per_provider[provider] += 1 + + # 1. Mention detection + mentions = find_mentions(text, all_names) + + # 2. Brand rank + brand_rank = compute_rank(mentions, brand) + + # 3. Track mention counts + for name in all_names: + if name in mentions: + mention_counts[name] += 1 + + # 4. Track ranks + if brand_rank is not None: + rank_sums[brand] += brand_rank + rank_counts[brand] += 1 + + for comp in competitors: + comp_rank = compute_rank(mentions, comp) + if comp_rank is not None: + rank_sums[comp] += comp_rank + rank_counts[comp] += 1 + + # 5. Winner + winner = determine_winner(mentions) + if winner: + win_counts[winner] += 1 + + # 6. Sentiment for brand + brand_sentiment = compute_sentiment(text, brand) + + # 7. Citations + cited_domains = extract_citations(text) + for domain in cited_domains: + all_cited_domains[domain] += 1 + + # Check if brand's domain is cited + if website_url: + brand_domain = re.sub(r'^https?://(www\.)?', '', website_url).split('/')[0].lower() + if brand_domain and any(brand_domain in d for d in cited_domains): + brand_cited_count += 1 + + # 8. Competitor mentions list + comps_mentioned = [c for c in competitors if c in mentions] + + # 9. Framing for competitors + for comp in comps_mentioned: + framing = extract_framing(text, comp) + if framing: + competitor_language[comp].extend(framing) + + prompt_results.append({ + "prompt": prompt_text, + "prompt_category": resp.get("prompt_category", "unknown"), + "provider": provider, + "brand_mentioned": brand in mentions, + "brand_rank": brand_rank, + "competitors_mentioned": comps_mentioned, + "winner": winner, + "sentiment": brand_sentiment, + "cited_domains": cited_domains, + }) + + # ── Aggregate Metrics ─────────────────────────────────────────────── + + total_responses = len(responses) + + # Share of voice + share_of_voice = {} + for name in all_names: + mc = mention_counts.get(name, 0) + avg_rank = round(rank_sums[name] / rank_counts[name], 2) if rank_counts[name] > 0 else None + share_of_voice[name] = { + "mention_count": mc, + "mention_rate": round(mc / total_responses, 3) if total_responses > 0 else 0, + "avg_rank": avg_rank, + "win_count": win_counts.get(name, 0), + "win_rate": round(win_counts.get(name, 0) / total_responses, 3) if total_responses > 0 else 0, + } + + # Wins and losses + wins = [r for r in prompt_results if r["brand_mentioned"] and r["brand_rank"] == 1] + losses = [r for r in prompt_results if not r["brand_mentioned"] or (r["brand_rank"] and r["brand_rank"] > 1)] + + # Citation gaps + citation_gaps = [] + for domain, count in sorted(all_cited_domains.items(), key=lambda x: -x[1])[:20]: + citation_gaps.append({ + "domain": domain, + "cited_count": count, + "is_brand_domain": False, # Updated below + }) + + if website_url: + brand_domain = re.sub(r'^https?://(www\.)?', '', website_url).split('/')[0].lower() + for gap in citation_gaps: + if brand_domain and brand_domain in gap["domain"]: + gap["is_brand_domain"] = True + + # Deduplicate competitor language + for comp in competitor_language: + unique = list(set(competitor_language[comp])) + competitor_language[comp] = sorted(unique)[:15] + + return { + "meta": { + "brand": brand, + "competitors": competitors, + "category": meta["category"], + "website_url": meta.get("website_url", ""), + "total_responses": total_responses, + "providers_used": meta["providers_used"], + "timestamp": datetime.now(timezone.utc).isoformat(), + }, + "share_of_voice": share_of_voice, + "prompt_results": prompt_results, + "wins": [{"prompt": w["prompt"], "provider": w["provider"]} for w in wins], + "losses": [ + { + "prompt": l["prompt"], + "provider": l["provider"], + "winner": l["winner"], + "brand_rank": l["brand_rank"], + } + for l in losses + ], + "citation_gaps": citation_gaps, + "competitor_language": dict(competitor_language), + } + + +# ── Entry Point ───────────────────────────────────────────────────────────── + + +def main(): + print("\n" + "=" * 60) + print(" GEO Gap Fixer — Analysis Engine") + print("=" * 60) + + if not RAW_RESPONSES_PATH.exists(): + print(f"\n[ERROR] Raw responses not found: {RAW_RESPONSES_PATH}") + print(" Run probe_llms.py first.") + sys.exit(1) + + try: + with open(RAW_RESPONSES_PATH, "r", encoding="utf-8") as f: + raw_data = json.load(f) + except json.JSONDecodeError as e: + print(f"\n[ERROR] Invalid JSON in {RAW_RESPONSES_PATH}: {e}") + sys.exit(1) + + # Validate structure + if "responses" not in raw_data or "meta" not in raw_data: + print("\n[ERROR] raw_responses.json is missing 'responses' or 'meta' key.") + print(" Re-run probe_llms.py to regenerate.") + sys.exit(1) + + total = len(raw_data.get("responses", [])) + print(f" Loaded {total} responses from {RAW_RESPONSES_PATH.name}") + print("-" * 60) + + # Run analysis + analysis = analyze(raw_data) + + # Save + DATA_DIR.mkdir(parents=True, exist_ok=True) + with open(ANALYSIS_PATH, "w", encoding="utf-8") as f: + json.dump(analysis, f, indent=2, ensure_ascii=False) + + # Print summary + brand = analysis["meta"]["brand"] + sov = analysis["share_of_voice"] + print(f"\n Share of Voice:") + print(f" {'Name':<20} {'Mention Rate':>12} {'Avg Rank':>10} {'Win Rate':>10}") + print(f" {'-'*20} {'-'*12} {'-'*10} {'-'*10}") + + for name in sorted(sov, key=lambda n: -sov[n]["mention_rate"]): + data = sov[name] + mr = f"{data['mention_rate']:.0%}" + ar = f"{data['avg_rank']}" if data['avg_rank'] else "—" + wr = f"{data['win_rate']:.0%}" + marker = " ← YOU" if name == brand else "" + print(f" {name:<20} {mr:>12} {ar:>10} {wr:>10}{marker}") + + print(f"\n Wins: {len(analysis['wins'])}") + print(f" Losses: {len(analysis['losses'])}") + print(f" Citations tracked: {len(analysis['citation_gaps'])} domains") + + print(f"\n ✅ Saved analysis to {ANALYSIS_PATH}") + print("=" * 60 + "\n") + + +if __name__ == "__main__": + main() diff --git a/skills/geo-gap-fixer/scripts/build_report.py b/skills/geo-gap-fixer/scripts/build_report.py new file mode 100644 index 0000000..b5bfc7a --- /dev/null +++ b/skills/geo-gap-fixer/scripts/build_report.py @@ -0,0 +1,445 @@ +#!/usr/bin/env python3 +""" +build_report.py — Assemble the GEO audit report from analysis data. + +Reads data/analysis.json and produces: + - report/geo_audit_report.md (human-readable markdown) + - report/geo_audit_report.json (machine-readable structured data) + +The markdown report contains all 5 required sections: + 1. Share-of-Voice Table + 2. Prompt-Level Loss Log + 3. Competitor Language Patterns + 4. Citation Gap List + 5. GEO Action Plan + +Usage: + python scripts/build_report.py +""" + +import json +import sys +from pathlib import Path + +# Schema version for the JSON report — bump when output shape changes. +SCHEMA_VERSION = "1.0.0" + +# Fix Windows console encoding +try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + sys.stderr.reconfigure(encoding="utf-8", errors="replace") +except AttributeError: + pass + +# ── Constants ─────────────────────────────────────────────────────────────── + +SCRIPT_DIR = Path(__file__).resolve().parent +SKILL_ROOT = SCRIPT_DIR.parent +DATA_DIR = SKILL_ROOT / "data" +ANALYSIS_PATH = DATA_DIR / "analysis.json" +REPORT_DIR = SKILL_ROOT / "report" +REPORT_MD_PATH = REPORT_DIR / "geo_audit_report.md" +REPORT_JSON_PATH = REPORT_DIR / "geo_audit_report.json" + + +# ── Report Sections ──────────────────────────────────────────────────────── + + +def build_header(meta: dict) -> str: + """Build the report header.""" + providers = ", ".join(meta["providers_used"]) + return f"""# GEO Gap Audit Report — {meta['brand']} + +> **Generated**: {meta['timestamp']} +> **Category**: {meta['category']} +> **Providers**: {providers} +> **Total responses analyzed**: {meta['total_responses']} + +--- +""" + + +def build_share_of_voice(sov: dict, brand: str) -> str: + """Section 1: Share-of-Voice Table.""" + lines = [ + "## 1. Share-of-Voice Table\n", + "How often each brand is mentioned and recommended by LLMs.\n", + "| Brand | Mention Rate | Avg Rank | Win Rate | Mentioned | Wins |", + "|-------|-------------|----------|----------|-----------|------|", + ] + + sorted_names = sorted(sov, key=lambda n: -sov[n]["mention_rate"]) + for name in sorted_names: + d = sov[name] + mr = f"{d['mention_rate']:.0%}" + ar = f"{d['avg_rank']:.1f}" if d["avg_rank"] is not None else "—" + wr = f"{d['win_rate']:.0%}" + mc = str(d["mention_count"]) + wc = str(d["win_count"]) + marker = " **← YOU**" if name == brand else "" + lines.append(f"| {name}{marker} | {mr} | {ar} | {wr} | {mc} | {wc} |") + + # Health assessment + brand_data = sov.get(brand, {}) + mr = brand_data.get("mention_rate", 0) + wr = brand_data.get("win_rate", 0) + + if mr >= 0.6 and wr >= 0.3: + health = "🟢 **Healthy** — LLMs consistently recommend you" + elif mr >= 0.3 or wr >= 0.1: + health = "🟡 **At Risk** — Visible but not dominant; targeted fixes needed" + else: + health = "🔴 **Critical** — LLMs rarely mention you; major GEO overhaul needed" + + lines.append(f"\n**Overall Health**: {health}\n") + lines.append("---\n") + return "\n".join(lines) + + +def build_loss_log(losses: list, brand: str) -> str: + """Section 2: Prompt-Level Loss Log.""" + lines = [ + "## 2. Prompt-Level Loss Log\n", + "Prompts where your brand was NOT mentioned first (or not at all).\n", + ] + + if not losses: + lines.append("🎉 **No losses detected!** Your brand was mentioned first in every response.\n") + lines.append("---\n") + return "\n".join(lines) + + lines.extend([ + "| # | Prompt | Provider | Winner | Your Rank |", + "|---|--------|----------|--------|-----------|", + ]) + + for i, loss in enumerate(losses[:30], 1): # Cap at 30 rows + prompt = loss["prompt"] + if len(prompt) > 60: + prompt = prompt[:57] + "..." + # Escape pipe characters that would break the markdown table + prompt = prompt.replace("|", "\\|") + provider = loss["provider"] + winner = (loss.get("winner") or "—").replace("|", "\\|") + rank = str(loss.get("brand_rank")) if loss.get("brand_rank") else "Not mentioned" + lines.append(f"| {i} | {prompt} | {provider} | {winner} | {rank} |") + + if len(losses) > 30: + lines.append(f"\n*...and {len(losses) - 30} more losses (see JSON report for full list)*\n") + + lines.append(f"\n**Total losses**: {len(losses)} out of all analyzed responses\n") + lines.append("---\n") + return "\n".join(lines) + + +def build_competitor_language(comp_lang: dict, brand: str) -> str: + """Section 3: Competitor Language Patterns.""" + lines = [ + "## 3. Competitor Language Patterns\n", + "How LLMs describe your competitors — keywords and framing you may be missing.\n", + ] + + if not comp_lang: + lines.append("No competitor language patterns detected.\n") + lines.append("---\n") + return "\n".join(lines) + + for comp, phrases in sorted(comp_lang.items()): + lines.append(f"### {comp}") + if phrases: + lines.append(f"**Framing used by LLMs**: {', '.join(f'`{p}`' for p in phrases[:10])}") + else: + lines.append("No distinctive framing detected.") + lines.append("") + + lines.append( + "**💡 Insight**: If competitors are described with keywords your brand lacks, " + "consider incorporating similar language into your website copy, " + "comparison pages, and product descriptions.\n" + ) + lines.append("---\n") + return "\n".join(lines) + + +def build_citation_gaps(gaps: list, meta: dict) -> str: + """Section 4: Citation Gap List.""" + lines = [ + "## 4. Citation Gap List\n", + "Domains cited by LLMs in their responses. If your domain isn't here, " + "LLMs don't consider your site an authoritative source.\n", + ] + + if not gaps: + lines.append("No citations detected in LLM responses.\n") + lines.append("---\n") + return "\n".join(lines) + + lines.extend([ + "| Domain | Times Cited | Your Domain? |", + "|--------|-------------|-------------|", + ]) + + for gap in gaps[:20]: + domain = gap["domain"] + count = gap["cited_count"] + is_brand = "✅ Yes" if gap.get("is_brand_domain") else "❌ No" + lines.append(f"| {domain} | {count} | {is_brand} |") + + brand_cited = any(g.get("is_brand_domain") for g in gaps) + if not brand_cited and meta.get("website_url"): + lines.append( + f"\n⚠️ **Your domain ({meta['website_url']}) was never cited by any LLM.**\n" + ) + + lines.append("---\n") + return "\n".join(lines) + + +def build_action_plan(analysis: dict) -> str: + """Section 5: GEO Action Plan — the core deliverable.""" + sov = analysis["share_of_voice"] + losses = analysis["losses"] + gaps = analysis["citation_gaps"] + comp_lang = analysis["competitor_language"] + brand = analysis["meta"]["brand"] + competitors = analysis["meta"]["competitors"] + category = analysis["meta"]["category"] + + brand_data = sov.get(brand, {}) + mr = brand_data.get("mention_rate", 0) + wr = brand_data.get("win_rate", 0) + + lines = [ + "## 5. GEO Action Plan\n", + "Prioritized content and positioning fixes based on the audit findings.\n", + ] + + # ── Critical Actions ──────────────────────────────────────────── + critical = [] + + if mr < 0.3: + critical.append( + f'- [ ] **Create FAQ page**: "What is the best {category}?" — ' + f"your brand is mentioned in only {mr:.0%} of responses" + ) + + # Find top competitor + top_comp = None + top_comp_wr = 0 + for comp in competitors: + cwr = sov.get(comp, {}).get("win_rate", 0) + if cwr > top_comp_wr: + top_comp = comp + top_comp_wr = cwr + + if top_comp and top_comp_wr > wr: + critical.append( + f'- [ ] **Create comparison page**: "{brand} vs {top_comp}" — ' + f"{top_comp} wins {top_comp_wr:.0%} of prompts vs your {wr:.0%}" + ) + + # Check for losses on direct prompts + direct_losses = [l for l in losses if "best" in l["prompt"].lower() or "recommend" in l["prompt"].lower()] + if len(direct_losses) > 3: + critical.append( + f"- [ ] **Optimize landing page** for \"{category}\" — " + f"you lost {len(direct_losses)} direct recommendation prompts" + ) + + if critical: + lines.append("### 🔴 Critical (Do First)\n") + lines.extend(critical) + lines.append("") + + # ── High Priority ─────────────────────────────────────────────── + high = [] + + # Alternative pages for each competitor that beats the brand + for comp in competitors: + comp_mr = sov.get(comp, {}).get("mention_rate", 0) + if comp_mr > mr and comp != top_comp: + high.append( + f'- [ ] **Create alternatives page**: "Best alternatives to {comp}" — ' + f"{comp} has {comp_mr:.0%} mention rate vs your {mr:.0%}" + ) + + # Schema markup if no citations + brand_cited = any(g.get("is_brand_domain") for g in gaps) + if not brand_cited: + high.append( + "- [ ] **Add schema markup**: FAQPage, SoftwareApplication, Organization — " + "your domain is not being cited by any LLM" + ) + + # Content depth if sentiment is neutral/negative + brand_sent = None + for pr in analysis.get("prompt_results", []): + if pr.get("sentiment", {}).get("label") == "negative": + brand_sent = "negative" + break + if brand_sent == "negative": + high.append( + "- [ ] **Address negative framing**: Review and update product descriptions " + "to counter negative keywords found in LLM responses" + ) + + if high: + lines.append("### 🟡 High Priority\n") + lines.extend(high) + lines.append("") + + # ── Growth Plays ──────────────────────────────────────────────── + growth = [] + + # Authority domains + cited_domains = [g["domain"] for g in gaps if not g.get("is_brand_domain") and g["cited_count"] >= 2] + if cited_domains[:3]: + domains_str = ", ".join(cited_domains[:3]) + growth.append( + f"- [ ] **Build authority on cited domains**: Get listed/mentioned on {domains_str} — " + "these are domains LLMs trust and cite" + ) + + # Competitor language mirroring + if comp_lang: + all_comp_phrases = set() + for phrases in comp_lang.values(): + all_comp_phrases.update(phrases[:5]) + if all_comp_phrases: + sample = ", ".join(f'"{p}"' for p in list(all_comp_phrases)[:5]) + growth.append( + f"- [ ] **Mirror competitor framing**: Incorporate language like {sample} " + "into your product pages and descriptions" + ) + + # Data-backed content + growth.append( + '- [ ] **Publish original data/statistics**: Add "according to" citations, ' + "benchmarks, and original research that LLMs can extract and cite" + ) + + # Regular auditing + growth.append( + "- [ ] **Schedule monthly re-audits**: Run this skill monthly to track " + "whether your GEO improvements are working" + ) + + if growth: + lines.append("### 🟢 Growth Plays\n") + lines.extend(growth) + lines.append("") + + # ── Summary ───────────────────────────────────────────────────── + total_actions = len(critical) + len(high) + len(growth) + lines.append(f"\n**Total actions**: {total_actions} items in your GEO backlog") + lines.append(f"**Focus first on**: 🔴 Critical items ({len(critical)} actions)\n") + lines.append("---\n") + lines.append( + "*Report generated by [geo-gap-fixer](https://github.com/Varnan-Tech/opendirectory) " + "— an OpenDirectory skill for GEO auditing.*\n" + ) + + return "\n".join(lines) + + +# ── Report Assembly ───────────────────────────────────────────────────────── + + +def build_full_report(analysis: dict) -> str: + """Assemble the full markdown report.""" + sections = [ + build_header(analysis["meta"]), + build_share_of_voice(analysis["share_of_voice"], analysis["meta"]["brand"]), + build_loss_log(analysis["losses"], analysis["meta"]["brand"]), + build_competitor_language(analysis["competitor_language"], analysis["meta"]["brand"]), + build_citation_gaps(analysis["citation_gaps"], analysis["meta"]), + build_action_plan(analysis), + ] + return "\n".join(sections) + + +def build_json_report(analysis: dict) -> dict: + """Build the structured JSON report.""" + sov = analysis["share_of_voice"] + brand = analysis["meta"]["brand"] + brand_data = sov.get(brand, {}) + + return { + "schema_version": SCHEMA_VERSION, + "meta": analysis["meta"], + "summary": { + "brand_mention_rate": brand_data.get("mention_rate", 0), + "brand_win_rate": brand_data.get("win_rate", 0), + "brand_avg_rank": brand_data.get("avg_rank"), + "total_wins": len(analysis.get("wins", [])), + "total_losses": len(analysis.get("losses", [])), + "citation_domains_found": len(analysis.get("citation_gaps", [])), + }, + "share_of_voice": analysis["share_of_voice"], + "prompt_results": analysis.get("prompt_results", []), + "losses": analysis.get("losses", []), + "citation_gaps": analysis.get("citation_gaps", []), + "competitor_language": analysis.get("competitor_language", {}), + } + + +# ── Entry Point ───────────────────────────────────────────────────────────── + + +def main(): + print("\n" + "=" * 60) + print(" GEO Gap Fixer — Report Builder") + print("=" * 60) + + if not ANALYSIS_PATH.exists(): + print(f"\n[ERROR] Analysis file not found: {ANALYSIS_PATH}") + print(" Run analyze_results.py first.") + sys.exit(1) + + try: + with open(ANALYSIS_PATH, "r", encoding="utf-8") as f: + analysis = json.load(f) + except json.JSONDecodeError as e: + print(f"\n[ERROR] Invalid JSON in {ANALYSIS_PATH}: {e}") + sys.exit(1) + + # Validate required keys + required_keys = ["meta", "share_of_voice", "losses"] + missing = [k for k in required_keys if k not in analysis] + if missing: + print(f"\n[ERROR] analysis.json is missing required keys: {', '.join(missing)}") + print(" Re-run analyze_results.py to regenerate.") + sys.exit(1) + + brand = analysis["meta"]["brand"] + print(f" Building report for: {brand}") + print("-" * 60) + + # Build reports + md_report = build_full_report(analysis) + json_report = build_json_report(analysis) + + # Save + REPORT_DIR.mkdir(parents=True, exist_ok=True) + + with open(REPORT_MD_PATH, "w", encoding="utf-8") as f: + f.write(md_report) + + with open(REPORT_JSON_PATH, "w", encoding="utf-8") as f: + json.dump(json_report, f, indent=2, ensure_ascii=False) + + print(f"\n ✅ Markdown report: {REPORT_MD_PATH}") + print(f" ✅ JSON report: {REPORT_JSON_PATH}") + + # Quick preview + sov = analysis["share_of_voice"] + brand_data = sov.get(brand, {}) + print(f"\n 📊 Brand mention rate: {brand_data.get('mention_rate', 0):.0%}") + print(f" 📊 Brand win rate: {brand_data.get('win_rate', 0):.0%}") + print(f" 📊 Total losses: {len(analysis['losses'])}") + print(f"\n → Open {REPORT_MD_PATH.name} to see your GEO action plan") + print("=" * 60 + "\n") + + +if __name__ == "__main__": + main() diff --git a/skills/geo-gap-fixer/scripts/probe_llms.py b/skills/geo-gap-fixer/scripts/probe_llms.py new file mode 100644 index 0000000..1b7ca82 --- /dev/null +++ b/skills/geo-gap-fixer/scripts/probe_llms.py @@ -0,0 +1,492 @@ +#!/usr/bin/env python3 +""" +probe_llms.py — Send buyer-intent prompts to LLM APIs and capture raw responses. + +Sends each prompt to every configured LLM provider, captures the full text +response, and saves all results to data/raw_responses.json. + +Graceful degradation: if an API key is missing or a provider errors out, +that provider is skipped with a warning — the script never crashes. + +Usage: + python scripts/probe_llms.py [--config path/to/config.json] +""" + +import json +import os +import re +import sys +import time +import argparse +from datetime import datetime, timezone +from pathlib import Path +from typing import Callable + +# Fix Windows console encoding +try: + sys.stdout.reconfigure(encoding="utf-8", errors="replace") + sys.stderr.reconfigure(encoding="utf-8", errors="replace") +except AttributeError: + pass + +# ── Constants ─────────────────────────────────────────────────────────────── + +SCRIPT_DIR = Path(__file__).resolve().parent +SKILL_ROOT = SCRIPT_DIR.parent +DEFAULT_CONFIG = SKILL_ROOT / "config.json" +DATA_DIR = SKILL_ROOT / "data" +RAW_RESPONSES_PATH = DATA_DIR / "raw_responses.json" +PROMPT_TEMPLATES_PATH = SKILL_ROOT / "references" / "prompt_templates.md" + +MAX_RESPONSE_CHARS = 4000 # Cap response length to avoid memory issues +SLEEP_BETWEEN_CALLS = 1.0 # seconds between API calls +MAX_RETRIES = 2 # Retry transient API failures +RETRY_BACKOFF = 2.0 # Exponential backoff base (seconds) + +# System prompt that normalizes LLM output for easier parsing +SYSTEM_PROMPT = ( + "You are a helpful assistant. When recommending tools or products, " + "list them clearly by name. If you cite sources, include full URLs." +) + +PROVIDER_ENV_KEYS = { + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "google": "GOOGLE_API_KEY", + "perplexity": "PERPLEXITY_API_KEY", +} + +# ── Config Loading ────────────────────────────────────────────────────────── + + +def load_config(config_path: Path) -> dict: + """Load and validate the user config file.""" + if not config_path.exists(): + print(f"[ERROR] Config file not found: {config_path}") + print(" Copy config.example.json to config.json and fill in your details.") + sys.exit(1) + + try: + with open(config_path, "r", encoding="utf-8") as f: + config = json.load(f) + except json.JSONDecodeError as e: + print(f"[ERROR] Invalid JSON in {config_path}: {e}") + sys.exit(1) + + # Validate required fields + required = ["brand_name", "competitors", "category"] + missing = [k for k in required if not config.get(k)] + if missing: + print(f"[ERROR] Missing required config fields: {', '.join(missing)}") + sys.exit(1) + + if not isinstance(config["competitors"], list) or len(config["competitors"]) < 1: + print("[ERROR] 'competitors' must be a list with at least 1 entry.") + sys.exit(1) + + if len(config["competitors"]) > 10: + print("[WARN] More than 10 competitors will produce many API calls. Trimming to 10.") + config["competitors"] = config["competitors"][:10] + + # Validate target_llms if provided + config.setdefault("buyer_intent_prompts", []) + config.setdefault("target_llms", ["openai", "anthropic", "google", "perplexity"]) + config.setdefault("website_url", "") + + valid_llms = set(PROVIDER_ENV_KEYS.keys()) + invalid = [llm for llm in config["target_llms"] if llm not in valid_llms] + if invalid: + print(f"[WARN] Unknown LLM providers ignored: {', '.join(invalid)}") + config["target_llms"] = [llm for llm in config["target_llms"] if llm in valid_llms] + + return config + + +# ── Prompt Generation ─────────────────────────────────────────────────────── + + +def load_prompt_templates() -> list[str]: + """Load prompt templates from references/prompt_templates.md.""" + if not PROMPT_TEMPLATES_PATH.exists(): + return get_fallback_templates() + + templates = [] + with open(PROMPT_TEMPLATES_PATH, "r", encoding="utf-8") as f: + for line in f: + # Extract templates from markdown table rows: | # | Category | "template" | + match = re.search(r'"([^"]+\{[^"]+)"', line) + if match: + templates.append(match.group(1)) + + return templates if templates else get_fallback_templates() + + +def get_fallback_templates() -> list[str]: + """Hardcoded fallback templates if the reference file is missing.""" + return [ + "What is the best {category}?", + "What {category} do you recommend?", + "Top {category} in 2026", + "Best {category} for startups", + "Best {category} for teams", + "{brand} vs {competitor}: which is better?", + "Compare {brand} and {competitor} for {category}", + "Should I use {brand} or {competitor}?", + "{brand} vs {competitor} pros and cons", + "Differences between {brand} and {competitor}", + "I need a {category} that is fast and simple", + "Best {category} for developer teams", + "What {category} has the best API?", + "Most affordable {category} for small teams", + "{category} with best integrations", + "Best alternatives to {competitor}", + "Cheaper alternatives to {competitor}", + "What to use instead of {competitor}", + "Moving away from {competitor}, what should I try?", + "{competitor} competitors worth trying", + ] + + +def generate_prompts(config: dict) -> list[dict]: + """Generate the full list of prompts from config or templates.""" + # If user supplied custom prompts, use them directly + if config["buyer_intent_prompts"]: + return [ + {"text": p, "category": "custom", "variables": {}} + for p in config["buyer_intent_prompts"] + ] + + templates = load_prompt_templates() + prompts = [] + brand = config["brand_name"] + category = config["category"] + competitors = config["competitors"] + + for template in templates: + try: + if "{competitor}" in template: + # Generate one prompt per competitor + for comp in competitors: + text = template.format( + brand=brand, competitor=comp, category=category + ) + prompts.append({ + "text": text, + "category": _classify_template(template), + "variables": {"brand": brand, "competitor": comp, "category": category}, + }) + else: + text = template.format(brand=brand, category=category) + prompts.append({ + "text": text, + "category": _classify_template(template), + "variables": {"brand": brand, "category": category}, + }) + except KeyError as e: + print(f" [WARN] Skipping template with unknown variable {e}: {template}") + + return prompts + + +def _classify_template(template: str) -> str: + """Classify a template into a category based on its pattern.""" + t = template.lower() + if "vs" in t or "compare" in t or "should i use" in t or "differences" in t: + return "comparison" + if "alternative" in t or "instead of" in t or "moving away" in t or "competitors" in t: + return "alternative" + if "i need" in t or "best api" in t or "affordable" in t or "integration" in t or "developer" in t: + return "problem" + return "direct" + + +# ── Provider Adapters ─────────────────────────────────────────────────────── + + +def _retry(fn: Callable, provider_name: str) -> dict | None: + """Retry a provider call with exponential backoff on transient errors.""" + for attempt in range(MAX_RETRIES + 1): + result = fn() + if result is not None: + return result + if attempt < MAX_RETRIES: + wait = RETRY_BACKOFF ** (attempt + 1) + print(f" → retrying {provider_name} in {wait:.0f}s (attempt {attempt + 2}/{MAX_RETRIES + 1})") + time.sleep(wait) + return None + + +def probe_openai(prompt_text: str) -> dict | None: + """Send prompt to OpenAI gpt-4o. Returns None if key missing or error.""" + api_key = os.environ.get("OPENAI_API_KEY") + if not api_key: + return None + + try: + from openai import OpenAI + except ImportError: + print(" [WARN] openai package not installed. pip install openai") + return None + + def _call() -> dict | None: + try: + client = OpenAI(api_key=api_key) + response = client.chat.completions.create( + model="gpt-4o", + messages=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": prompt_text}, + ], + max_tokens=2048, + temperature=0.0, # Deterministic output + ) + text = response.choices[0].message.content or "" + return { + "provider": "openai", + "model": "gpt-4o", + "response": text[:MAX_RESPONSE_CHARS], + } + except Exception as e: + print(f" [WARN] OpenAI error: {e}") + return None + + return _retry(_call, "openai") + + +def probe_anthropic(prompt_text: str) -> dict | None: + """Send prompt to Anthropic claude-sonnet-4-6.""" + api_key = os.environ.get("ANTHROPIC_API_KEY") + if not api_key: + return None + + try: + import anthropic + except ImportError: + print(" [WARN] anthropic package not installed. pip install anthropic") + return None + + def _call() -> dict | None: + try: + client = anthropic.Anthropic(api_key=api_key) + response = client.messages.create( + model="claude-sonnet-4-6", + max_tokens=2048, + system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": prompt_text}], + ) + text = response.content[0].text if response.content else "" + return { + "provider": "anthropic", + "model": "claude-sonnet-4-6", + "response": text[:MAX_RESPONSE_CHARS], + } + except Exception as e: + print(f" [WARN] Anthropic error: {e}") + return None + + return _retry(_call, "anthropic") + + +def probe_google(prompt_text: str) -> dict | None: + """Send prompt to Google gemini-2.5-flash.""" + api_key = os.environ.get("GOOGLE_API_KEY") + if not api_key: + return None + + try: + from google import genai + except ImportError: + print(" [WARN] google-genai package not installed. pip install google-genai") + return None + + def _call() -> dict | None: + try: + client = genai.Client(api_key=api_key) + response = client.models.generate_content( + model="gemini-2.5-flash", + contents=f"{SYSTEM_PROMPT}\n\n{prompt_text}", + ) + text = response.text or "" + return { + "provider": "google", + "model": "gemini-2.5-flash", + "response": text[:MAX_RESPONSE_CHARS], + } + except Exception as e: + print(f" [WARN] Google error: {e}") + return None + + return _retry(_call, "google") + + +def probe_perplexity(prompt_text: str) -> dict | None: + """Send prompt to Perplexity sonar-pro via OpenAI-compatible API.""" + api_key = os.environ.get("PERPLEXITY_API_KEY") + if not api_key: + return None + + try: + from openai import OpenAI + except ImportError: + print(" [WARN] openai package not installed. pip install openai") + return None + + def _call() -> dict | None: + try: + client = OpenAI( + api_key=api_key, + base_url="https://api.perplexity.ai", + ) + response = client.chat.completions.create( + model="sonar-pro", + messages=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": prompt_text}, + ], + ) + text = response.choices[0].message.content or "" + return { + "provider": "perplexity", + "model": "sonar-pro", + "response": text[:MAX_RESPONSE_CHARS], + } + except Exception as e: + print(f" [WARN] Perplexity error: {e}") + return None + + return _retry(_call, "perplexity") + + +# Provider dispatch map +PROVIDERS = { + "openai": probe_openai, + "anthropic": probe_anthropic, + "google": probe_google, + "perplexity": probe_perplexity, +} + + +# ── Main Orchestrator ─────────────────────────────────────────────────────── + + +def main(): + parser = argparse.ArgumentParser(description="Probe LLMs with buyer-intent prompts") + parser.add_argument( + "--config", + type=Path, + default=DEFAULT_CONFIG, + help="Path to config.json (default: config.json in skill root)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="Validate config and show prompts without making API calls", + ) + args = parser.parse_args() + + print("\n" + "=" * 60) + print(" GEO Gap Fixer — LLM Probe") + print("=" * 60) + + # 1. Load config + config = load_config(args.config) + print(f" Brand: {config['brand_name']}") + print(f" Competitors: {', '.join(config['competitors'])}") + print(f" Category: {config['category']}") + + # 2. Determine available providers + target_llms = config["target_llms"] + available = [] + skipped = [] + for llm in target_llms: + env_key = PROVIDER_ENV_KEYS.get(llm) + if env_key and os.environ.get(env_key): + available.append(llm) + else: + skipped.append(llm) + + if len(available) < 1 and not args.dry_run: + print("\n[ERROR] No API keys found. Set at least 2 of:") + for llm, key in PROVIDER_ENV_KEYS.items(): + print(f" export {key}=...") + sys.exit(1) + + print(f"\n Providers: {', '.join(available)}") + if skipped: + print(f" Skipped: {', '.join(skipped)} (no API key)") + + # 3. Generate prompts + prompts = generate_prompts(config) + print(f" Prompts: {len(prompts)}") + + if args.dry_run: + print("\n [DRY RUN] Showing first 5 prompts:") + for i, p in enumerate(prompts[:5], 1): + print(f" {i}. [{p['category']}] {p['text']}") + print(f"\n Total: {len(prompts)} prompts × {len(available)} providers = {len(prompts) * len(available)} API calls") + print(" No API calls made. Remove --dry-run to execute.") + sys.exit(0) + + print("-" * 60) + + # 4. Probe each prompt against each provider + results = [] + total = len(prompts) * len(available) + count = 0 + + for prompt_obj in prompts: + for llm in available: + count += 1 + prompt_text = prompt_obj["text"] + short = prompt_text[:50] + "..." if len(prompt_text) > 50 else prompt_text + print(f" [{count}/{total}] {llm}: {short}") + + probe_fn = PROVIDERS[llm] + result = probe_fn(prompt_text) + + if result: + result["prompt"] = prompt_text + result["prompt_category"] = prompt_obj["category"] + result["prompt_variables"] = prompt_obj["variables"] + results.append(result) + else: + print(f" → skipped (error or missing key)") + + time.sleep(SLEEP_BETWEEN_CALLS) + + # 5. Save results + DATA_DIR.mkdir(parents=True, exist_ok=True) + + output = { + "meta": { + "brand_name": config["brand_name"], + "competitors": config["competitors"], + "category": config["category"], + "website_url": config.get("website_url", ""), + "target_llms": target_llms, + "providers_used": available, + "providers_skipped": skipped, + "total_prompts": len(prompts), + "total_responses": len(results), + "timestamp": datetime.now(timezone.utc).isoformat(), + }, + "responses": results, + } + + with open(RAW_RESPONSES_PATH, "w", encoding="utf-8") as f: + json.dump(output, f, indent=2, ensure_ascii=False) + + print("\n" + "=" * 60) + print(f" ✅ Saved {len(results)} responses to {RAW_RESPONSES_PATH}") + print(f" {len(prompts)} prompts × {len(available)} providers") + if skipped: + print(f" ⚠️ Skipped providers: {', '.join(skipped)}") + print("=" * 60 + "\n") + + if len(results) == 0: + print("[ERROR] No responses collected. Check your API keys and network.") + sys.exit(1) + + +if __name__ == "__main__": + main() From a7f000a3007fe01a06f8d7c23f607f3c81398f39 Mon Sep 17 00:00:00 2001 From: ajaycodesitbetter Date: Fri, 5 Jun 2026 23:07:47 +0530 Subject: [PATCH 2/6] Fix Greptile logic bugs and Copilot review feedback --- .../geo-gap-fixer/scripts/analyze_results.py | 6 +- skills/geo-gap-fixer/scripts/build_report.py | 2 +- skills/geo-gap-fixer/scripts/probe_llms.py | 78 +++++++++++-------- 3 files changed, 51 insertions(+), 35 deletions(-) diff --git a/skills/geo-gap-fixer/scripts/analyze_results.py b/skills/geo-gap-fixer/scripts/analyze_results.py index cd6f002..921f3a8 100644 --- a/skills/geo-gap-fixer/scripts/analyze_results.py +++ b/skills/geo-gap-fixer/scripts/analyze_results.py @@ -41,7 +41,7 @@ ] NEGATIVE_KEYWORDS = [ - "limited", "lacks", "however", "downside", "expensive", "complex", + "limited", "lacks", "downside", "expensive", "complex", "steep learning curve", "missing", "outdated", "slow", "clunky", "basic", "restrictive", "difficult", "confusing", "poor", "frustrating", "buggy", "unreliable", "overpriced", @@ -261,7 +261,7 @@ def analyze(raw_data: dict) -> dict: # Check if brand's domain is cited if website_url: brand_domain = re.sub(r'^https?://(www\.)?', '', website_url).split('/')[0].lower() - if brand_domain and any(brand_domain in d for d in cited_domains): + if brand_domain and any(d == brand_domain or d.endswith("." + brand_domain) for d in cited_domains): brand_cited_count += 1 # 8. Competitor mentions list @@ -318,7 +318,7 @@ def analyze(raw_data: dict) -> dict: if website_url: brand_domain = re.sub(r'^https?://(www\.)?', '', website_url).split('/')[0].lower() for gap in citation_gaps: - if brand_domain and brand_domain in gap["domain"]: + if brand_domain and (gap["domain"] == brand_domain or gap["domain"].endswith("." + brand_domain)): gap["is_brand_domain"] = True # Deduplicate competitor language diff --git a/skills/geo-gap-fixer/scripts/build_report.py b/skills/geo-gap-fixer/scripts/build_report.py index b5bfc7a..fc5bca2 100644 --- a/skills/geo-gap-fixer/scripts/build_report.py +++ b/skills/geo-gap-fixer/scripts/build_report.py @@ -86,7 +86,7 @@ def build_share_of_voice(sov: dict, brand: str) -> str: if mr >= 0.6 and wr >= 0.3: health = "🟢 **Healthy** — LLMs consistently recommend you" - elif mr >= 0.3 or wr >= 0.1: + elif mr >= 0.3 and wr >= 0.1: health = "🟡 **At Risk** — Visible but not dominant; targeted fixes needed" else: health = "🔴 **Critical** — LLMs rarely mention you; major GEO overhaul needed" diff --git a/skills/geo-gap-fixer/scripts/probe_llms.py b/skills/geo-gap-fixer/scripts/probe_llms.py index 1b7ca82..dcdb905 100644 --- a/skills/geo-gap-fixer/scripts/probe_llms.py +++ b/skills/geo-gap-fixer/scripts/probe_llms.py @@ -216,22 +216,25 @@ def _retry(fn: Callable, provider_name: str) -> dict | None: return None +_openai_client = None def probe_openai(prompt_text: str) -> dict | None: """Send prompt to OpenAI gpt-4o. Returns None if key missing or error.""" api_key = os.environ.get("OPENAI_API_KEY") if not api_key: return None - try: - from openai import OpenAI - except ImportError: - print(" [WARN] openai package not installed. pip install openai") - return None + global _openai_client + if _openai_client is None: + try: + from openai import OpenAI + _openai_client = OpenAI(api_key=api_key) + except ImportError: + print(" [WARN] openai package not installed. pip install openai") + return None def _call() -> dict | None: try: - client = OpenAI(api_key=api_key) - response = client.chat.completions.create( + response = _openai_client.chat.completions.create( model="gpt-4o", messages=[ {"role": "system", "content": SYSTEM_PROMPT}, @@ -253,22 +256,25 @@ def _call() -> dict | None: return _retry(_call, "openai") +_anthropic_client = None def probe_anthropic(prompt_text: str) -> dict | None: """Send prompt to Anthropic claude-sonnet-4-6.""" api_key = os.environ.get("ANTHROPIC_API_KEY") if not api_key: return None - try: - import anthropic - except ImportError: - print(" [WARN] anthropic package not installed. pip install anthropic") - return None + global _anthropic_client + if _anthropic_client is None: + try: + import anthropic + _anthropic_client = anthropic.Anthropic(api_key=api_key) + except ImportError: + print(" [WARN] anthropic package not installed. pip install anthropic") + return None def _call() -> dict | None: try: - client = anthropic.Anthropic(api_key=api_key) - response = client.messages.create( + response = _anthropic_client.messages.create( model="claude-sonnet-4-6", max_tokens=2048, system=SYSTEM_PROMPT, @@ -287,22 +293,25 @@ def _call() -> dict | None: return _retry(_call, "anthropic") +_google_client = None def probe_google(prompt_text: str) -> dict | None: """Send prompt to Google gemini-2.5-flash.""" api_key = os.environ.get("GOOGLE_API_KEY") if not api_key: return None - try: - from google import genai - except ImportError: - print(" [WARN] google-genai package not installed. pip install google-genai") - return None + global _google_client + if _google_client is None: + try: + from google import genai + _google_client = genai.Client(api_key=api_key) + except ImportError: + print(" [WARN] google-genai package not installed. pip install google-genai") + return None def _call() -> dict | None: try: - client = genai.Client(api_key=api_key) - response = client.models.generate_content( + response = _google_client.models.generate_content( model="gemini-2.5-flash", contents=f"{SYSTEM_PROMPT}\n\n{prompt_text}", ) @@ -319,25 +328,28 @@ def _call() -> dict | None: return _retry(_call, "google") +_perplexity_client = None def probe_perplexity(prompt_text: str) -> dict | None: """Send prompt to Perplexity sonar-pro via OpenAI-compatible API.""" api_key = os.environ.get("PERPLEXITY_API_KEY") if not api_key: return None - try: - from openai import OpenAI - except ImportError: - print(" [WARN] openai package not installed. pip install openai") - return None - - def _call() -> dict | None: + global _perplexity_client + if _perplexity_client is None: try: - client = OpenAI( + from openai import OpenAI + _perplexity_client = OpenAI( api_key=api_key, base_url="https://api.perplexity.ai", ) - response = client.chat.completions.create( + except ImportError: + print(" [WARN] openai package not installed. pip install openai") + return None + + def _call() -> dict | None: + try: + response = _perplexity_client.chat.completions.create( model="sonar-pro", messages=[ {"role": "system", "content": SYSTEM_PROMPT}, @@ -406,11 +418,15 @@ def main(): skipped.append(llm) if len(available) < 1 and not args.dry_run: - print("\n[ERROR] No API keys found. Set at least 2 of:") + print("\n[ERROR] No API keys found. Set at least 1 of the following to proceed:") for llm, key in PROVIDER_ENV_KEYS.items(): print(f" export {key}=...") sys.exit(1) + if len(available) == 1 and not args.dry_run: + print(f"\n[WARN] Only 1 API key found ({available[0]}). Coverage will be limited.") + print(" For a comprehensive GEO audit, we strongly recommend using 2 or more providers.") + print(f"\n Providers: {', '.join(available)}") if skipped: print(f" Skipped: {', '.join(skipped)} (no API key)") From c3734e014afcba3e9e3b31d6d7167cd67a7a0d94 Mon Sep 17 00:00:00 2001 From: ajaycodesitbetter Date: Fri, 5 Jun 2026 23:21:21 +0530 Subject: [PATCH 3/6] Address remaining Copilot review feedback --- skills/geo-gap-fixer/README.md | 2 +- skills/geo-gap-fixer/scripts/analyze_results.py | 16 ++++++++++++++++ skills/geo-gap-fixer/scripts/build_report.py | 7 +++---- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/skills/geo-gap-fixer/README.md b/skills/geo-gap-fixer/README.md index 65b4fca..77b45fd 100644 --- a/skills/geo-gap-fixer/README.md +++ b/skills/geo-gap-fixer/README.md @@ -83,7 +83,7 @@ Open `report/geo_audit_report.md` for your action plan. | Field | Type | Required | Description | |-------|------|----------|-------------| | `brand_name` | string | ✅ | Your brand name (e.g., "Linear") | -| `competitors` | list | ✅ | 1–5 competitor brand names | +| `competitors` | list | ✅ | 1–10 competitor brand names | | `category` | string | ✅ | Product category (e.g., "project management tool for developers") | | `buyer_intent_prompts` | list | ❌ | Custom prompts. Empty = auto-generate from templates | | `target_llms` | list | ❌ | Which LLMs to probe. Default: all 4 | diff --git a/skills/geo-gap-fixer/scripts/analyze_results.py b/skills/geo-gap-fixer/scripts/analyze_results.py index 921f3a8..79d3614 100644 --- a/skills/geo-gap-fixer/scripts/analyze_results.py +++ b/skills/geo-gap-fixer/scripts/analyze_results.py @@ -317,9 +317,22 @@ def analyze(raw_data: dict) -> dict: if website_url: brand_domain = re.sub(r'^https?://(www\.)?', '', website_url).split('/')[0].lower() + brand_gap_found = False for gap in citation_gaps: if brand_domain and (gap["domain"] == brand_domain or gap["domain"].endswith("." + brand_domain)): gap["is_brand_domain"] = True + brand_gap_found = True + + if brand_domain and not brand_gap_found: + actual_count = 0 + for d, c in all_cited_domains.items(): + if d == brand_domain or d.endswith("." + brand_domain): + actual_count += c + citation_gaps.append({ + "domain": brand_domain, + "cited_count": actual_count, + "is_brand_domain": True, + }) # Deduplicate competitor language for comp in competitor_language: @@ -335,6 +348,9 @@ def analyze(raw_data: dict) -> dict: "total_responses": total_responses, "providers_used": meta["providers_used"], "timestamp": datetime.now(timezone.utc).isoformat(), + "brand_domain_cited": brand_cited_count > 0, + "brand_domain_citation_count": brand_cited_count, + "responses_per_provider": dict(total_prompts_per_provider), }, "share_of_voice": share_of_voice, "prompt_results": prompt_results, diff --git a/skills/geo-gap-fixer/scripts/build_report.py b/skills/geo-gap-fixer/scripts/build_report.py index fc5bca2..d1547ee 100644 --- a/skills/geo-gap-fixer/scripts/build_report.py +++ b/skills/geo-gap-fixer/scripts/build_report.py @@ -185,8 +185,7 @@ def build_citation_gaps(gaps: list, meta: dict) -> str: is_brand = "✅ Yes" if gap.get("is_brand_domain") else "❌ No" lines.append(f"| {domain} | {count} | {is_brand} |") - brand_cited = any(g.get("is_brand_domain") for g in gaps) - if not brand_cited and meta.get("website_url"): + if not meta.get("brand_domain_cited", False) and meta.get("website_url"): lines.append( f"\n⚠️ **Your domain ({meta['website_url']}) was never cited by any LLM.**\n" ) @@ -264,8 +263,8 @@ def build_action_plan(analysis: dict) -> str: ) # Schema markup if no citations - brand_cited = any(g.get("is_brand_domain") for g in gaps) - if not brand_cited: + brand_cited = meta.get("brand_domain_cited", False) + if not brand_cited and meta.get("website_url"): high.append( "- [ ] **Add schema markup**: FAQPage, SoftwareApplication, Organization — " "your domain is not being cited by any LLM" From b8fcf7b2457e6250baf28b7bc1f6e30e184ac39d Mon Sep 17 00:00:00 2001 From: ajaycodesitbetter Date: Fri, 5 Jun 2026 23:28:45 +0530 Subject: [PATCH 4/6] Fix build_action_plan meta NameError --- skills/geo-gap-fixer/scripts/build_report.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/skills/geo-gap-fixer/scripts/build_report.py b/skills/geo-gap-fixer/scripts/build_report.py index d1547ee..fa4ae79 100644 --- a/skills/geo-gap-fixer/scripts/build_report.py +++ b/skills/geo-gap-fixer/scripts/build_report.py @@ -200,9 +200,10 @@ def build_action_plan(analysis: dict) -> str: losses = analysis["losses"] gaps = analysis["citation_gaps"] comp_lang = analysis["competitor_language"] - brand = analysis["meta"]["brand"] - competitors = analysis["meta"]["competitors"] - category = analysis["meta"]["category"] + meta = analysis["meta"] + brand = meta["brand"] + competitors = meta["competitors"] + category = meta["category"] brand_data = sov.get(brand, {}) mr = brand_data.get("mention_rate", 0) From 7ce67a42f492277585e80e0a44abcb2c1e95e95e Mon Sep 17 00:00:00 2001 From: ajaycodesitbetter Date: Fri, 5 Jun 2026 23:37:33 +0530 Subject: [PATCH 5/6] Fix health-label logic in build_share_of_voice --- skills/geo-gap-fixer/scripts/build_report.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/skills/geo-gap-fixer/scripts/build_report.py b/skills/geo-gap-fixer/scripts/build_report.py index fa4ae79..fa699c4 100644 --- a/skills/geo-gap-fixer/scripts/build_report.py +++ b/skills/geo-gap-fixer/scripts/build_report.py @@ -88,6 +88,8 @@ def build_share_of_voice(sov: dict, brand: str) -> str: health = "🟢 **Healthy** — LLMs consistently recommend you" elif mr >= 0.3 and wr >= 0.1: health = "🟡 **At Risk** — Visible but not dominant; targeted fixes needed" + elif mr >= 0.3: + health = "🟡 **At Risk** — Mentioned but rarely ranked first; improve win rate" else: health = "🔴 **Critical** — LLMs rarely mention you; major GEO overhaul needed" From 8e26ee885c135a2f41971bcc0c90a9107e82bb3c Mon Sep 17 00:00:00 2001 From: ajaycodesitbetter Date: Sat, 6 Jun 2026 21:58:46 +0530 Subject: [PATCH 6/6] chore: rebase geo gap fixer and refresh registry --- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- README.md | 8 +- packages/cli/registry.json | 2 +- skills/geo-gap-fixer/README.md | 58 +++++-- .../geo-gap-fixer/scripts/analyze_results.py | 47 ++++-- skills/geo-gap-fixer/scripts/probe_llms.py | 158 ++++++++++-------- 7 files changed, 171 insertions(+), 106 deletions(-) diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index f6d5e78..97c61e4 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -8,7 +8,7 @@ { "name": "opendirectory-gtm-skills", "source": "./", - "description": "A collection of 57 agent skills for founders who hate marketing" + "description": "A collection of 61 agent skills for founders who hate marketing" } ] } \ No newline at end of file diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 2bf6289..592e2c2 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "opendirectory-gtm-skills", "version": "1.0.1", - "description": "A collection of 57 agent skills for founders who hate marketing", + "description": "A collection of 61 agent skills for founders who hate marketing", "author": { "name": "Varnan" }, diff --git a/README.md b/README.md index 3eff1ba..3001642 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@
- Typing SVG + Typing SVG

@@ -17,7 +17,7 @@
[![npm version](https://img.shields.io/npm/v/@opendirectory.dev/skills.svg?style=flat-square)](https://www.npmjs.com/package/@opendirectory.dev/skills) -[![Skills](https://img.shields.io/badge/skills-60-blue.svg?style=flat-square)](skills/) +[![Skills](https://img.shields.io/badge/skills-61-blue.svg?style=flat-square)](skills/) [![Stars](https://img.shields.io/github/stars/Varnan-Tech/opendirectory?style=flat-square&color=yellow)](https://github.com/Varnan-Tech/opendirectory/stargazers) [![Contributors](https://img.shields.io/github/contributors/Varnan-Tech/opendirectory?style=flat-square&color=orange)](https://github.com/Varnan-Tech/opendirectory/graphs/contributors) [![Agents](https://img.shields.io/badge/agents-8-blueviolet.svg?style=flat-square)](#quick-start) @@ -44,7 +44,7 @@ Or list all skills: ```bash npx "@opendirectory.dev/skills" list ``` -*60 specialized skills across GTM, growth, and developer tooling* +*61 specialized skills across GTM, growth, and developer tooling* ### 2. Pick your agent ```bash @@ -248,7 +248,7 @@ Manus AI users can import a skill directly from its OpenDirectory skill page. Th ## All Skills -60 skills across GTM, growth automation, technical marketing, and developer tooling. +61 skills across GTM, growth automation, technical marketing, and developer tooling. diff --git a/packages/cli/registry.json b/packages/cli/registry.json index 702f13d..66e3703 100644 --- a/packages/cli/registry.json +++ b/packages/cli/registry.json @@ -588,4 +588,4 @@ "version": "0.0.1", "path": "skills/yc-intent-radar-skill" } -] +] \ No newline at end of file diff --git a/skills/geo-gap-fixer/README.md b/skills/geo-gap-fixer/README.md index 77b45fd..66074c7 100644 --- a/skills/geo-gap-fixer/README.md +++ b/skills/geo-gap-fixer/README.md @@ -9,26 +9,60 @@ In 2026, buyers discover tools by asking ChatGPT, Claude, Gemini, and Perplexity > One skill run replaces a $200/month AI-monitoring SaaS subscription. ---- + +## Install + +### Option A: npx CLI (Recommended) -## Quickstart +No global install. Always runs the latest version. -**1. Clone and Configure** ```bash -cp .env.example .env -cp config.example.json config.json +npx "@opendirectory.dev/skills" install geo-gap-fixer --target claude ``` -Edit `config.json` with your brand, competitors, and category. Edit `.env` with at least 2 API keys (`OPENAI_API_KEY`, `ANTHROPIC_API_KEY`, `GOOGLE_API_KEY`, `PERPLEXITY_API_KEY`). -**2. Run the Pipeline** +### Option B: Claude Desktop App + + + +**Step 1: Download the skill from GitHub** + +1. Copy the URL of this specific skill folder from your browser's address bar. +2. Go to [download-directory.github.io](https://download-directory.github.io/). +3. Paste the URL and click **Enter** to download. + +**Step 2: Install in Claude** + +1. Open your **Claude desktop app**. +2. Go to the sidebar on the left side and click on the **Customize** section. +3. Click on the **Skills** tab, then click on the **+** button to create a new skill. +4. Choose **Upload a skill**, then drag and drop the `.zip` file or extracted folder. + +> **Note:** For some skills, the `SKILL.md` file might be located inside a subfolder. Always upload the specific folder that contains the `SKILL.md` file. + +### Option C: Claude Code Native + +Run these commands inside Claude Code: + ```bash -python scripts/probe_llms.py # Probes LLM APIs -python scripts/analyze_results.py # Analyzes mentions & sentiment -python scripts/build_report.py # Generates the GEO report +/plugin marketplace add Varnan-Tech/opendirectory +/plugin install opendirectory-gtm-skills@opendirectory-marketplace ``` -**3. Read the Report** -Open `report/geo_audit_report.md` for your action plan. +### Option D: Manus AI + + + +[**Install in Manus AI**](https://manus.im/import-skills?githubUrl=https%3A%2F%2Fgithub.com%2FVarnan-Tech%2Fopendirectory%2Ftree%2Fmain%2Fskills%2Fgeo-gap-fixer&utm_source=opendirectory) + +Manus AI users can import a skill directly from its OpenDirectory skill page. This is the easiest path when you want Manus to pull the skill from GitHub for you. + +1. Open the skill you want from the [opendirectory homepage](https://opendirectory.dev). +2. In the install panel, select the **Manus AI** tab. +3. Click **Install in Manus AI** - this opens Manus with the skill GitHub URL already attached. +4. Confirm the import inside Manus AI. + +> If your Manus workspace prefers file uploads, use the **Download** tab instead and upload the downloaded `.skill.zip` file inside Manus. + --- diff --git a/skills/geo-gap-fixer/scripts/analyze_results.py b/skills/geo-gap-fixer/scripts/analyze_results.py index 79d3614..b152164 100644 --- a/skills/geo-gap-fixer/scripts/analyze_results.py +++ b/skills/geo-gap-fixer/scripts/analyze_results.py @@ -47,9 +47,10 @@ "frustrating", "buggy", "unreliable", "overpriced", ] -# Pre-compile word-boundary patterns for sentiment keywords -_POS_PATTERNS = [re.compile(r'\b' + re.escape(kw) + r'\b') for kw in POSITIVE_KEYWORDS] -_NEG_PATTERNS = [re.compile(r'\b' + re.escape(kw) + r'\b') for kw in NEGATIVE_KEYWORDS] +# Pre-compile lookaround patterns for sentiment keywords +# Use lookarounds instead of \b so names with non-word chars (e.g. "C++") work +_POS_PATTERNS = [re.compile(r'(? dict[str, list[int]]: """ Find all mentions of each name in the text. - Returns {name: [char_positions]} using word-boundary matching. + Returns {name: [char_positions]} using lookaround matching. + Uses (? dict: """ Compute sentiment for a specific brand/name within the text. Uses keyword proximity scoring: look within ±200 chars of each mention. + Counts keyword frequency via findall() for more accurate weighted scores. """ text_lower = text.lower() - pattern = re.compile(r'\b' + re.escape(name.lower()) + r'\b') + pattern = re.compile(r'(? dict: context = " ".join(context_chars) - pos_found = [POSITIVE_KEYWORDS[i] for i, p in enumerate(_POS_PATTERNS) if p.search(context)] - neg_found = [NEGATIVE_KEYWORDS[i] for i, p in enumerate(_NEG_PATTERNS) if p.search(context)] - - total = len(pos_found) + len(neg_found) + # Use findall() for frequency-weighted scoring instead of binary search() + pos_found = [] + pos_count = 0 + for i, p in enumerate(_POS_PATTERNS): + hits = len(p.findall(context)) + if hits > 0: + pos_found.append(POSITIVE_KEYWORDS[i]) + pos_count += hits + + neg_found = [] + neg_count = 0 + for i, p in enumerate(_NEG_PATTERNS): + hits = len(p.findall(context)) + if hits > 0: + neg_found.append(NEGATIVE_KEYWORDS[i]) + neg_count += hits + + total = pos_count + neg_count if total == 0: score = 0.0 label = "neutral" else: - score = round((len(pos_found) - len(neg_found)) / total, 2) + score = round((pos_count - neg_count) / total, 2) if score > 0.2: label = "positive" elif score < -0.2: @@ -144,9 +162,12 @@ def extract_citations(text: str) -> list[str]: match = re.match(r'https?://([^/\s?#]+)', url) if match: domain = match.group(1).lower() + # Strip trailing punctuation that the URL regex may have captured + domain = domain.rstrip('.!?,;:') # Remove common prefixes domain = re.sub(r'^www\.', '', domain) - domains.add(domain) + if domain: # guard against empty string after stripping + domains.add(domain) return sorted(domains) @@ -156,7 +177,7 @@ def extract_framing(text: str, name: str) -> list[str]: Captures adjectives and short phrases within ±60 chars. """ text_lower = text.lower() - pattern = re.compile(r'\b' + re.escape(name.lower()) + r'\b') + pattern = re.compile(r'(? str: # ── Provider Adapters ─────────────────────────────────────────────────────── +def _is_transient(exc: Exception) -> bool: + """Check if an exception represents a transient (retryable) error.""" + status = getattr(exc, 'status_code', None) or getattr(exc, 'status', None) + # Some SDKs use 'code' or nest status in response + if status is None: + resp = getattr(exc, 'response', None) + if resp is not None: + status = getattr(resp, 'status_code', None) or getattr(resp, 'status', None) + if isinstance(status, int): + return status == 429 or 500 <= status <= 599 + # ConnectionError, Timeout, etc. are transient + exc_name = type(exc).__name__.lower() + return any(k in exc_name for k in ('timeout', 'connection', 'temporary', 'unavailable')) + + def _retry(fn: Callable, provider_name: str) -> dict | None: - """Retry a provider call with exponential backoff on transient errors.""" + """Retry a provider call with exponential backoff on transient errors only. + + Permanent errors (e.g. 401 Unauthorized) propagate immediately + instead of wasting retry attempts. + """ for attempt in range(MAX_RETRIES + 1): - result = fn() - if result is not None: - return result - if attempt < MAX_RETRIES: - wait = RETRY_BACKOFF ** (attempt + 1) - print(f" → retrying {provider_name} in {wait:.0f}s (attempt {attempt + 2}/{MAX_RETRIES + 1})") - time.sleep(wait) + try: + result = fn() + if result is not None: + return result + # fn returned None without raising — treat as non-retryable + return None + except Exception as e: + if _is_transient(e) and attempt < MAX_RETRIES: + wait = RETRY_BACKOFF ** (attempt + 1) + print(f" → transient error ({e}); retrying {provider_name} in {wait:.0f}s (attempt {attempt + 2}/{MAX_RETRIES + 1})") + time.sleep(wait) + else: + print(f" [WARN] {provider_name} error (not retrying): {e}") + return None return None @@ -233,25 +259,21 @@ def probe_openai(prompt_text: str) -> dict | None: return None def _call() -> dict | None: - try: - response = _openai_client.chat.completions.create( - model="gpt-4o", - messages=[ - {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": prompt_text}, - ], - max_tokens=2048, - temperature=0.0, # Deterministic output - ) - text = response.choices[0].message.content or "" - return { - "provider": "openai", - "model": "gpt-4o", - "response": text[:MAX_RESPONSE_CHARS], - } - except Exception as e: - print(f" [WARN] OpenAI error: {e}") - return None + response = _openai_client.chat.completions.create( + model="gpt-4o", + messages=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": prompt_text}, + ], + max_tokens=2048, + temperature=0.0, # Deterministic output + ) + text = response.choices[0].message.content or "" + return { + "provider": "openai", + "model": "gpt-4o", + "response": text[:MAX_RESPONSE_CHARS], + } return _retry(_call, "openai") @@ -273,22 +295,18 @@ def probe_anthropic(prompt_text: str) -> dict | None: return None def _call() -> dict | None: - try: - response = _anthropic_client.messages.create( - model="claude-sonnet-4-6", - max_tokens=2048, - system=SYSTEM_PROMPT, - messages=[{"role": "user", "content": prompt_text}], - ) - text = response.content[0].text if response.content else "" - return { - "provider": "anthropic", - "model": "claude-sonnet-4-6", - "response": text[:MAX_RESPONSE_CHARS], - } - except Exception as e: - print(f" [WARN] Anthropic error: {e}") - return None + response = _anthropic_client.messages.create( + model="claude-sonnet-4-6", + max_tokens=2048, + system=SYSTEM_PROMPT, + messages=[{"role": "user", "content": prompt_text}], + ) + text = response.content[0].text if response.content else "" + return { + "provider": "anthropic", + "model": "claude-sonnet-4-6", + "response": text[:MAX_RESPONSE_CHARS], + } return _retry(_call, "anthropic") @@ -310,20 +328,16 @@ def probe_google(prompt_text: str) -> dict | None: return None def _call() -> dict | None: - try: - response = _google_client.models.generate_content( - model="gemini-2.5-flash", - contents=f"{SYSTEM_PROMPT}\n\n{prompt_text}", - ) - text = response.text or "" - return { - "provider": "google", - "model": "gemini-2.5-flash", - "response": text[:MAX_RESPONSE_CHARS], - } - except Exception as e: - print(f" [WARN] Google error: {e}") - return None + response = _google_client.models.generate_content( + model="gemini-2.5-flash", + contents=f"{SYSTEM_PROMPT}\n\n{prompt_text}", + ) + text = response.text or "" + return { + "provider": "google", + "model": "gemini-2.5-flash", + "response": text[:MAX_RESPONSE_CHARS], + } return _retry(_call, "google") @@ -348,23 +362,19 @@ def probe_perplexity(prompt_text: str) -> dict | None: return None def _call() -> dict | None: - try: - response = _perplexity_client.chat.completions.create( - model="sonar-pro", - messages=[ - {"role": "system", "content": SYSTEM_PROMPT}, - {"role": "user", "content": prompt_text}, - ], - ) - text = response.choices[0].message.content or "" - return { - "provider": "perplexity", - "model": "sonar-pro", - "response": text[:MAX_RESPONSE_CHARS], - } - except Exception as e: - print(f" [WARN] Perplexity error: {e}") - return None + response = _perplexity_client.chat.completions.create( + model="sonar-pro", + messages=[ + {"role": "system", "content": SYSTEM_PROMPT}, + {"role": "user", "content": prompt_text}, + ], + ) + text = response.choices[0].message.content or "" + return { + "provider": "perplexity", + "model": "sonar-pro", + "response": text[:MAX_RESPONSE_CHARS], + } return _retry(_call, "perplexity")