diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 033c47d..85a78a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -3,6 +3,7 @@ name: CI on: push: branches: [main] + tags: ['v*'] pull_request: branches: [main] @@ -10,8 +11,9 @@ jobs: build-and-test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - node-version: [20, 22] + node-version: [18, 20, 22] steps: - uses: actions/checkout@v7 @@ -31,9 +33,48 @@ jobs: - name: Test run: npm test + quality: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v7 + + - name: Use Node.js 22 + uses: actions/setup-node@v7 + with: + node-version: 22 + cache: 'npm' + + - name: Install dependencies + run: npm ci --legacy-peer-deps + + - name: Lint + run: npm run lint + - name: TypeScript check run: npx tsc --noEmit + - name: Verify frozen benchmark fixture + run: npm run benchmark:verify + + windows-build: + runs-on: windows-latest + + steps: + - uses: actions/checkout@v7 + + - name: Use Node.js 22 + uses: actions/setup-node@v7 + with: + node-version: 22 + cache: 'npm' + + - name: Install dependencies + run: npm ci --legacy-peer-deps + + - name: Build + run: npm run build + publish: if: startsWith(github.ref, 'refs/tags/v') runs-on: ubuntu-latest @@ -73,4 +114,4 @@ jobs: uses: softprops/action-gh-release@v2 with: body_path: CHANGELOG.md - generate_release_notes: true \ No newline at end of file + generate_release_notes: true diff --git a/.gitignore b/.gitignore index 23a7202..8c539ea 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ dist/ .pi/ .github/prompts coverage/ +benchmarks/fixtures/live-latest.json # Claude Code hooks (不要提交到 repo, 因为路径可能不同) .claude/hooks/ .claude/session_state/ diff --git a/AGENTS.md b/AGENTS.md index 0ec4230..e9efe24 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -9,19 +9,19 @@ tags: --- # Agent Search MCP — 多引擎统一搜索 MCP Server -一句话：11 引擎搜索（ddg/sogou/bing/baidu/brave/tavily/exa/wikipedia/startpage/yandex/mojeek），MCP 协议接入，**免费 + 多源验证 + Token 优化**。 +一句话：12 个搜索适配器（8 个零密钥 + 4 个可选 API），MCP 协议接入，**中文原生 + 多源聚合 + Token 可控**。 ## 当前阶段 **版本**: v3.1.0（已发布 npm + GitHub Release）— [查看完整路线图](docs/superpowers/plans/2026-07-22-iteration-roadmap.md) -**测试**: 438 passed, 38 files | **引擎**: 11 (8 免费, 3 付费) | **Python**: 可选（DDG 自动 HTML 回退） +**测试**: 510 passed, 43 files | **适配器**: 12（8 零密钥, 4 可选 API）| **Python**: 可选（DDG 自动 HTML 回退） 当前优先事项： -1. **Phase A: Agent UX** — `setupFetchTools` 拆分、MCP annotations、错误区分度 -2. **Phase C: 性能** — DDG News HTML 回退、lite.ddg 第三层回退 -3. **Phase D: 测试** — brave/tavily mock、E2E 集成测试、SSRF 安全测试 -4. **分发推广** — awesome-mcp-servers PR、掘金文章（持续） +1. **搜索质量证据** — 在稳定网络 runner 上捕获真实 fixture 并增加人工相关性标签 +2. **HTTP 部署指南** — Bearer 密钥轮换、Origin allowlist 与反向代理配置 +3. **信号校准** — 用真实失败查询持续校准 relevance/confidence/source_count +4. **分发推广** — 发布已校准口径的掘金/Reddit/V2EX 素材（持续） ## 常用命令 @@ -55,7 +55,7 @@ fasm extract "https://..." # CLI 提取 ## 架构 -`src/` 下按职责分层：`tools/`（MCP 工具定义）、`engines/`（11 引擎适配）、`aggregation/`（评分/去重/丰富）、`synthesis/`（结果合成）、`infrastructure/`（安全/缓存/限速）。Agent 自己探索 `src/` 目录获取最新结构。 +`src/` 下按职责分层：`tools/`（MCP 工具定义）、`engines/`（12 个引擎适配器）、`aggregation/`（评分/去重/丰富）、`synthesis/`（结果合成）、`infrastructure/`（安全/缓存/限速）。Agent 自己探索 `src/` 目录获取最新结构。 ## 编码规范 @@ -92,12 +92,16 @@ vitest，`tests/` 按功能目录组织。公共函数 + 新功能必须有测 - **Bing/Baidu 测试**: 实际搜索需要网络，单测用 mock 模拟 HTTP 响应 - **ddgs 依赖**: Python 库 `ddgs` 为可选依赖。未安装时 DDG 引擎自动回退到 Node.js HTML 引擎（cheerio 解析）。Docker 镜像不含 Python，仅使用 HTML 引擎。`isDdgsAvailable()` 检测可用性，结果缓存在进程生命周期内 -- **cheerio 依赖**: DuckDuckGo HTML 引擎依赖 cheerio（纯 JS，无 native binding），npm install 自动安装 +- **cheerio 依赖**: DuckDuckGo HTML 引擎依赖 cheerio（纯 JS，无 native binding）。必须固定在 `1.0.0` 以维持 Node 18 支持；Cheerio 1.2+ 要求 Node 20.18.1+ - **中文搜索**: Sogou + Baidu 专供中文搜索，不要用 Google Translate 翻译替代 - **请求合并**: 相同查询在 100ms 内自动合并，避免并发重复请求 - **Env 变量**: API key 通过环境变量传入，不走配置文件 - **npm publish**: 当前 registry 是腾讯镜像（mirrors.tencentyun.com），publish 前必须切到 registry.npmjs.org - **工具可见性**: `ENABLED_TOOLS` / `DISABLED_TOOLS` 环境变量控制 MCP 工具注册。`DISABLED_TOOLS` 优先级高于 `ENABLED_TOOLS`。默认全部启用。资源（capabilities/health）不受此策略影响。 +- **路由能力面**: 12 个适配器已统一进入 MCP / CLI / 瀑布路由；You.com 必须有 `YDC_API_KEY`，不要把“包内存在”与“当前凭证可用”混淆。 +- **Benchmark 口径**: 可保留 2026-07-24 历史 30 查询实测的 28.7% / 35.5% / 75%，但必须限定当时查询集和环境。当前冻结 fixture + `gpt-tokenizer` 用于可重现的格式化回归，不代表搜索质量。 +- **HTTP 安全默认值**: HTTP / both 模式必须配置 `HTTP_AUTH_TOKEN`；只有显式 `HTTP_ALLOW_UNAUTHENTICATED=true` 才允许无认证运行。带 Origin 的浏览器请求必须命中 `ALLOWED_ORIGINS`。 +- **stdio 日志**: stdout 只用于 MCP JSON-RPC。运行日志必须走 `logger`（stderr）或 `console.error`，禁止在服务路径使用 `console.log`。 ## 文档索引 diff --git a/CHANGELOG.md b/CHANGELOG.md index e9ba787..f1f60e2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,19 @@ tags: --- # Changelog +## Unreleased + +### Features + +- Unified all 12 search adapters across MCP, advanced search, CLI, and waterfall routing. +- Split result signals into `relevance`, normalized `confidence`, and independent `source_count`; retained `score` as a deprecated compatibility alias and mapped legacy `MIN_CONFIDENCE=2/3` values to source count. +- Added a capture/replay benchmark with production execution telemetry, frozen fixtures, locked `gpt-tokenizer`, and a CI regression gate. Historical 30-query measurements remain published with their environment scope. +- Secured HTTP MCP mode with required Bearer authentication and browser Origin allowlisting. Unauthenticated mode now requires explicit `HTTP_ALLOW_UNAUTHENTICATED=true`. + +### Documentation + +- Restored the historical 28.7% / 35.5% token and 75% engine-call measurements in README and promotion drafts with explicit query-set and environment boundaries. + ## v3.3.0 (2026-07-24) > **Headline: Semantic dedup + rerank via Model2Vec. <10ms latency. Optional, opt-in.** @@ -21,6 +34,23 @@ tags: - **Zero dependency by default**: Semantic features are OFF by default. No Python/model2vec required unless explicitly enabled. - **Graceful degradation**: If the Python bridge is unavailable (no model2vec installed, process crash, etc.), results pass through unchanged — no broken searches. +### 🔧 Fixes + +- **Restored zero-Python DDG fallback**: The search orchestrator no longer rejects DuckDuckGo before its Node.js HTML fallback can run. +- **Protected stdio JSON-RPC**: Circuit-breaker transitions now use the stderr logger instead of writing to stdout. +- **Closed CSDN SSRF path**: `fetch_csdn_article` now accepts only HTTPS `blog.csdn.net` URLs and rejects redirects. +- **Cross-platform build**: Replaced POSIX-only `mkdir`/`cp` commands with a Node.js build helper; `npm run build` now works on Windows. +- **CI coverage**: Restored Node.js 18/20/22 runtime coverage, added a Node.js 22 quality gate, disabled matrix fail-fast, and added a Windows build job. +- **Node.js 18 compatibility**: Pinned Cheerio to its Node 18-compatible release and close idle HTTP connections during shutdown. +- **Lint compatibility**: Pinned TypeScript to the supported 6.x API until `typescript-eslint` supports TypeScript 7. +- **Runtime metadata**: MCP initialization, HTTP health, and capabilities now report v3.1.3 / Apache-2.0 consistently with the published package. + +### 📚 Documentation + +- Replaced volatile competitor pricing claims with a capability-based comparison linked to official repositories. +- Marked historical benchmark percentages as exploratory until engine-call telemetry and frozen fixtures are implemented. +- Added a reusable English/Chinese promotion kit and rewrote the Juejin draft around verified capabilities. + ### 🔧 Env vars | Variable | Default | Description | @@ -34,8 +64,8 @@ tags: ### 📊 Stats -- **Tests**: 480 passing (+17: 6 semantic + 11 config) -- **Files**: 42 test files (+1: semantic.test.ts) +- **Tests**: 498 passing +- **Files**: 43 test files ## v3.2.0 (2026-07-24) diff --git a/HANDOVER.md b/HANDOVER.md index 097f902..759296a 100644 --- a/HANDOVER.md +++ b/HANDOVER.md @@ -1,7 +1,7 @@ --- type: HandoverDoc title: agent-search-mcp HANDOVER -timestamp: '2026-07-24T09:10:00+08:00' +timestamp: '2026-07-25T04:30:00+08:00' description: 会话日志和项目状态 tags: - agent-search-mcp @@ -11,15 +11,24 @@ tags: ## 项目状态 -**版本**: v3.3.0（P2 语义层已完成，待发布） -**引擎**: 11 个（ddg/sogou/bing/baidu/brave/tavily/exa/yandex/mojeek/wikipedia/startpage） -**测试**: vitest — 480 passed, 42 test files -**最后更新**: 2026-07-24 +**版本**: npm v3.1.3；main 含 v3.3.0 候选功能，尚未发布 +**引擎**: 12 个适配器；`free_search`/`free_search_advanced`/CLI/瀑布模式已全部统一路由 +**测试**: vitest — 510 passed, 43 test files +**最后更新**: 2026-07-25 **npm**: https://www.npmjs.com/package/agent-search-mcp **Python 依赖**: 可选（DDG 自动回退到 cheerio HTML 引擎；语义层需 `pip install model2vec`） ## 最近活动 +- [2026-07-25] ✅ 宣传层级定稿：“免费 + 省 Token”作为第一卖点，“Agent 搜索路由器”作为独特机制和长期路线 +- [2026-07-25] ✅ 统一 12 适配器路由，拆分 relevance/confidence/source_count 契约 +- [2026-07-25] ✅ Benchmark v3：真实执行遥测、冻结 fixture、锁定 tokenizer 与 CI 回归门禁 +- [2026-07-25] ✅ HTTP Bearer 认证 + Origin allowlist；无认证模式必须显式开启 + +- [2026-07-25] ✅ Node 18 兼容：Cheerio 固定到 1.0.0；HTTP 关闭时主动清理 keep-alive 空闲连接 +- [2026-07-25] ✅ CI 分层：Node 18/20/22 各自 build/test；Node 22 独立执行 lint/typecheck，矩阵不再 fail-fast +- [2026-07-25] ✅ 产品加固：DDG HTML fallback、stdio 日志隔离、CSDN SSRF 防护、Windows 构建 +- [2026-07-25] ✅ 市场口径校准：竞品对比改为能力矩阵，历史 benchmark 标为探索性，新增推广素材包 - [2026-07-24] ✅ P2 语义层：Model2Vec 语义去重 + 语义重排（SEMANTIC_DEDUP/SEMANTIC_RERANK，默认 off） - [2026-07-24] ✅ P0 渐进披露 + 置信度过滤（MAX_FULL_RESULTS/MIN_CONFIDENCE，compact 模式） - [2026-07-22] ✅ v3.1.1: Streamable HTTP + Capabilities 声明 + MCP annotations + EngineError + DDG News HTML 回退 @@ -36,10 +45,16 @@ tags: **已完成 (v3.1.1)**: A1/A2/A3 + C1 + D1/D2/D3 + B1/B2 — 全部绿色 ✅ -**下一阶段 (v3.2.0+)**: C2 第三层回退 / C3 引擎惰性加载 / O1 awesome-mcp-servers PR / O2 掘金文章 +**下一阶段**: + +1. 在稳定网络 runner 上捕获非空真实 fixture，并补人工相关性标签 +2. 在真实反向代理环境验收 Bearer 密钥轮换、Origin 策略和限流 +3. 合并加固分支后，按“Agent 搜索路由器”独特路线发布掘金文章和短帖素材 ## 已知限制 - **DDG HTML 限流**：POST 大量请求触发 HTTP 202，Python 路径不受此限制 -- **DDG News 无 HTML 回退**：News 搜索仅支持 Python 路径 - **无分页**：所有引擎目前只返回第一页结果 +- **Benchmark 边界**：冻结 fixture 只验证格式和 token 回归，暂无人工相关性标签；历史精确数字必须带当时查询集/环境限定 +- **HTTP 部署**：已有 Bearer/Origin 防护，但生产环境仍需 TLS、密钥轮换和反向代理限流 +- **依赖审计**：本次安装报告 5 项（1 low / 2 moderate / 2 high）；当前 runner 访问 npm audit endpoint 被 EACCES 拦截，未能刷新 advisory 明细。不要为清零审计而盲目降级 MCP/测试协议栈。 diff --git a/README.md b/README.md index 1023fbf..5ee4ec2 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # Agent Search MCP -> **12 search engines (8 free, zero API keys), one MCP server.** -> Chinese search via Sogou + Baidu. Multi-source verification with confidence scoring. Waterfall progressive search. Content extraction. `npx agent-search-mcp` is all you need. +> **Free, token-efficient web search for AI agents.** +> Start with eight zero-key sources. Spend less context through waterfall stopping and compact output. Route Chinese queries natively and escalate to optional commercial APIs only when needed. `npx agent-search-mcp` is all you need. [![npm version](https://img.shields.io/npm/v/agent-search-mcp)](https://www.npmjs.com/package/agent-search-mcp) [![npm downloads](https://img.shields.io/npm/dm/agent-search-mcp)](https://www.npmjs.com/package/agent-search-mcp) @@ -16,35 +16,32 @@ ## Why Agent Search MCP -Most MCP search servers wrap a single paid API — one engine, one bill. Agent Search MCP is built differently. +The immediate value is simple: **search without paying for an API, and return fewer tokens without blindly throwing away evidence**. The mechanism behind that promise is a search policy for **where to search, when to stop, how much context to spend, and what evidence to trust**. Agent Search MCP is that control layer: a local-first search router, not another single-backend search API. -| | Agent Search MCP | Tavily | Exa | Brave | -|---|:---:|:---:|:---:|:---:| -| **Price** | **$0** | ~$30/mo | $50/mo | ~$15/mo | -| **Free engines** | **8** | 0 | 0 | 1 (2K/mo cap) | -| **API key required** | No | Yes | Yes | Yes | -| **Multi-source verify** | ✅ 8 engines | ❌ | ❌ | ❌ | -| **Chinese search** | ✅ Sogou+Baidu | ❌ | ❌ | ❌ | -| **Self-hosted** | ✅ | ❌ | ❌ | ❌ | +The route is deliberate: **zero-key start → Chinese-native routing → inspectable multi-source evidence → token-aware progressive search → optional commercial escalation**. The 12 adapters serve this route; adapter count is not the product story by itself. -### Free, forever +| | Agent Search MCP | [Tavily](https://github.com/tavily-ai/tavily-mcp) | [Exa](https://github.com/exa-labs/exa-mcp-server) | [Brave](https://github.com/brave/brave-search-mcp-server) | +|---|:---:|:---:|:---:|:---:| +| **Search without account/API key** | **Yes** | No | No | No | +| **Search backends** | **8 zero-key + 4 optional APIs** | Tavily API | Exa index | Brave index | +| **Cross-engine aggregation** | **Yes** | Single upstream | Single upstream | Single upstream | +| **Dedicated Chinese engines** | **Sogou + Baidu** | No | No | No | +| **Local MCP server** | Yes | Yes | Yes | Yes | +| **Best fit** | Zero-key, multilingual verification | Hosted search/extract/map/crawl | Semantic, code, and company research | Independent index + vertical search | -8 engines work with zero configuration — DuckDuckGo, Sogou, Bing, Baidu, Wikipedia, Startpage, Yandex, Mojeek. No API keys, no signup, no credit card. At 100 searches/day, that saves $30–50/month vs paid alternatives. +Comparison last checked 2026-07-25 against the linked official repositories. Commercial services have useful free allowances, but still require an account or credential; pricing changes, so this table intentionally avoids monthly-price claims. -### 75% fewer engine calls +### Zero-key by default -Benchmarked on [30 diverse queries](./benchmarks/) (15 English + 15 Chinese): **100% satisfied at waterfall phase 1** with just 2 engines. A naive multi-search would call all 8 every time. The waterfall stops early when confidence is sufficient — fewer calls, less latency, less data fed to the LLM. +Eight adapters need no credentials — DuckDuckGo, Sogou, Bing, Baidu, Wikipedia, Startpage, Yandex, and Mojeek. Brave, Tavily, Exa, and You.com remain optional when you want their APIs. -### Multi-source verification +### Progressive multi-source search -DDG and Sogou return **zero-overlap result sets** — you get genuinely broader coverage than any single engine can provide. Each result is scored 1–3 based on how many sources agree. Confidence-2+ results have been cross-checked by multiple engines. +Parallel and waterfall orchestration, URL/title deduplication, ranking, and graceful fallback are built in. Compact mode supports progressive disclosure so agents can inspect the top results first and call `free_extract` only when deeper content is needed. -### Token-efficient architecture +### Token control is a product feature -Two layers of savings, benchmarked on 30 EN+ZH queries: -- **Waterfall stops early** → 2 engines instead of 8 → **75% fewer engine calls** -- **Compact mode** → strips metadata noise + progressive disclosure → **28.7% fewer tokens** (1582 → 1128) -- **Compact Aggressive** (`SNIPPET_LENGTH=120`) → **35.5% fewer tokens** (1582 → 1020) +`OUTPUT_STYLE=compact`, `MAX_FULL_RESULTS`, `SNIPPET_LENGTH`, `MIN_CONFIDENCE`, and `MIN_SOURCE_COUNT` let operators trade context size against detail. A historical 30-query live run measured **28.7% fewer tokens in Compact mode**, **35.5% in Compact+**, and **75% fewer engine calls than naive eight-engine fan-out**. These are scoped measurements for that query set and environment, not universal guarantees. The new frozen-fixture replay uses an in-process tokenizer and currently measures 30.2% / 33.9% formatting savings reproducibly. ### Native Chinese search @@ -123,6 +120,8 @@ mcp_servers: ## Engines +The package contains 12 engine adapters, all selectable through `free_search`, `free_search_advanced`, the CLI, and waterfall routing. Eight work without credentials; Brave, Tavily, Exa, and You.com are enabled when their API keys are present. + | Engine | Free | Strengths | |--------|:----:|-----------| | **DuckDuckGo** | ✅ | Privacy-focused, English web | @@ -173,7 +172,11 @@ All tools are read-only and idempotent with MCP 2025 annotations. | `OUTPUT_STYLE` | `normal` | `compact` for token-optimized output | | `SNIPPET_LENGTH` | `200` | Max snippet characters (60–500) | | `MAX_FULL_RESULTS` | `3` | Full results before compacting (compact mode) | -| `MIN_CONFIDENCE` | `0` | Confidence threshold filter (0.0–3.0) | +| `MIN_CONFIDENCE` | `0` | Confidence threshold filter (0.0–1.0); legacy values 2–3 map to source count | +| `MIN_SOURCE_COUNT` | `1` | Minimum number of independent engine sources (1–12) | +| `HTTP_AUTH_TOKEN` | — | Bearer token required by HTTP mode | +| `HTTP_ALLOW_UNAUTHENTICATED` | `false` | Explicitly opt out of HTTP authentication (trusted local networks only) | +| `ALLOWED_ORIGINS` | — | Comma-separated browser origins allowed to call HTTP endpoints | | `SEMANTIC_DEDUP` | `false` | Semantic dedup via Model2Vec (requires `pip install model2vec`) | | `DEDUP_THRESHOLD` | `0.85` | Cosine similarity threshold for semantic dedup | | `SEMANTIC_RERANK` | `false` | Semantic rerank via Model2Vec | @@ -193,6 +196,10 @@ DISABLED_TOOLS=free_extract,fetch_github_readme `DISABLED_TOOLS` takes priority over `ENABLED_TOOLS`. +### HTTP deployment + +HTTP mode is secure-by-default: `/mcp` requires `Authorization: Bearer `, and browser requests with an `Origin` header must match `ALLOWED_ORIGINS`. Keep `/health` available for probes, terminate TLS at a trusted reverse proxy, and rotate the token as a secret. See the [HTTP deployment guide](./docs/http-deployment.md). + ### Engine Filtering ```bash @@ -215,28 +222,17 @@ fasm search "query" --count 5 --engines bing,baidu,youcom --json fasm extract "https://example.com" fasm extract "https://example.com" --json -# HTTP server -fasm serve --port 8080 +# HTTP server (Bearer auth is required for MCP HTTP mode) +HTTP_AUTH_TOKEN=change-me MODE=http npx agent-search-mcp ``` --- ## Benchmark -Benchmarked on [30 queries](./benchmarks/queries.json) (15 EN + 15 ZH, covering tech, news, and general knowledge) with default config, no API keys. - -| Metric | Result | -|--------|--------| -| **Success rate** | **30/30 (100%)** | -| **Waterfall efficiency** | **100%** stopped at phase 1 | -| **Avg engines per query** | **2.0** (vs 8 in naive = **75% fewer calls**) | -| **Multi-source diversity** | **0% URL overlap** DDG vs Sogou | -| **Avg confidence** | 0.64 / 1.0 | -| **Avg latency** | 15.2s (P50: 14.8s, P95: 18.4s) | -| **Token savings (compact)** | **28.7%** (1582 → 1128 tokens) | -| **Token savings (aggressive)** | **35.5%** (1582 → 1020 tokens) | +The benchmark has two evidence tracks. The historical 2026-07-24 live run covers 30 EN/ZH queries and measured 28.7% Compact, 35.5% Compact+, and 75% fewer calls versus naive eight-engine fan-out. Its raw responses were not frozen, so those figures remain historical environment-scoped measurements. The current runner captures actual execution telemetry and replays a frozen fixture through every output style with locked `gpt-tokenizer`; CI verifies the expected summary (currently 30.2% / 33.9%). Neither track is a cross-product quality ranking or a production guarantee. -→ [Full methodology & reports](./benchmarks/) +→ [Methodology, queries, limitations, and reports](./benchmarks/) --- @@ -277,6 +273,8 @@ npm run dev # stdio mode npm run dev:http # HTTP mode (port 3000) ``` +The build helper is cross-platform; CI checks Node.js 18/20/22 on Linux and performs a Windows build smoke test. + --- ## License diff --git a/README_zh.md b/README_zh.md index a20786f..0f1501c 100644 --- a/README_zh.md +++ b/README_zh.md @@ -1,7 +1,7 @@ # Agent Search MCP -> **12 个搜索引擎（8 个免费，无需 API Key），一个 MCP Server。** -> 搜狗 + 百度原生中文搜索。多源交叉验证 + 置信度评分。瀑布式渐进搜索。内容提取。`npx agent-search-mcp` 即可使用。 +> **给 AI Agent 的免费、省 Token 网页搜索。** +> 8 个零密钥来源直接起步；用瀑布停止和紧凑输出减少上下文；中文查询原生路由，只在需要时升级商业 API。`npx agent-search-mcp` 即可使用。 [![npm version](https://img.shields.io/npm/v/agent-search-mcp)](https://www.npmjs.com/package/agent-search-mcp) [![npm downloads](https://img.shields.io/npm/dm/agent-search-mcp)](https://www.npmjs.com/package/agent-search-mcp) @@ -16,35 +16,32 @@ ## 为什么选择 Agent Search MCP -大多数 MCP 搜索服务器只是封装单个付费 API — 一个引擎，一张账单。Agent Search MCP 架构完全不同。 +用户第一眼能感受到的价值很直接：**不买搜索 API 也能搜，不盲目丢证据也能少花 token**。支撑这两个卖点的机制，是一套决定**去哪里搜、什么时候停、花多少上下文、哪些证据值得信**的搜索策略。Agent Search MCP 做的就是这个控制层。 -| | Agent Search MCP | Tavily | Exa | Brave | -|---|:---:|:---:|:---:|:---:| -| **价格** | **$0** | ~$30/月 | $50/月 | ~$15/月 | -| **免费引擎数** | **8** | 0 | 0 | 1（2K/月封顶） | -| **需要 API Key** | 不需要 | 需要 | 需要 | 需要 | -| **多源验证** | ✅ 8 引擎交叉验证 | ❌ | ❌ | ❌ | -| **中文搜索** | ✅ 搜狗+百度 | ❌ | ❌ | ❌ | -| **自托管** | ✅ | ❌ | ❌ | ❌ | +这条路线是：**零密钥起步 → 中文原生路由 → 多源证据可检查 → 按 token 预算渐进搜索 → 必要时升级商业 API**。12 个适配器是这条路线的实现，适配器数量本身不是产品故事。 -### 完全免费 +| | Agent Search MCP | [Tavily](https://github.com/tavily-ai/tavily-mcp) | [Exa](https://github.com/exa-labs/exa-mcp-server) | [Brave](https://github.com/brave/brave-search-mcp-server) | +|---|:---:|:---:|:---:|:---:| +| **无需账号/API Key 搜索** | **是** | 否 | 否 | 否 | +| **搜索后端** | **8 个零密钥 + 4 个可选 API** | Tavily API | Exa 索引 | Brave 索引 | +| **跨引擎聚合** | **支持** | 单一上游 | 单一上游 | 单一上游 | +| **专用中文引擎** | **搜狗 + 百度** | 无 | 无 | 无 | +| **本地 MCP Server** | 支持 | 支持 | 支持 | 支持 | +| **最适合** | 零密钥、多语种互补搜索 | 托管搜索/提取/Map/Crawl | 语义、代码、企业研究 | 独立索引 + 垂直搜索 | -8 个引擎零配置即可使用 — DuckDuckGo、搜狗、Bing、百度、Wikipedia、Startpage、Yandex、Mojeek。无需 API Key，无需注册，无需信用卡。按每天 100 次搜索算，比付费方案省 $30–50/月。 +对比信息于 2026-07-25 按上述官方仓库核对。商业服务也提供免费额度，但仍需要账号或凭证；价格变化快，因此这里不再使用容易过期的月费对比。 -### 减少 75% 引擎调用 +### 默认零密钥 -基于 [30 条多样化查询](./benchmarks/)（15 条英文 + 15 条中文）的基准测试：**100% 在瀑布阶段一即满足要求**，仅需 2 个引擎。简单多引擎搜索每次都调全部 8 个引擎。瀑布式搜索在置信度足够时提前停止 — 更少调用、更低延迟、更少数据喂给 LLM。 +8 个适配器无需凭证：DuckDuckGo、搜狗、Bing、百度、Wikipedia、Startpage、Yandex、Mojeek。需要商业 API 时，可选启用 Brave、Tavily、Exa 和 You.com。 -### 多源交叉验证 +### 渐进式多源搜索 -DDG 和搜狗返回**零重叠结果集** — 覆盖面真正比单一引擎更广。每个结果经多源比对后给出 1–3 分置信度评分。置信度 ≥2 的结果经过至少 2 个引擎交叉验证。 +内置并行/瀑布编排、URL/标题去重、排序与自动降级。Compact 模式支持渐进披露，让 Agent 先看高优先级结果，只在需要时调用 `free_extract` 深挖正文。 -### Token 省在架构里 +### Token 控制是产品能力 -两层节省机制，基于 30 条 EN+ZH 查询的基准测试： -- **瀑布提前停止** → 调 2 个而非 8 个引擎 → **减少 75% 引擎调用** -- **Compact 模式** → 去除元数据噪声 + 渐进式披露 → **减少 28.7% token**（1582 → 1128） -- **Compact Aggressive**（`SNIPPET_LENGTH=120`）→ **减少 35.5% token**（1582 → 1020） +通过 `OUTPUT_STYLE=compact`、`MAX_FULL_RESULTS`、`SNIPPET_LENGTH`、`MIN_CONFIDENCE`和 `MIN_SOURCE_COUNT`，使用者可在上下文体积和信息细节间主动取舍。历史 30 查询真实运行实测了 **Compact 节省 28.7% token**、**Compact+ 节省 35.5%**，以及相比 8 引擎全并发 **少 75% 引擎调用**。这是特定查询集和环境的实测，不是通用保证。新的冻结 fixture 回放使用进程内 tokenizer，当前可重现 30.2% / 33.9% 的格式化节省。 ### 原生中文搜索 @@ -123,6 +120,8 @@ mcp_servers: ## 搜索引擎 +包内包含 12 个引擎适配器，现在均可从 `free_search`、`free_search_advanced`、CLI 和瀑布路由选择。8 个零密钥引擎直接可用；Brave、Tavily、Exa 和 You.com 在配置 API Key 后启用。 + | 引擎 | 免费 | 优势 | |------|:----:|------| | **DuckDuckGo** | ✅ | 隐私保护，英文搜索 | @@ -173,7 +172,11 @@ mcp_servers: | `OUTPUT_STYLE` | `normal` | `compact` 开启 token 优化输出 | | `SNIPPET_LENGTH` | `200` | 摘要最大字符数（60–500） | | `MAX_FULL_RESULTS` | `3` | compact 模式下的完整结果数 | -| `MIN_CONFIDENCE` | `0` | 置信度过滤阈值（0.0–3.0） | +| `MIN_CONFIDENCE` | `0` | 置信度阈值（0.0–1.0）；历史值 2–3 自动映射为来源数 | +| `MIN_SOURCE_COUNT` | `1` | 最少独立引擎来源数（1–12） | +| `HTTP_AUTH_TOKEN` | — | HTTP 模式必需的 Bearer Token | +| `HTTP_ALLOW_UNAUTHENTICATED` | `false` | 显式关闭 HTTP 认证（仅限受信任本地网络） | +| `ALLOWED_ORIGINS` | — | 允许访问 HTTP 端点的浏览器 Origin，逗号分隔 | | `SEMANTIC_DEDUP` | `false` | 语义去重（需 `pip install model2vec`） | | `DEDUP_THRESHOLD` | `0.85` | 语义去重的余弦相似度阈值 | | `SEMANTIC_RERANK` | `false` | 语义重排（需 `pip install model2vec`） | @@ -193,6 +196,10 @@ DISABLED_TOOLS=free_extract,fetch_github_readme `DISABLED_TOOLS` 优先级高于 `ENABLED_TOOLS`。 +### HTTP 部署 + +HTTP 模式采用安全默认值：`/mcp` 要求 `Authorization: Bearer `，带 `Origin` 请求头的浏览器请求必须命中 `ALLOWED_ORIGINS`。`/health` 保留给健康检查；生产环境应在受信反向代理终止 TLS，并把 Token 当作密钥轮换。详见 [HTTP 部署指南](./docs/http-deployment.md)。 + ### 引擎过滤 ```bash @@ -215,28 +222,17 @@ fasm search "关键词" --count 5 --engines bing,baidu,youcom --json fasm extract "https://example.com" fasm extract "https://example.com" --json -# HTTP 服务 -fasm serve --port 8080 +# HTTP MCP 服务（默认要求 Bearer 认证） +HTTP_AUTH_TOKEN=change-me MODE=http npx agent-search-mcp ``` --- ## 基准测试 -基于 [30 条查询](./benchmarks/queries.json)（15 EN + 15 ZH，覆盖技术、新闻、通用知识），默认配置，无 API Key。 - -| 指标 | 结果 | -|------|------| -| **成功率** | **30/30 (100%)** | -| **瀑布效率** | **100%** 在阶段一停止 | -| **平均引擎数** | **2.0**（对比 8 引擎全量 = **减少 75% 调用**）| -| **多源多样性** | DDG 与搜狗 **0% URL 重叠** | -| **平均置信度** | 0.64 / 1.0 | -| **平均延迟** | 15.2s（P50: 14.8s, P95: 18.4s）| -| **Token 节省（compact）** | **28.7%**（1582 → 1128 tokens）| -| **Token 节省（aggressive）** | **35.5%**（1582 → 1020 tokens）| +基准现在分两条证据线。2026-07-24 历史真实运行覆盖 30 条中英文查询，实测 Compact 28.7%、Compact+ 35.5%，以及相比 8 引擎全并发少 75% 调用；由于当时没有保存原始响应 fixture，它们仍是限定查询集和环境的历史实测。新 runner 记录真实执行遥测，并用锁定的 `gpt-tokenizer` 对同一冻结 fixture 进行三种输出回放；CI 校验当前 30.2% / 33.9% 的预期摘要。两者都不是跨产品质量排名或生产保证。 -→ [完整方法与报告](./benchmarks/) +→ [方法、查询、限制与报告](./benchmarks/) --- @@ -277,6 +273,8 @@ npm run dev # stdio 模式 npm run dev:http # HTTP 模式（端口 3000） ``` +构建脚本支持跨平台；CI 在 Linux 上覆盖 Node.js 18/20/22，并执行 Windows 构建冒烟测试。 + --- ## 许可证 diff --git a/benchmarks/README.md b/benchmarks/README.md index 9571abd..8b96feb 100644 --- a/benchmarks/README.md +++ b/benchmarks/README.md @@ -1,66 +1,57 @@ # Benchmarks -Reproducible benchmarks measuring search quality, engine efficiency, and token optimization. +Agent Search MCP keeps two complementary evidence tracks: a historical live-search measurement and a deterministic formatting regression benchmark. -## Latest Results (2026-07-24) +## Historical live result (2026-07-24) -30 queries (15 EN + 15 ZH), default config, no API keys. Token counts via character estimation (~1 token per 3 chars). +The original runner completed 30 queries (15 EN + 15 ZH), with no paid API keys. It measured: | Metric | Normal | Compact | Compact+ | -|--------|--------|---------|----------| -| **Success rate** | 100% | 100% | 96.7% | -| **Avg engines** | 2.0 | 2.0 | 2.0 | -| **Waterfall phase 1** | 100% | 100% | 100% | -| **Avg tokens** | 1582 | 1128 | 1020 | -| **Token savings vs Normal** | — | **28.7%** | **35.5%** | -| **Avg latency** | 15.2s | 16.6s | 16.2s | -| **P50 latency** | 14.8s | 16.3s | 15.9s | -| **P95 latency** | 18.4s | 19.9s | 19.3s | +|--------|-------:|--------:|---------:| +| Success rate | 100% | 100% | 96.7% | +| Average estimated tokens | 1582 | 1128 | 1020 | +| Savings vs Normal | — | **28.7%** | **35.5%** | +| Average latency | 15.2s | 16.6s | 16.2s | -→ [Full report](./reports/2026-07-24.md) · [JSON data](./reports/2026-07-24.json) +The run also reported two average engines per query, equivalent to **75% fewer calls than naive eight-engine fan-out**. These are real measurements for that query set, runner, and network environment. The raw responses were not frozen, the token fallback was `characters / 3`, and the output scenarios used separate live requests, so the exact figures are retained as scoped historical evidence rather than universal guarantees. -Key findings: -- **100% waterfall efficiency** — every query satisfied at phase 1 (2 engines). Saves 75% vs naive 8-engine search. -- **28.7% token reduction** with compact mode (progressive disclosure + metadata stripping) -- **35.5% token reduction** with aggressive settings (compact + 120-char snippets) -- **0% URL overlap** between DDG and Sogou — genuine multi-source diversity +[Full report](./reports/2026-07-24.md) · [JSON data](./reports/2026-07-24.json) -### Historical +## Reproducible fixture replay -| Date | Success | Tokens (Normal) | Compact Savings | Report | -|------|---------|-----------------|-----------------|--------| -| 2026-07-24 | 100% | 1582 | 28.7% | [report](./reports/2026-07-24.md) | -| 2026-07-23 | 100% | — | 6.7% (bytes) | [report](./reports/2026-07-23.md) | +The current runner replays the same frozen results through all output styles with the locked in-process `gpt-tokenizer`. CI compares the generated summary with the fixture's expected values. -## How to Run +| Metric | Normal | Compact | Compact+ | +|--------|-------:|--------:|---------:| +| Average tokens | 2122.0 | 1480.3 | 1401.7 | +| Savings vs Normal | — | **30.2%** | **33.9%** | + +This synthetic bilingual fixture verifies formatting, field semantics, and token-count regressions. It makes no search-quality or live engine-efficiency claim. ```bash -# Build first -npm run build +# Deterministic replay; fails if the expected summary changes +npm run benchmark:verify + +# Replay and write a report +npm run benchmark -# Run benchmarks (3 scenarios: normal, compact, compact+aggressive) -node benchmarks/run.cjs +# Capture real search responses and execution telemetry (network required) +npm run benchmark:capture ``` -**Optional: install tiktoken for precise token counts** (otherwise falls back to character estimation): +Live capture records searched engines, calls, completed phases, early stop, raw results, and tokenizer identity once per query. A captured fixture can then be replayed without network variance: ```bash -pip install tiktoken +node benchmarks/run.mjs --fixture benchmarks/fixtures/live-latest.json ``` -## Scenarios Tested - -| Scenario | OUTPUT_STYLE | MAX_FULL_RESULTS | SNIPPET_LENGTH | -|----------|-------------|-----------------|----------------| -| Normal | (default) | — | 200 | -| Compact | compact | 3 | 200 | -| Compact Aggressive | compact | 3 | 120 | - ## Contents | File | Description | |------|-------------| -| [`queries.json`](./queries.json) | 30 test queries (15 EN + 15 ZH, tech/news/general) | -| [`run.cjs`](./run.cjs) | Benchmark runner with optional tiktoken support | -| [`methodology.md`](./methodology.md) | Testing methodology | -| [`reports/`](./reports) | Published benchmark reports | +| [`queries.json`](./queries.json) | 30 bilingual live-search queries | +| [`run.mjs`](./run.mjs) | Current capture/replay runner | +| [`fixtures/format-regression.json`](./fixtures/format-regression.json) | Frozen deterministic regression fixture | +| [`methodology.md`](./methodology.md) | Evidence model and limitations | +| [`run.cjs`](./run.cjs) | Legacy historical runner | +| [`reports/`](./reports) | Historical and replay reports | diff --git a/benchmarks/fixtures/format-regression.json b/benchmarks/fixtures/format-regression.json new file mode 100644 index 0000000..567adfd --- /dev/null +++ b/benchmarks/fixtures/format-regression.json @@ -0,0 +1,1467 @@ +{ + "schema_version": 1, + "kind": "format-regression", + "captured_at": "2026-07-25T03:59:16.404Z", + "package_version": "3.1.3", + "query_set_sha256": "a65ec4034ca5eece226956d41f9b2642519c5685279616b9a5291ab709b6e08a", + "tokenizer": "gpt-tokenizer@3.4.0", + "zero_key_engine_baseline": 8, + "naive_engine_baseline": 8, + "samples": [ + { + "id": "format-1", + "query": "latest AI news 2026", + "language": "en", + "duration_ms": 0, + "response": { + "query": "latest AI news 2026", + "engines": [ + "duckduckgo", + "sogou", + "bing", + "baidu", + "wikipedia", + "startpage", + "yandex", + "mojeek" + ], + "results": [ + { + "title": "latest AI news 2026 — Reference result 1", + "url": "https://benchmark.example/1/1", + "snippet": "Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 1. Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 1.", + "confidence": 0.94, + "relevance": 0.91, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "latest AI news 2026 — Reference result 2", + "url": "https://benchmark.example/1/2", + "snippet": "Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 2. Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 2.", + "confidence": 0.905, + "relevance": 0.87, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "latest AI news 2026 — Reference result 3", + "url": "https://benchmark.example/1/3", + "snippet": "Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 3. Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 3.", + "confidence": 0.87, + "relevance": 0.83, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "latest AI news 2026 — Reference result 4", + "url": "https://benchmark.example/1/4", + "snippet": "Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 4. Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 4.", + "confidence": 0.835, + "relevance": 0.79, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "latest AI news 2026 — Reference result 5", + "url": "https://benchmark.example/1/5", + "snippet": "Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 5. Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 5.", + "confidence": 0.8, + "relevance": 0.75, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "latest AI news 2026 — Reference result 6", + "url": "https://benchmark.example/1/6", + "snippet": "Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 6. Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 6.", + "confidence": 0.765, + "relevance": 0.71, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "latest AI news 2026 — Reference result 7", + "url": "https://benchmark.example/1/7", + "snippet": "Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 7. Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 7.", + "confidence": 0.73, + "relevance": 0.67, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "latest AI news 2026 — Reference result 8", + "url": "https://benchmark.example/1/8", + "snippet": "Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 8. Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 8.", + "confidence": 0.695, + "relevance": 0.63, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "latest AI news 2026 — Reference result 9", + "url": "https://benchmark.example/1/9", + "snippet": "Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 9. Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 9.", + "confidence": 0.66, + "relevance": 0.59, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "latest AI news 2026 — Reference result 10", + "url": "https://benchmark.example/1/10", + "snippet": "Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 10. Frozen benchmark summary about “latest AI news 2026”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 10.", + "confidence": 0.625, + "relevance": 0.55, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + } + ], + "meta": { + "total": 10, + "high_confidence": 5, + "engines": [ + "duckduckgo", + "wikipedia", + "sogou" + ] + }, + "security_note": "Frozen format-regression fixture; content is synthetic and contains no search-quality claim." + } + }, + { + "id": "format-2", + "query": "TypeScript vs Rust performance comparison", + "language": "en", + "duration_ms": 0, + "response": { + "query": "TypeScript vs Rust performance comparison", + "engines": [ + "duckduckgo", + "sogou", + "bing", + "baidu", + "wikipedia", + "startpage", + "yandex", + "mojeek" + ], + "results": [ + { + "title": "TypeScript vs Rust performance comparison — Reference result 1", + "url": "https://benchmark.example/2/1", + "snippet": "Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 1. Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 1.", + "confidence": 0.94, + "relevance": 0.91, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "TypeScript vs Rust performance comparison — Reference result 2", + "url": "https://benchmark.example/2/2", + "snippet": "Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 2. Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 2.", + "confidence": 0.905, + "relevance": 0.87, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "TypeScript vs Rust performance comparison — Reference result 3", + "url": "https://benchmark.example/2/3", + "snippet": "Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 3. Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 3.", + "confidence": 0.87, + "relevance": 0.83, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "TypeScript vs Rust performance comparison — Reference result 4", + "url": "https://benchmark.example/2/4", + "snippet": "Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 4. Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 4.", + "confidence": 0.835, + "relevance": 0.79, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "TypeScript vs Rust performance comparison — Reference result 5", + "url": "https://benchmark.example/2/5", + "snippet": "Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 5. Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 5.", + "confidence": 0.8, + "relevance": 0.75, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "TypeScript vs Rust performance comparison — Reference result 6", + "url": "https://benchmark.example/2/6", + "snippet": "Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 6. Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 6.", + "confidence": 0.765, + "relevance": 0.71, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "TypeScript vs Rust performance comparison — Reference result 7", + "url": "https://benchmark.example/2/7", + "snippet": "Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 7. Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 7.", + "confidence": 0.73, + "relevance": 0.67, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "TypeScript vs Rust performance comparison — Reference result 8", + "url": "https://benchmark.example/2/8", + "snippet": "Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 8. Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 8.", + "confidence": 0.695, + "relevance": 0.63, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "TypeScript vs Rust performance comparison — Reference result 9", + "url": "https://benchmark.example/2/9", + "snippet": "Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 9. Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 9.", + "confidence": 0.66, + "relevance": 0.59, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "TypeScript vs Rust performance comparison — Reference result 10", + "url": "https://benchmark.example/2/10", + "snippet": "Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 10. Frozen benchmark summary about “TypeScript vs Rust performance comparison”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 10.", + "confidence": 0.625, + "relevance": 0.55, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + } + ], + "meta": { + "total": 10, + "high_confidence": 5, + "engines": [ + "duckduckgo", + "wikipedia", + "sogou" + ] + }, + "security_note": "Frozen format-regression fixture; content is synthetic and contains no search-quality claim." + } + }, + { + "id": "format-3", + "query": "MCP server best practices", + "language": "en", + "duration_ms": 0, + "response": { + "query": "MCP server best practices", + "engines": [ + "duckduckgo", + "sogou", + "bing", + "baidu", + "wikipedia", + "startpage", + "yandex", + "mojeek" + ], + "results": [ + { + "title": "MCP server best practices — Reference result 1", + "url": "https://benchmark.example/3/1", + "snippet": "Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 1. Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 1.", + "confidence": 0.94, + "relevance": 0.91, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "MCP server best practices — Reference result 2", + "url": "https://benchmark.example/3/2", + "snippet": "Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 2. Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 2.", + "confidence": 0.905, + "relevance": 0.87, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "MCP server best practices — Reference result 3", + "url": "https://benchmark.example/3/3", + "snippet": "Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 3. Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 3.", + "confidence": 0.87, + "relevance": 0.83, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "MCP server best practices — Reference result 4", + "url": "https://benchmark.example/3/4", + "snippet": "Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 4. Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 4.", + "confidence": 0.835, + "relevance": 0.79, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "MCP server best practices — Reference result 5", + "url": "https://benchmark.example/3/5", + "snippet": "Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 5. Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 5.", + "confidence": 0.8, + "relevance": 0.75, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "MCP server best practices — Reference result 6", + "url": "https://benchmark.example/3/6", + "snippet": "Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 6. Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 6.", + "confidence": 0.765, + "relevance": 0.71, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "MCP server best practices — Reference result 7", + "url": "https://benchmark.example/3/7", + "snippet": "Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 7. Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 7.", + "confidence": 0.73, + "relevance": 0.67, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "MCP server best practices — Reference result 8", + "url": "https://benchmark.example/3/8", + "snippet": "Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 8. Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 8.", + "confidence": 0.695, + "relevance": 0.63, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "MCP server best practices — Reference result 9", + "url": "https://benchmark.example/3/9", + "snippet": "Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 9. Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 9.", + "confidence": 0.66, + "relevance": 0.59, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "MCP server best practices — Reference result 10", + "url": "https://benchmark.example/3/10", + "snippet": "Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 10. Frozen benchmark summary about “MCP server best practices”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 10.", + "confidence": 0.625, + "relevance": 0.55, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + } + ], + "meta": { + "total": 10, + "high_confidence": 5, + "engines": [ + "duckduckgo", + "wikipedia", + "sogou" + ] + }, + "security_note": "Frozen format-regression fixture; content is synthetic and contains no search-quality claim." + } + }, + { + "id": "format-4", + "query": "Python async await tutorial", + "language": "en", + "duration_ms": 0, + "response": { + "query": "Python async await tutorial", + "engines": [ + "duckduckgo", + "sogou", + "bing", + "baidu", + "wikipedia", + "startpage", + "yandex", + "mojeek" + ], + "results": [ + { + "title": "Python async await tutorial — Reference result 1", + "url": "https://benchmark.example/4/1", + "snippet": "Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 1. Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 1.", + "confidence": 0.94, + "relevance": 0.91, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Python async await tutorial — Reference result 2", + "url": "https://benchmark.example/4/2", + "snippet": "Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 2. Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 2.", + "confidence": 0.905, + "relevance": 0.87, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Python async await tutorial — Reference result 3", + "url": "https://benchmark.example/4/3", + "snippet": "Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 3. Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 3.", + "confidence": 0.87, + "relevance": 0.83, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "Python async await tutorial — Reference result 4", + "url": "https://benchmark.example/4/4", + "snippet": "Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 4. Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 4.", + "confidence": 0.835, + "relevance": 0.79, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Python async await tutorial — Reference result 5", + "url": "https://benchmark.example/4/5", + "snippet": "Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 5. Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 5.", + "confidence": 0.8, + "relevance": 0.75, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Python async await tutorial — Reference result 6", + "url": "https://benchmark.example/4/6", + "snippet": "Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 6. Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 6.", + "confidence": 0.765, + "relevance": 0.71, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Python async await tutorial — Reference result 7", + "url": "https://benchmark.example/4/7", + "snippet": "Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 7. Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 7.", + "confidence": 0.73, + "relevance": 0.67, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "Python async await tutorial — Reference result 8", + "url": "https://benchmark.example/4/8", + "snippet": "Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 8. Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 8.", + "confidence": 0.695, + "relevance": 0.63, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Python async await tutorial — Reference result 9", + "url": "https://benchmark.example/4/9", + "snippet": "Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 9. Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 9.", + "confidence": 0.66, + "relevance": 0.59, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Python async await tutorial — Reference result 10", + "url": "https://benchmark.example/4/10", + "snippet": "Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 10. Frozen benchmark summary about “Python async await tutorial”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 10.", + "confidence": 0.625, + "relevance": 0.55, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + } + ], + "meta": { + "total": 10, + "high_confidence": 5, + "engines": [ + "duckduckgo", + "wikipedia", + "sogou" + ] + }, + "security_note": "Frozen format-regression fixture; content is synthetic and contains no search-quality claim." + } + }, + { + "id": "format-5", + "query": "Next.js 15 server components", + "language": "en", + "duration_ms": 0, + "response": { + "query": "Next.js 15 server components", + "engines": [ + "duckduckgo", + "sogou", + "bing", + "baidu", + "wikipedia", + "startpage", + "yandex", + "mojeek" + ], + "results": [ + { + "title": "Next.js 15 server components — Reference result 1", + "url": "https://benchmark.example/5/1", + "snippet": "Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 1. Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 1.", + "confidence": 0.94, + "relevance": 0.91, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Next.js 15 server components — Reference result 2", + "url": "https://benchmark.example/5/2", + "snippet": "Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 2. Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 2.", + "confidence": 0.905, + "relevance": 0.87, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Next.js 15 server components — Reference result 3", + "url": "https://benchmark.example/5/3", + "snippet": "Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 3. Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 3.", + "confidence": 0.87, + "relevance": 0.83, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "Next.js 15 server components — Reference result 4", + "url": "https://benchmark.example/5/4", + "snippet": "Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 4. Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 4.", + "confidence": 0.835, + "relevance": 0.79, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Next.js 15 server components — Reference result 5", + "url": "https://benchmark.example/5/5", + "snippet": "Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 5. Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 5.", + "confidence": 0.8, + "relevance": 0.75, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Next.js 15 server components — Reference result 6", + "url": "https://benchmark.example/5/6", + "snippet": "Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 6. Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 6.", + "confidence": 0.765, + "relevance": 0.71, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Next.js 15 server components — Reference result 7", + "url": "https://benchmark.example/5/7", + "snippet": "Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 7. Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 7.", + "confidence": 0.73, + "relevance": 0.67, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "Next.js 15 server components — Reference result 8", + "url": "https://benchmark.example/5/8", + "snippet": "Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 8. Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 8.", + "confidence": 0.695, + "relevance": 0.63, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Next.js 15 server components — Reference result 9", + "url": "https://benchmark.example/5/9", + "snippet": "Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 9. Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 9.", + "confidence": 0.66, + "relevance": 0.59, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Next.js 15 server components — Reference result 10", + "url": "https://benchmark.example/5/10", + "snippet": "Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 10. Frozen benchmark summary about “Next.js 15 server components”, used to verify progressive disclosure, field semantics, and token counts across environments. Result 10.", + "confidence": 0.625, + "relevance": 0.55, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + } + ], + "meta": { + "total": 10, + "high_confidence": 5, + "engines": [ + "duckduckgo", + "wikipedia", + "sogou" + ] + }, + "security_note": "Frozen format-regression fixture; content is synthetic and contains no search-quality claim." + } + }, + { + "id": "format-6", + "query": "2026年人工智能发展趋势", + "language": "zh", + "duration_ms": 0, + "response": { + "query": "2026年人工智能发展趋势", + "engines": [ + "duckduckgo", + "sogou", + "bing", + "baidu", + "wikipedia", + "startpage", + "yandex", + "mojeek" + ], + "results": [ + { + "title": "2026年人工智能发展趋势 — 参考结果 1", + "url": "https://benchmark.example/6/1", + "snippet": "这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 1。 这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 1。", + "confidence": 0.94, + "relevance": 0.91, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "2026年人工智能发展趋势 — 参考结果 2", + "url": "https://benchmark.example/6/2", + "snippet": "这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 2。 这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 2。", + "confidence": 0.905, + "relevance": 0.87, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "2026年人工智能发展趋势 — 参考结果 3", + "url": "https://benchmark.example/6/3", + "snippet": "这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 3。 这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 3。", + "confidence": 0.87, + "relevance": 0.83, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "2026年人工智能发展趋势 — 参考结果 4", + "url": "https://benchmark.example/6/4", + "snippet": "这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 4。 这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 4。", + "confidence": 0.835, + "relevance": 0.79, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "2026年人工智能发展趋势 — 参考结果 5", + "url": "https://benchmark.example/6/5", + "snippet": "这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 5。 这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 5。", + "confidence": 0.8, + "relevance": 0.75, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "2026年人工智能发展趋势 — 参考结果 6", + "url": "https://benchmark.example/6/6", + "snippet": "这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 6。 这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 6。", + "confidence": 0.765, + "relevance": 0.71, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "2026年人工智能发展趋势 — 参考结果 7", + "url": "https://benchmark.example/6/7", + "snippet": "这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 7。 这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 7。", + "confidence": 0.73, + "relevance": 0.67, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "2026年人工智能发展趋势 — 参考结果 8", + "url": "https://benchmark.example/6/8", + "snippet": "这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 8。 这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 8。", + "confidence": 0.695, + "relevance": 0.63, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "2026年人工智能发展趋势 — 参考结果 9", + "url": "https://benchmark.example/6/9", + "snippet": "这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 9。 这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 9。", + "confidence": 0.66, + "relevance": 0.59, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "2026年人工智能发展趋势 — 参考结果 10", + "url": "https://benchmark.example/6/10", + "snippet": "这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 10。 这是关于“2026年人工智能发展趋势”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 10。", + "confidence": 0.625, + "relevance": 0.55, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + } + ], + "meta": { + "total": 10, + "high_confidence": 5, + "engines": [ + "duckduckgo", + "wikipedia", + "sogou" + ] + }, + "security_note": "Frozen format-regression fixture; content is synthetic and contains no search-quality claim." + } + }, + { + "id": "format-7", + "query": "TypeScript 最佳实践", + "language": "zh", + "duration_ms": 0, + "response": { + "query": "TypeScript 最佳实践", + "engines": [ + "duckduckgo", + "sogou", + "bing", + "baidu", + "wikipedia", + "startpage", + "yandex", + "mojeek" + ], + "results": [ + { + "title": "TypeScript 最佳实践 — 参考结果 1", + "url": "https://benchmark.example/7/1", + "snippet": "这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 1。 这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 1。", + "confidence": 0.94, + "relevance": 0.91, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "TypeScript 最佳实践 — 参考结果 2", + "url": "https://benchmark.example/7/2", + "snippet": "这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 2。 这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 2。", + "confidence": 0.905, + "relevance": 0.87, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "TypeScript 最佳实践 — 参考结果 3", + "url": "https://benchmark.example/7/3", + "snippet": "这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 3。 这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 3。", + "confidence": 0.87, + "relevance": 0.83, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "TypeScript 最佳实践 — 参考结果 4", + "url": "https://benchmark.example/7/4", + "snippet": "这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 4。 这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 4。", + "confidence": 0.835, + "relevance": 0.79, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "TypeScript 最佳实践 — 参考结果 5", + "url": "https://benchmark.example/7/5", + "snippet": "这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 5。 这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 5。", + "confidence": 0.8, + "relevance": 0.75, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "TypeScript 最佳实践 — 参考结果 6", + "url": "https://benchmark.example/7/6", + "snippet": "这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 6。 这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 6。", + "confidence": 0.765, + "relevance": 0.71, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "TypeScript 最佳实践 — 参考结果 7", + "url": "https://benchmark.example/7/7", + "snippet": "这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 7。 这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 7。", + "confidence": 0.73, + "relevance": 0.67, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "TypeScript 最佳实践 — 参考结果 8", + "url": "https://benchmark.example/7/8", + "snippet": "这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 8。 这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 8。", + "confidence": 0.695, + "relevance": 0.63, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "TypeScript 最佳实践 — 参考结果 9", + "url": "https://benchmark.example/7/9", + "snippet": "这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 9。 这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 9。", + "confidence": 0.66, + "relevance": 0.59, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "TypeScript 最佳实践 — 参考结果 10", + "url": "https://benchmark.example/7/10", + "snippet": "这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 10。 这是关于“TypeScript 最佳实践”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 10。", + "confidence": 0.625, + "relevance": 0.55, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + } + ], + "meta": { + "total": 10, + "high_confidence": 5, + "engines": [ + "duckduckgo", + "wikipedia", + "sogou" + ] + }, + "security_note": "Frozen format-regression fixture; content is synthetic and contains no search-quality claim." + } + }, + { + "id": "format-8", + "query": "MCP 服务器开发教程", + "language": "zh", + "duration_ms": 0, + "response": { + "query": "MCP 服务器开发教程", + "engines": [ + "duckduckgo", + "sogou", + "bing", + "baidu", + "wikipedia", + "startpage", + "yandex", + "mojeek" + ], + "results": [ + { + "title": "MCP 服务器开发教程 — 参考结果 1", + "url": "https://benchmark.example/8/1", + "snippet": "这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 1。 这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 1。", + "confidence": 0.94, + "relevance": 0.91, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "MCP 服务器开发教程 — 参考结果 2", + "url": "https://benchmark.example/8/2", + "snippet": "这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 2。 这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 2。", + "confidence": 0.905, + "relevance": 0.87, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "MCP 服务器开发教程 — 参考结果 3", + "url": "https://benchmark.example/8/3", + "snippet": "这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 3。 这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 3。", + "confidence": 0.87, + "relevance": 0.83, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "MCP 服务器开发教程 — 参考结果 4", + "url": "https://benchmark.example/8/4", + "snippet": "这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 4。 这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 4。", + "confidence": 0.835, + "relevance": 0.79, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "MCP 服务器开发教程 — 参考结果 5", + "url": "https://benchmark.example/8/5", + "snippet": "这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 5。 这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 5。", + "confidence": 0.8, + "relevance": 0.75, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "MCP 服务器开发教程 — 参考结果 6", + "url": "https://benchmark.example/8/6", + "snippet": "这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 6。 这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 6。", + "confidence": 0.765, + "relevance": 0.71, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "MCP 服务器开发教程 — 参考结果 7", + "url": "https://benchmark.example/8/7", + "snippet": "这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 7。 这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 7。", + "confidence": 0.73, + "relevance": 0.67, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "MCP 服务器开发教程 — 参考结果 8", + "url": "https://benchmark.example/8/8", + "snippet": "这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 8。 这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 8。", + "confidence": 0.695, + "relevance": 0.63, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "MCP 服务器开发教程 — 参考结果 9", + "url": "https://benchmark.example/8/9", + "snippet": "这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 9。 这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 9。", + "confidence": 0.66, + "relevance": 0.59, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "MCP 服务器开发教程 — 参考结果 10", + "url": "https://benchmark.example/8/10", + "snippet": "这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 10。 这是关于“MCP 服务器开发教程”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 10。", + "confidence": 0.625, + "relevance": 0.55, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + } + ], + "meta": { + "total": 10, + "high_confidence": 5, + "engines": [ + "duckduckgo", + "wikipedia", + "sogou" + ] + }, + "security_note": "Frozen format-regression fixture; content is synthetic and contains no search-quality claim." + } + }, + { + "id": "format-9", + "query": "Python 异步编程入门", + "language": "zh", + "duration_ms": 0, + "response": { + "query": "Python 异步编程入门", + "engines": [ + "duckduckgo", + "sogou", + "bing", + "baidu", + "wikipedia", + "startpage", + "yandex", + "mojeek" + ], + "results": [ + { + "title": "Python 异步编程入门 — 参考结果 1", + "url": "https://benchmark.example/9/1", + "snippet": "这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 1。 这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 1。", + "confidence": 0.94, + "relevance": 0.91, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Python 异步编程入门 — 参考结果 2", + "url": "https://benchmark.example/9/2", + "snippet": "这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 2。 这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 2。", + "confidence": 0.905, + "relevance": 0.87, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Python 异步编程入门 — 参考结果 3", + "url": "https://benchmark.example/9/3", + "snippet": "这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 3。 这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 3。", + "confidence": 0.87, + "relevance": 0.83, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "Python 异步编程入门 — 参考结果 4", + "url": "https://benchmark.example/9/4", + "snippet": "这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 4。 这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 4。", + "confidence": 0.835, + "relevance": 0.79, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Python 异步编程入门 — 参考结果 5", + "url": "https://benchmark.example/9/5", + "snippet": "这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 5。 这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 5。", + "confidence": 0.8, + "relevance": 0.75, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Python 异步编程入门 — 参考结果 6", + "url": "https://benchmark.example/9/6", + "snippet": "这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 6。 这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 6。", + "confidence": 0.765, + "relevance": 0.71, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Python 异步编程入门 — 参考结果 7", + "url": "https://benchmark.example/9/7", + "snippet": "这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 7。 这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 7。", + "confidence": 0.73, + "relevance": 0.67, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "Python 异步编程入门 — 参考结果 8", + "url": "https://benchmark.example/9/8", + "snippet": "这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 8。 这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 8。", + "confidence": 0.695, + "relevance": 0.63, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Python 异步编程入门 — 参考结果 9", + "url": "https://benchmark.example/9/9", + "snippet": "这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 9。 这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 9。", + "confidence": 0.66, + "relevance": 0.59, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Python 异步编程入门 — 参考结果 10", + "url": "https://benchmark.example/9/10", + "snippet": "这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 10。 这是关于“Python 异步编程入门”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 10。", + "confidence": 0.625, + "relevance": 0.55, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + } + ], + "meta": { + "total": 10, + "high_confidence": 5, + "engines": [ + "duckduckgo", + "wikipedia", + "sogou" + ] + }, + "security_note": "Frozen format-regression fixture; content is synthetic and contains no search-quality claim." + } + }, + { + "id": "format-10", + "query": "Next.js 服务端组件详解", + "language": "zh", + "duration_ms": 0, + "response": { + "query": "Next.js 服务端组件详解", + "engines": [ + "duckduckgo", + "sogou", + "bing", + "baidu", + "wikipedia", + "startpage", + "yandex", + "mojeek" + ], + "results": [ + { + "title": "Next.js 服务端组件详解 — 参考结果 1", + "url": "https://benchmark.example/10/1", + "snippet": "这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 1。 这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 1。", + "confidence": 0.94, + "relevance": 0.91, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Next.js 服务端组件详解 — 参考结果 2", + "url": "https://benchmark.example/10/2", + "snippet": "这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 2。 这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 2。", + "confidence": 0.905, + "relevance": 0.87, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Next.js 服务端组件详解 — 参考结果 3", + "url": "https://benchmark.example/10/3", + "snippet": "这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 3。 这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 3。", + "confidence": 0.87, + "relevance": 0.83, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "Next.js 服务端组件详解 — 参考结果 4", + "url": "https://benchmark.example/10/4", + "snippet": "这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 4。 这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 4。", + "confidence": 0.835, + "relevance": 0.79, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Next.js 服务端组件详解 — 参考结果 5", + "url": "https://benchmark.example/10/5", + "snippet": "这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 5。 这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 5。", + "confidence": 0.8, + "relevance": 0.75, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Next.js 服务端组件详解 — 参考结果 6", + "url": "https://benchmark.example/10/6", + "snippet": "这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 6。 这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 6。", + "confidence": 0.765, + "relevance": 0.71, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Next.js 服务端组件详解 — 参考结果 7", + "url": "https://benchmark.example/10/7", + "snippet": "这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 7。 这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 7。", + "confidence": 0.73, + "relevance": 0.67, + "source_count": 1, + "sources": [ + "sogou" + ] + }, + { + "title": "Next.js 服务端组件详解 — 参考结果 8", + "url": "https://benchmark.example/10/8", + "snippet": "这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 8。 这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 8。", + "confidence": 0.695, + "relevance": 0.63, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + }, + { + "title": "Next.js 服务端组件详解 — 参考结果 9", + "url": "https://benchmark.example/10/9", + "snippet": "这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 9。 这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 9。", + "confidence": 0.66, + "relevance": 0.59, + "source_count": 2, + "sources": [ + "duckduckgo", + "wikipedia" + ] + }, + { + "title": "Next.js 服务端组件详解 — 参考结果 10", + "url": "https://benchmark.example/10/10", + "snippet": "这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 10。 这是关于“Next.js 服务端组件详解”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 10。", + "confidence": 0.625, + "relevance": 0.55, + "source_count": 1, + "sources": [ + "duckduckgo" + ] + } + ], + "meta": { + "total": 10, + "high_confidence": 5, + "engines": [ + "duckduckgo", + "wikipedia", + "sogou" + ] + }, + "security_note": "Frozen format-regression fixture; content is synthetic and contains no search-quality claim." + } + } + ], + "expected_summary": { + "successful_queries": 10, + "tokenizer": "gpt-tokenizer@3.4.0", + "average_tokens": { + "normal": 2122, + "compact": 1480.3, + "compact_aggressive": 1401.7 + }, + "savings_percent": { + "compact": 30.2, + "compact_aggressive": 33.9, + "waterfall_engine_calls": null + }, + "average_engine_calls": null + } +} diff --git a/benchmarks/methodology.md b/benchmarks/methodology.md index 079bf38..89f00e1 100644 --- a/benchmarks/methodology.md +++ b/benchmarks/methodology.md @@ -1,55 +1,40 @@ # Benchmark Methodology -## Overview +## Evidence model -This benchmark measures Agent Search MCP's real-world performance across four dimensions: -**search quality, engine efficiency, token optimization, and deduplication effectiveness.** +The benchmark separates live retrieval from deterministic output measurement. -## Query Set +1. **Capture:** each query invokes the production search orchestrator once and stores raw results plus execution telemetry. +2. **Replay:** normal, compact, and aggressive output styles format the same stored results. +3. **Count:** the locked `gpt-tokenizer` dependency counts serialized output tokens in-process. +4. **Check:** a fixture may contain an expected summary; `--check` fails on drift and is used by CI. -- **30 queries** (15 English, 15 Chinese) -- Mix of: technical queries, news queries, general knowledge -- Designed to represent real AI agent usage patterns +This design prevents network variance or different search results from being mistaken for formatting savings. -## Metrics - -| Metric | Definition | -|--------|-----------| -| **Success rate** | % of queries returning ≥1 result | -| **Latency (P50/P95)** | Total wall-clock time per query | -| **Avg engines/query** | Number of engines queried before waterfall stopped | -| **Waterfall saved** | Queries that stopped at phase 1 (≤2 engines) | -| **Avg confidence** | Mean confidence score across all results (0-1 scale) | -| **Dedup rate** | % of duplicate URLs removed | -| **Token savings** | % reduction vs estimated raw SERP text | - -## Waterfall Search +## Query and fixture sets -Agent Search MCP uses a 3-phase waterfall: -1. Phase 1: DDG + Sogou (parallel) -2. Phase 2: Bing + Baidu (only if phase 1 insufficient) -3. Phase 3: Brave + Tavily + Exa (paid, only if needed) +- `queries.json`: 30 live-search prompts, 15 English and 15 Chinese; primarily developer topics. +- `format-regression.json`: deterministic 10-query bilingual synthetic fixture used only for formatting and token regression. +- Live captures: environment-specific snapshots created with `--capture`; results with zero returned documents are excluded from savings summaries. -Early stopping at phase 1 means the waterfall works as designed. - -## Token Savings Estimation - -"Raw SERP" is estimated per result as: -``` -raw_chars = title_length + url_length + 300 (estimated snippet) -``` +## Metrics -"Compressed" is the actual JSON output size. This is a conservative estimate since raw SERP pages typically return much more content (full abstracts, dates, source metadata). +| Metric | Definition | +|--------|------------| +| Token count | `gpt-tokenizer` count of the serialized formatted response | +| Token savings | Difference between styles when replaying the same captured result set | +| Engine calls | Actual adapter attempts reported by the production orchestrator | +| Early stop | Whether waterfall routing stopped before exhausting its eligible phase | +| Success | Query returned at least one result | -## Environment +## Historical result boundary -- Node.js v20.x -- Default config (no Brave/Tavily/Exa API keys) -- All 8 free engines enabled -- Network: standard internet connection (no proxy) +The 2026-07-24 30-query report measured 28.7% Compact savings, 35.5% Compact+ savings, and 75% fewer calls than naive eight-engine fan-out. It remains useful measured evidence, but cannot be regenerated byte-for-byte because its raw responses were not saved and its token fallback used `characters / 3`. ## Limitations -- Latency varies by network conditions -- No paid engines tested (requires API keys) -- Quality is measured by confidence scores, not human relevance judging +- The deterministic fixture validates formatting, not retrieval quality. +- The live query set has no human relevance labels and is weighted toward technical topics. +- Engine availability, rate limits, latency, and returned pages vary with network, geography, and time. +- Paid adapters require their API keys and are omitted when unavailable. +- Historical and current replay percentages describe different fixtures and token counters; neither is a production guarantee. diff --git a/benchmarks/reports/2026-07-25-replay.json b/benchmarks/reports/2026-07-25-replay.json new file mode 100644 index 0000000..fd11588 --- /dev/null +++ b/benchmarks/reports/2026-07-25-replay.json @@ -0,0 +1,165 @@ +{ + "schema_version": 1, + "fixture": "benchmarks/fixtures/format-regression.json", + "fixture_sha256": "de1f34144eba54bb27e6778f97b7a0d9033e68226dcb73c1eb48cf7d176bb7bb", + "generated_at": "2026-07-25T04:10:18.036Z", + "scenarios": { + "normal": { + "style": "normal", + "snippetMax": 200 + }, + "compact": { + "style": "compact", + "snippetMax": 200, + "maxFullResults": 3 + }, + "compact_aggressive": { + "style": "compact", + "snippetMax": 120, + "maxFullResults": 3 + } + }, + "summary": { + "successful_queries": 10, + "tokenizer": "gpt-tokenizer@3.4.0", + "average_tokens": { + "normal": 2122, + "compact": 1480.3, + "compact_aggressive": 1401.7 + }, + "savings_percent": { + "compact": 30.2, + "compact_aggressive": 33.9, + "waterfall_engine_calls": null + }, + "average_engine_calls": null + }, + "per_query": { + "normal": [ + { + "id": "format-1", + "tokens": 1686 + }, + { + "id": "format-2", + "tokens": 1646 + }, + { + "id": "format-3", + "tokens": 1655 + }, + { + "id": "format-4", + "tokens": 1624 + }, + { + "id": "format-5", + "tokens": 1656 + }, + { + "id": "format-6", + "tokens": 2617 + }, + { + "id": "format-7", + "tokens": 2545 + }, + { + "id": "format-8", + "tokens": 2546 + }, + { + "id": "format-9", + "tokens": 2638 + }, + { + "id": "format-10", + "tokens": 2607 + } + ], + "compact": [ + { + "id": "format-1", + "tokens": 1297 + }, + { + "id": "format-2", + "tokens": 1257 + }, + { + "id": "format-3", + "tokens": 1273 + }, + { + "id": "format-4", + "tokens": 1249 + }, + { + "id": "format-5", + "tokens": 1267 + }, + { + "id": "format-6", + "tokens": 1710 + }, + { + "id": "format-7", + "tokens": 1666 + }, + { + "id": "format-8", + "tokens": 1667 + }, + { + "id": "format-9", + "tokens": 1717 + }, + { + "id": "format-10", + "tokens": 1700 + } + ], + "compact_aggressive": [ + { + "id": "format-1", + "tokens": 1276 + }, + { + "id": "format-2", + "tokens": 1221 + }, + { + "id": "format-3", + "tokens": 1249 + }, + { + "id": "format-4", + "tokens": 1222 + }, + { + "id": "format-5", + "tokens": 1240 + }, + { + "id": "format-6", + "tokens": 1578 + }, + { + "id": "format-7", + "tokens": 1540 + }, + { + "id": "format-8", + "tokens": 1541 + }, + { + "id": "format-9", + "tokens": 1582 + }, + { + "id": "format-10", + "tokens": 1568 + } + ] + } +} diff --git a/benchmarks/run.mjs b/benchmarks/run.mjs new file mode 100644 index 0000000..679722a --- /dev/null +++ b/benchmarks/run.mjs @@ -0,0 +1,287 @@ +#!/usr/bin/env node +import { createHash } from 'node:crypto'; +import { mkdir, readFile, writeFile } from 'node:fs/promises'; +import { dirname, relative, resolve } from 'node:path'; +import { encode } from 'gpt-tokenizer'; + +const ROOT = resolve(import.meta.dirname, '..'); +const ALL_ENGINES = [ + 'duckduckgo', 'sogou', 'bing', 'baidu', + 'wikipedia', 'startpage', 'yandex', 'mojeek', + 'brave', 'tavily', 'exa', 'youcom', +]; +const ZERO_KEY_ENGINE_COUNT = 8; +const OPTIONAL_KEY_ENV = ['BRAVE_API_KEY', 'TAVILY_API_KEY', 'EXA_API_KEY', 'YDC_API_KEY']; +const SCENARIOS = { + normal: { style: 'normal', snippetMax: 200 }, + compact: { style: 'compact', snippetMax: 200, maxFullResults: 3 }, + compact_aggressive: { style: 'compact', snippetMax: 120, maxFullResults: 3 }, +}; + +function option(name) { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : undefined; +} + +function round(value, digits = 1) { + const factor = 10 ** digits; + return Math.round(value * factor) / factor; +} + +function sha256(value) { + return createHash('sha256').update(value).digest('hex'); +} + +function tokenCount(value) { + return encode(JSON.stringify(value)).length; +} + +async function readJson(path) { + return JSON.parse(await readFile(path, 'utf8')); +} + +async function writeJson(path, value) { + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, 'utf8'); +} + +async function capture(fixturePath) { + process.env.OUTPUT_STYLE = 'normal'; + process.env.MAX_FULL_RESULTS = '50'; + process.env.MIN_CONFIDENCE = '0'; + process.env.MIN_SOURCE_COUNT = '1'; + + const [{ searchWithFallback }, packageJson, querySet] = await Promise.all([ + import('../dist/tools/free-search.js'), + readJson(resolve(ROOT, 'package.json')), + readJson(resolve(ROOT, 'benchmarks/queries.json')), + ]); + const allQueries = Array.isArray(querySet) ? querySet : querySet.queries; + const requestedLimit = Number.parseInt(option('--limit') || String(allQueries?.length || 0), 10); + const english = allQueries?.filter(item => (item.language || item.lang) !== 'zh') || []; + const chinese = allQueries?.filter(item => (item.language || item.lang) === 'zh') || []; + const englishLimit = Math.ceil(requestedLimit / 2); + const queries = requestedLimit < (allQueries?.length || 0) + ? [...english.slice(0, englishLimit), ...chinese.slice(0, requestedLimit - englishLimit)] + : allQueries; + if (!Array.isArray(queries)) throw new Error('benchmarks/queries.json must contain an array'); + + const samples = []; + const fixture = { + schema_version: 1, + kind: 'live-capture', + captured_at: new Date().toISOString(), + package_version: packageJson.version, + query_set_sha256: sha256(JSON.stringify(queries)), + tokenizer: 'gpt-tokenizer@3.4.0', + zero_key_engine_baseline: ZERO_KEY_ENGINE_COUNT, + naive_engine_baseline: ZERO_KEY_ENGINE_COUNT + OPTIONAL_KEY_ENV.filter(name => process.env[name]).length, + samples, + }; + for (let index = 0; index < queries.length; index++) { + const item = typeof queries[index] === 'string' ? { query: queries[index] } : queries[index]; + const query = item.query || item.q; + if (!query) throw new Error(`Query ${index + 1} has no query/q field`); + const startedAt = Date.now(); + try { + const response = await searchWithFallback({ + query, + count: 10, + engines: ALL_ENGINES, + waterfall: true, + minConfidence: 0, + minSourceCount: 1, + enrich: false, + expandQueries: false, + }); + samples.push({ + id: item.id || `q${index + 1}`, + query, + language: item.language || item.lang || 'unknown', + duration_ms: Date.now() - startedAt, + response, + }); + console.log(`[${index + 1}/${queries.length}] ${query} — ${response.results.length} results, ${response.meta.execution?.engine_calls ?? 0} calls`); + } catch (error) { + samples.push({ + id: item.id || `q${index + 1}`, + query, + language: item.language || item.lang || 'unknown', + duration_ms: Date.now() - startedAt, + error: error instanceof Error ? error.message : String(error), + }); + console.error(`[${index + 1}/${queries.length}] ${query} — failed`); + } + // Checkpoint after every query so an interrupted live run remains inspectable. + await writeJson(fixturePath, fixture); + } + console.log(`Captured ${samples.length} samples to ${fixturePath}`); +} + +async function generateFormatFixture(fixturePath) { + const querySet = await readJson(resolve(ROOT, 'benchmarks/queries.json')); + const selected = [ + ...querySet.filter(item => item.lang === 'en').slice(0, 5), + ...querySet.filter(item => item.lang === 'zh').slice(0, 5), + ]; + const samples = selected.map((item, queryIndex) => { + const results = Array.from({ length: 10 }, (_, resultIndex) => { + const sources = resultIndex % 4 === 0 + ? ['duckduckgo', 'wikipedia'] + : [resultIndex % 2 === 0 ? 'sogou' : 'duckduckgo']; + const suffix = item.lang === 'zh' + ? `这是关于“${item.q}”的冻结基准摘要，用于验证渐进披露、字段语义和中文 Token 计算在不同环境中保持一致。结果编号 ${resultIndex + 1}。` + : `Frozen benchmark summary about “${item.q}”, used to verify progressive disclosure, field semantics, and token counts across environments. Result ${resultIndex + 1}.`; + return { + title: `${item.q} — ${item.lang === 'zh' ? '参考结果' : 'Reference result'} ${resultIndex + 1}`, + url: `https://benchmark.example/${queryIndex + 1}/${resultIndex + 1}`, + snippet: `${suffix} ${suffix}`, + confidence: round(0.94 - resultIndex * 0.035, 3), + relevance: round(0.91 - resultIndex * 0.04, 3), + source_count: sources.length, + sources, + }; + }); + return { + id: `format-${queryIndex + 1}`, + query: item.q, + language: item.lang, + duration_ms: 0, + response: { + query: item.q, + engines: ALL_ENGINES.slice(0, ZERO_KEY_ENGINE_COUNT), + results, + meta: { + total: results.length, + high_confidence: results.filter(result => result.confidence >= 0.8).length, + engines: [...new Set(results.flatMap(result => result.sources))], + }, + security_note: 'Frozen format-regression fixture; content is synthetic and contains no search-quality claim.', + }, + }; + }); + await writeJson(fixturePath, { + schema_version: 1, + kind: 'format-regression', + captured_at: new Date().toISOString(), + package_version: (await readJson(resolve(ROOT, 'package.json'))).version, + query_set_sha256: sha256(JSON.stringify(selected)), + tokenizer: 'gpt-tokenizer@3.4.0', + zero_key_engine_baseline: ZERO_KEY_ENGINE_COUNT, + naive_engine_baseline: ZERO_KEY_ENGINE_COUNT, + samples, + }); + console.log(`Generated deterministic format fixture at ${fixturePath}`); +} + +function toScoredResult(result) { + const sources = result.sources || ['unknown']; + const relevance = result.relevance ?? 0; + return { + title: result.title, + url: result.url, + snippet: result.snippet || '', + source: sources[0] || 'unknown', + engines: sources, + confidence: result.confidence ?? 0, + relevance, + source_count: result.source_count ?? sources.length, + score: relevance, + }; +} + +async function replay(fixturePath, outputPath, check) { + const fixtureText = await readFile(fixturePath, 'utf8'); + const fixture = JSON.parse(fixtureText); + const { formatResults } = await import('../dist/aggregation/format.js'); + const successful = fixture.samples.filter(sample => sample.response?.results?.length > 0); + if (successful.length === 0) throw new Error('Fixture contains no successful samples'); + + const scenarioRows = {}; + for (const [name, formatOptions] of Object.entries(SCENARIOS)) { + scenarioRows[name] = successful.map(sample => { + const scored = sample.response.results.map(toScoredResult); + const formatted = formatResults(scored, formatOptions); + const payload = { + query: sample.query, + engines: sample.response.engines, + results: formatted.results, + meta: { + ...formatted.meta, + execution: sample.response.meta.execution, + }, + security_note: formatted.security_note, + }; + return { id: sample.id, tokens: tokenCount(payload) }; + }); + } + + const average = rows => round(rows.reduce((sum, row) => sum + row.tokens, 0) / rows.length, 2); + const normalTokens = average(scenarioRows.normal); + const compactTokens = average(scenarioRows.compact); + const aggressiveTokens = average(scenarioRows.compact_aggressive); + const engineCalls = successful + .map(sample => sample.response.meta.execution?.engine_calls) + .filter(count => Number.isFinite(count)); + const avgEngineCalls = engineCalls.length > 0 + ? round(engineCalls.reduce((sum, count) => sum + count, 0) / engineCalls.length, 2) + : null; + const naiveEngineBaseline = fixture.naive_engine_baseline || fixture.zero_key_engine_baseline; + + const summary = { + successful_queries: successful.length, + tokenizer: fixture.tokenizer, + average_tokens: { + normal: normalTokens, + compact: compactTokens, + compact_aggressive: aggressiveTokens, + }, + savings_percent: { + compact: round((1 - compactTokens / normalTokens) * 100), + compact_aggressive: round((1 - aggressiveTokens / normalTokens) * 100), + waterfall_engine_calls: avgEngineCalls === null + ? null + : round((1 - avgEngineCalls / naiveEngineBaseline) * 100), + }, + average_engine_calls: avgEngineCalls, + }; + + const report = { + schema_version: 1, + fixture: relative(ROOT, fixturePath).replaceAll('\\', '/'), + fixture_sha256: sha256(fixtureText), + generated_at: new Date().toISOString(), + scenarios: SCENARIOS, + summary, + per_query: scenarioRows, + }; + + if (check) { + const expected = fixture.expected_summary; + if (!expected) throw new Error('Fixture has no expected_summary; run with --update-expected first'); + if (JSON.stringify(expected) !== JSON.stringify(summary)) { + throw new Error(`Benchmark drift detected.\nExpected: ${JSON.stringify(expected)}\nActual: ${JSON.stringify(summary)}`); + } + console.log('Benchmark replay matches expected_summary'); + } else if (process.argv.includes('--update-expected')) { + fixture.expected_summary = summary; + await writeJson(fixturePath, fixture); + console.log(`Updated expected_summary in ${fixturePath}`); + } + + if (outputPath) await writeJson(outputPath, report); + console.log(JSON.stringify(summary, null, 2)); +} + +const capturePath = option('--capture'); +const generatedFixturePath = option('--generate-format-fixture'); +const fixturePath = resolve(ROOT, option('--fixture') || 'benchmarks/fixtures/latest.json'); +const outputPath = option('--output') ? resolve(ROOT, option('--output')) : undefined; + +if (generatedFixturePath) { + await generateFormatFixture(resolve(ROOT, generatedFixturePath)); +} else if (capturePath) { + await capture(resolve(ROOT, capturePath)); +} else { + await replay(fixturePath, outputPath, process.argv.includes('--check')); +} diff --git a/docs/geo/DISTRIBUTION_STATUS.md b/docs/geo/DISTRIBUTION_STATUS.md index d00a8b6..e58247c 100644 --- a/docs/geo/DISTRIBUTION_STATUS.md +++ b/docs/geo/DISTRIBUTION_STATUS.md @@ -1,7 +1,7 @@ # 分发渠道状态追踪 — Agent Search MCP -> 最后更新: 2026-07-23 -> npm: v3.1.3 | GitHub Stars: 9 | 月下载: 1,876 +> 最后更新: 2026-07-25 +> npm: v3.1.3 | GitHub Stars: 10 | 近 30 天下载: 1,952 --- @@ -10,9 +10,9 @@ | 指标 | 数值 | |------|------| | npm 版本 | v3.1.3 | -| npm 月下载 | 1,876 | -| npm 周下载 | 845 | -| GitHub Stars | 9 | +| npm 近 30 天下载 | 1,952 | +| npm 近 7 天下载 | 1,049 | +| GitHub Stars | 10 | | GitHub Topics | 20 | | npm keywords | 31 | @@ -33,6 +33,7 @@ | **mcpmarket.com** | 自动收录 | — | | **punkpeye/awesome-mcp-servers** | [PR #10383](https://github.com/punkpeye/awesome-mcp-servers/pull/10383) ✅ **已合并** | 条目已在主 README 中 | | **patriksimek/awesome-mcp-servers-2** | [PR #21](https://github.com/patriksimek/awesome-mcp-servers-2/pull/21) | ✅ | +| **重复 PR #10640** | [已关闭](https://github.com/punkpeye/awesome-mcp-servers/pull/10640) | 已注明由 #10383 收录 | ### ⏳ 等待中 @@ -40,7 +41,6 @@ |------|------|------| | **mcp.directory** | 你已提交，待收录 | 未在搜索中找到 (404) | | **mcpservers.org** | 你已提交，待收录 | 未在搜索中找到 (404) | -| **duplicate PR #10640** | OPEN | PR #10383 已合并，这条是重复的，可以关掉 | ### ❌ 未提交 @@ -51,22 +51,22 @@ | **PulseMCP** | 发邮件到 `hello@pulsemcp.com` 或提交表单 | 2min | | **FastMCP** | 打开 https://fastmcp.com/submit | 2min | | **awesome-remote-mcp-servers** | GitHub PR 加一行 | 5min | +| **Rodert/awesome-mcp** | 已达到 10 Stars 门槛，重新检查提交规范 | 5min | ### 📝 内容营销待发布 | 平台 | 文件 | 状态 | |------|------|:----:| -| **掘金** (ZH) | `docs/geo/juejin-agent-search-mcp.md` | 📝 草稿就绪 | -| **dev.to** (EN) | 未写 | ❌ | -| **V2EX** (ZH) | 未写 | ❌ | -| **Reddit r/mcp** (EN) | 未写 | ❌ | -| **MCP Discord #showcase** | 未写 | ❌ | +| **掘金** (ZH) | `docs/geo/juejin-agent-search-mcp.md` | ✅ 已按现状重写 | +| **dev.to** (EN) | `docs/geo/PROMOTION_KIT.md` | ✅ 大纲就绪 | +| **V2EX** (ZH) | `docs/geo/PROMOTION_KIT.md` | ✅ 短文就绪 | +| **Reddit r/mcp** (EN) | `docs/geo/PROMOTION_KIT.md` | ✅ 短文就绪 | +| **MCP Discord #showcase** | `docs/geo/PROMOTION_KIT.md` | ✅ 短文就绪 | ### 🚫 跳过 | 渠道 | 原因 | |------|------| -| Rodert/awesome-mcp | 要求 ≥10 stars | | Docker Hub | 本机无 Docker，需 Mac 操作 | --- @@ -74,13 +74,13 @@ ## 行动清单 (按优先级) ``` -P0 ─── Claim Glama → 打开 glama.ai 点 Claim - ├── 关掉重复 PR #10640 +P0 ─── 合并产品加固改动，等待 Linux/Windows CI 通过 + ├── Claim Glama → 打开 glama.ai 点 Claim ├── PulseMCP 提交 └── FastMCP 提交 -P1 ─── 发掘金文章（草稿已就绪） - ├── 写 dev.to / V2EX 草稿 +P1 ─── 发掘金文章（已按真实能力重写） + ├── 发 dev.to / V2EX / Reddit 短文 ├── LobeHub 登录后提交 └── Smithery 提交 diff --git a/docs/geo/PROMOTION_KIT.md b/docs/geo/PROMOTION_KIT.md new file mode 100644 index 0000000..7887bc9 --- /dev/null +++ b/docs/geo/PROMOTION_KIT.md @@ -0,0 +1,111 @@ +# Agent Search MCP — Promotion Kit + +Updated: 2026-07-25 + +## Positioning + +**Category:** Agent Search Router / Agent 搜索控制层 + +**Primary hook (EN):** Free, token-efficient web search for AI agents. + +**Primary hook (ZH):** 给 AI Agent 的免费、省 Token 网页搜索。 + +**Mechanism:** A local-first search router: zero-key start, Chinese-native routing, inspectable multi-source evidence, token-aware stopping, and optional commercial escalation. + +**Best audience:** Claude Code, Cursor, Codex, and other MCP users who want a local search path without creating an API account first. + +## The distinctive route + +Lead with **free + token-efficient**. Use the control loop to explain why the promise is credible and distinctive: + +1. **Start free:** useful search before account creation or API procurement. +2. **Route by context:** Chinese queries go to Chinese-native sources instead of a translation shim. +3. **Return evidence, not a black-box answer:** expose `relevance`, `confidence`, `source_count`, URLs, and source provenance. +4. **Spend context progressively:** waterfall stopping and compact output keep search inside the agent's token budget. +5. **Escalate, do not lock in:** Brave/Tavily/Exa/You.com are optional higher-cost routes, not mandatory foundations. + +The product is the routing policy and evidence contract. Adapters are replaceable supply. + +**Do not position as:** a cheaper Tavily, a better Exa index, a fully offline search engine, or a hosted Crawl/Map platform. + +**Proof points safe to publish:** + +- `npx agent-search-mcp` starts the stdio server on Node.js 18+. +- 12 adapters are included; 8 require no credentials and 4 use optional API keys. +- All 12 adapters are selectable from MCP, CLI, and waterfall routing; eight require no credentials and four use optional API keys. +- Native Sogou and Baidu adapters support Chinese-web discovery. +- 510 Vitest tests pass on the 2026-07-25 audit baseline. +- stdio and Streamable HTTP transports are implemented. + +## Claim boundaries + +- Scope exact figures: the historical 30-query 2026-07-24 run measured 28.7% Compact, 35.5% Compact+, and 75% fewer calls than naive eight-engine fan-out. Do not present them as universal guarantees. +- The frozen formatting fixture currently reproduces 30.2% / 33.9% savings with locked `gpt-tokenizer`; do not use it as a search-quality claim. +- Monthly savings comparisons or “competitors cannot self-host.” +- Treating `confidence` as a count of independent verifying sources; use `source_count` for that. +- Presenting the historical 30-query report as a cross-product quality ranking. + +## GitHub / directory description + +> Free, token-efficient web search for AI agents: eight zero-key sources, token-aware waterfall search, Chinese-native routing, inspectable evidence, and optional commercial escalation. + +## Reddit / Hacker News + +**Title** + +> Agent Search MCP: free, token-efficient web search for Claude Code, Cursor and Codex + +**Body** + +> I built an Apache-2.0 MCP search server around two practical goals: search without paying for an API, and spend fewer context tokens without blindly discarding evidence. +> +> `npx agent-search-mcp` starts a local stdio server. It begins with eight zero-key routes, sends Chinese queries to native Chinese sources, exposes relevance/confidence/source provenance, stops progressively to control token use, and escalates to Brave/Tavily/Exa/You.com only when configured. +> +> This is not positioned as a replacement for hosted crawl/research products. Tavily, Exa, Brave, and Firecrawl are stronger when you need hosted endpoints, Crawl/Map, vertical search, or an SLA. The niche here is zero-key local setup, Chinese-web coverage, and upstream choice. +> +> The historical 30-query run measured 28.7% fewer tokens in Compact mode, 35.5% in Compact+, and 75% fewer engine calls than naive eight-engine fan-out. Those are environment-scoped measurements, not guarantees. A new frozen-fixture replay with a locked tokenizer reproducibly measures 30.2% / 33.9% formatting savings. +> +> GitHub: https://github.com/lennney/agent-search-mcp +> +> Feedback I especially want: failing queries, Windows setup issues, and whether the tool surface should expose all adapters directly or keep a smaller default set. + +## V2EX + +**标题** + +> [开源] Agent Search MCP：免费起步，还能节省 Token 的 Agent 搜索 + +**正文** + +> 做了一个给 Claude Code / Cursor / Codex 用的 MCP 搜索，就解决两个直接问题：不买搜索 API 也能先搜，返回结果时少花 token。底层用路由、瀑布停止和渐进披露做到，不是简单把结果砍掉。`npx agent-search-mcp` 就能启动。 +> +> 目前重点不是替代 Tavily/Exa/Brave，而是补一个本地、零密钥、中文友好的选择。包内有搜狗、百度等零密钥适配器，也可以选择性接 Brave/Tavily/Exa/You.com。 +> +> 这轮刚修了 DDG 无 Python 时的 HTML fallback、stdio 日志污染、CSDN SSRF 和 Windows 构建；510 项测试通过。 +> +> 历史 30 查询实测中，Compact 节省 28.7% token，Compact+ 节省 35.5%，瀑布调用数相比 8 引擎全并发少 75%。这些数字限定于当时查询集和环境，不是通用保证；新的冻结 fixture 可稳定回放 30.2% / 33.9% 的格式化节省。欢迎用真实失败查询来打脸，比 Star 更有价值。 +> +> https://github.com/lennney/agent-search-mcp + +## Discord / Slack showcase + +> 🔎 **Agent Search MCP** — zero-key web search for MCP clients, with native Sogou/Baidu sources, multi-engine aggregation, optional commercial backends, news, and extraction. Runs locally with `npx agent-search-mcp`. Apache-2.0: https://github.com/lennney/agent-search-mcp + +## Dev.to article outline + +1. Why “search before signup” is a useful MCP product boundary. +2. One-minute configuration for Claude Code, Cursor, and Codex. +3. Native Chinese sources versus translated queries. +4. Multi-engine orchestration and graceful fallback. +5. Compact output and progressive disclosure. +6. Honest limits: no managed hosted endpoint, no Crawl/Map, no labeled retrieval-quality benchmark. +7. Reliability work: stdio safety, HTTP Bearer/Origin protection, SSRF protection, Windows build. +8. Evidence: scoped historical measurements plus reproducible frozen-fixture regression. + +## Distribution priority + +1. Merge and publish the hardening changes; let CI verify Linux and Windows. +2. Refresh GitHub description and directory metadata with the one-line positioning. +3. Publish the revised Juejin article. +4. Post the short Reddit/V2EX variants and collect failing-query examples. +5. Submit to remaining directories only after their listing can link to the current release and CI state. diff --git a/docs/geo/juejin-agent-search-mcp.md b/docs/geo/juejin-agent-search-mcp.md index 13e9e45..ac9b5ff 100644 --- a/docs/geo/juejin-agent-search-mcp.md +++ b/docs/geo/juejin-agent-search-mcp.md @@ -1,140 +1,141 @@ -# Tavily 太贵？我搭了一个 11 引擎的免费 MCP 搜索服务器，开源了 +# 免费起步，还能省 Token：我给 AI Agent 做了一个搜索路由器 -> AI Agent 每天搜上百次技术文档，Tavily 月费 $100+ 吃不消？花 15 分钟搭一个自己的搜索服务器，**11 个引擎聚合、8 个完全免费、零 API Key**。刚开源，求 Star 🙏 +> `npx agent-search-mcp`：8 个零密钥来源免费起步，用瀑布停止和渐进披露节省 token，同时保留中文原生搜索和多源证据。 ---- +很多 AI Agent 的搜索接入都从“注册账号、创建 API Key、绑定额度”开始；搜到之后，又会把大段重复摘要塞进 Agent 上下文。这个项目先解决两个最直接的问题：**搜索免费起步，输出尽量少花 token**。 -## 一、先算一笔账 +但“省”不是粗暴删掉结果。Agent 还需要一个搜索控制层：去哪里搜、什么时候停、花多少上下文、哪些证据值得信。 -如果你在用 AI Agent（Claude Code、Cursor、Cline 等）做开发，大概率绕不开搜索。查 API 文档、搜技术方案、验证新闻事实——Agent 每天都在搜。 +所以这个项目选的不是“再做一个 Tavily/Exa”，而是一条不同的路线： -一开始我用 Tavily，质量确实不错。但用了两个月看账单： +- 不注册账号也能开始搜索； +- 能直接覆盖搜狗、百度等中文来源； +- 一个 MCP Server 里做多源聚合、去重、排序和自动降级； +- 需要更强商业搜索时，再选择性接入 Brave、Tavily、Exa 或 You.com。 -| 方案 | 免费额度 | 月费(1000次/天) | -|------|---------|----------------| -| Tavily | 1000/月 | ~$240/月 | -| Exa | $10 额度 | $50/月起 | -| Brave Search | 2000/月 | ~$84/月 | -| Serper | 2500/月 | ~$9/月 | -| **agent-search-mcp** | **无限** | **$0** | +于是有了开源项目 [agent-search-mcp](https://github.com/lennney/agent-search-mcp)。 -**每年轻轻松松省下几千美元。** 而且 agent-search-mcp 不单是便宜的替代品——它把 11 个引擎的结果做了多源交叉验证，比单引擎更可靠。 +## 一分钟接入 ---- +项目要求 Node.js 18 或更高版本。 -## 二、市场方案全景对比 +```json +{ + "mcpServers": { + "agent-search": { + "command": "npx", + "args": ["-y", "agent-search-mcp"] + } + } +} +``` -| 方案 | 免费额度 | 引擎数 | 内容提取 | 自托管 | Token 优化 | API Key | -|------|---------|:------:|:--------:|:------:|:----------:|:-------:| -| Tavily | 1000/月 | 单源 | ✅ | ❌ | ❌ | 必填 | -| Exa | $10/月 | 单源 | ✅ | ❌ | ❌ | 必填 | -| Brave | 2000/月 | 单源 | ❌ | ❌ | ❌ | 必填 | -| Serper | 2500/月 | 单源 | ❌ | ❌ | ❌ | 必填 | -| DDG MCP | 无限 | 单源 | ❌ | ✅ | ❌ | 非必须 | -| **agent-search-mcp** | **无限** | **11 引擎** | **✅** | **✅** | **✅** | **非必须** | +不需要预先全局安装，也不需要 API Key。保存配置、重启 MCP 客户端即可。 -唯一一个在「免费」「多引擎」「自托管」「Token 优化」「内容提取」五个维度同时满足的方案。 +项目还提供 CLI： ---- +```bash +fasm search "MCP Server 中文搜索" +fasm extract "https://example.com" +``` -## 三、凭什么免费？8 个免费引擎 +## 它和 Tavily、Exa、Brave 的关系 -核心思路很简单：**聚合已有的免费搜索接口，不做昂贵的中间层。** +这不是“谁替代谁”的关系，而是不同产品边界： -``` -你的 Agent → agent-search-mcp → 11 个引擎并发搜索 - ↓ - 去重 + 评分 + 排序 - ↓ - 返回最优结果 -``` +| 方案 | 主要优势 | 更适合 | +|------|----------|--------| +| Agent Search MCP | 本地、零密钥起步、中文引擎、多源聚合 | 本地 Agent、中文检索、开源自托管 | +| Tavily MCP | 托管 Search/Extract/Map/Crawl | 需要完整托管采集工作流 | +| Exa MCP | 语义、代码、企业研究 | 高相关语义检索和研究任务 | +| Brave Search MCP | 独立索引、新闻/图片/视频等垂直搜索 | 需要稳定商业索引和垂直结果 | + +商业服务通常提供免费额度，但仍需要账号或凭证。价格和额度会变化，所以项目不再用容易过期的“每月省多少钱”作为卖点。 -### 8 个免费引擎（零 API Key） +## 当前搜索架构 -| 引擎 | 用途 | -|------|------| -| DuckDuckGo | 通用搜索，隐私优先 | -| Bing | 微软必应，英文技术内容 | -| Sogou | 搜狗搜索——中文技术博客/文档 | -| Baidu | 百度搜索——国内信息 | -| Wikipedia | 百科知识 | -| Startpage | Google 结果，匿名代理 | -| Yandex | 俄语/东欧内容 | -| Mojeek | 独立爬虫，不跟踪 | +包内有 12 个搜索适配器，其中 8 个不需要凭证，4 个为可选商业 API。 -### 3 个付费引擎（可选增强） +当前 `free_search`、`free_search_advanced`、CLI 和瀑布模式已统一路由全部 12 个适配器： -Brave / Tavily / Exa —— 填写 API Key 后自动作为 fallback 增强层。 +- DuckDuckGo +- 搜狗 +- Bing +- 百度 +- Wikipedia +- Startpage +- Yandex +- Mojeek +- Brave（可选 Key） +- Tavily（可选 Key） +- Exa（可选 Key） +- You.com（可选 Key） -**实际使用中，8 个免费引擎已经覆盖了 95% 的需求。** +一次搜索会经过： ---- +```text +查询 + → 语言识别与引擎选择 + → 并行或瀑布编排 + → URL/标题去重 + → 排序与安全标记 + → normal / compact 输出 +``` -## 四、多源交叉验证——Agent 不会被骗 +如果 Python `ddgs` 不可用，DuckDuckGo 会自动回退到纯 Node.js HTML 路径。这个回退刚补上了主编排层的回归测试。 -单引擎搜索的最大问题是：**你不知道结果靠不靠谱。** +## 为 Agent 控制上下文体积 -agent-search-mcp 把同一个查询同时发到多个引擎，跨源对比： +搜索工具很容易把大量摘要和元数据塞进上下文。项目提供几组可选配置： -- 每个结果附带**置信度评分**（1-3 分） -- ≥2 分 = 至少被两个不同引擎独立验证过 -- 跨语言去重（中英文结果自动融合） +```bash +OUTPUT_STYLE=compact +MAX_FULL_RESULTS=3 +SNIPPET_LENGTH=160 +MIN_CONFIDENCE=0 +``` -比如说搜 "React 19 新特性"，DuckDuckGo 和 Bing 和 Sogou 各返回一堆链接。agent-search-mcp 会： -1. URL 去重 → 去掉重复的 -2. 标题相似去重 → 去掉换源转载的 -3. 频次评分 → 同时出现在多个引擎的排前面 -4. 域名权威加分 → GitHub/官方文档权重更高 +Compact 模式会完整展示前几个结果，其余结果只保留标题和 URL；Agent 需要时再调用 `free_extract`。历史 30 查询真实运行实测 Compact 节省 28.7% token、Compact+ 节省 35.5%，瀑布调用数相比 8 引擎全并发少 75%。这些是当时查询集和环境的实测，不是生产保证。新的冻结 fixture + 锁定 tokenizer 回放可稳定验证 30.2% / 33.9% 的格式化节省。 ---- +## 可靠性和安全边界 -## 五、5 分钟跑起来 +当前版本包含： -```bash -# 直接 npx 跑（零安装） -npx -y agent-search-mcp +- stdio 与 Streamable HTTP，HTTP 默认要求 Bearer Token 并校验浏览器 Origin； +- MCP read-only / idempotent annotations； +- 引擎限速、健康状态与熔断； +- URL 安全检查和搜索内容注入标记； +- 510 项 Vitest 测试； +- Linux、macOS、Windows 通用构建脚本。 -# 或者全局安装 -npm install -g agent-search-mcp -``` +最近还修复了两个容易被忽略的问题： -然后在 Claude Desktop / Cursor / Cline 的配置里加： +1. 熔断日志从 stdout 移到 stderr，避免污染 stdio JSON-RPC； +2. CSDN 抓取只允许 `https://blog.csdn.net`，并拒绝重定向，封住直接 SSRF 入口。 -```json -{ - "mcpServers": { - "agent-search": { - "command": "npx", - "args": ["-y", "agent-search-mcp"] - } - } -} -``` +## 适合与不适合 -重启 Agent，你的 AI 就有 11 个搜索引擎了。 +适合： -**中文搜索直接搜，不用任何配置** —— agent-search-mcp 自动检测语言，中文查询自动启用 Sogou + Baidu。 +- 想让本地 Agent 先获得无需账号的网页搜索； +- 经常搜索中文技术资料； +- 希望保留自托管和多上游选择权； +- 愿意用开源项目换取更低的接入门槛。 ---- +不适合： -## 六、进阶功能 +- 需要企业 SLA、稳定高并发和统一托管； +- 需要站点 Crawl/Map、浏览器交互或图片/视频垂直搜索； +- 要求搜索质量由人工标注 benchmark 或商业合同保证。 -| 功能 | 说明 | -|------|------| -| **瀑布搜索** | 先搜免费引擎，结果不够自动切换到付费引擎 | -| **内容丰富化** | 对低置信度结果自动提取页面全文 | -| **新闻搜索** | 专用新闻搜索模式（DDG News + Bing News） | -| **Token 优化** | 自动裁剪冗余内容，节省 ~40-50% token | -| **安全防护** | SSRF 保护 + 注入检测 + 速率限制 | -| **Docker 部署** | 一键 `docker pull`，生产环境友好 | +## 接下来 ---- +下一阶段会优先做三件事： -## 七、总结 +1. 给真实搜索 benchmark 增加人工相关性标签和稳定的网络 runner； +2. 为 HTTP 部署补充反向代理、密钥轮换和部署指南； +3. 继续用真实失败查询校准 relevance、confidence 和 `source_count`。 -**agent-search-mcp 是给 AI Agent 用的免费多引擎搜索服务器。** 不需要 API Key，不需要付费，15 分钟搭好就能用。 +项目地址：[github.com/lennney/agent-search-mcp](https://github.com/lennney/agent-search-mcp) -- ⭐ GitHub：[lennney/agent-search-mcp](https://github.com/lennney/agent-search-mcp)（点个 Star 支持一下 🙏） -- 📦 npm：[agent-search-mcp](https://www.npmjs.com/package/agent-search-mcp) -- 📖 英文版：[Read in English](https://gh.l-web.com/blog/tavily-alternative-agent-search-mcp-en) -- 438 个测试，MIT 协议 +如果你正在用 Claude Code、Cursor 或 Codex，欢迎试一下；Issue、测试样例和失败查询，比一句“好用”更有帮助。 diff --git a/docs/http-deployment.md b/docs/http-deployment.md new file mode 100644 index 0000000..4612eff --- /dev/null +++ b/docs/http-deployment.md @@ -0,0 +1,48 @@ +# HTTP Deployment Guide + +Agent Search MCP keeps stdio as the zero-configuration local default. HTTP mode is intended for controlled network deployment and requires authentication unless the operator explicitly opts out. + +## Secure local start + +Generate a high-entropy token and pass it through the environment rather than a config file: + +```bash +node -e "console.log(require('node:crypto').randomBytes(32).toString('base64url'))" + +HTTP_AUTH_TOKEN="replace-with-generated-token" \ +ALLOWED_ORIGINS="https://agent.example.com" \ +MODE=http PORT=3000 npx agent-search-mcp +``` + +Clients call the MCP endpoint with: + +```http +Authorization: Bearer replace-with-generated-token +``` + +`GET /health` remains unauthenticated for process and load-balancer probes. It does not expose MCP tools or search results. + +## Origin policy + +`ALLOWED_ORIGINS` is a comma-separated exact allowlist for browser `Origin` values. Requests without an Origin header, such as normal server-to-server MCP clients, are still accepted after Bearer authentication. Requests with an unlisted Origin receive `403`. + +Set `ENABLE_CORS=true` only when a browser client needs CORS response headers. Avoid `*` when credentials are involved; list the actual HTTPS origins. + +## Reverse proxy checklist + +- Terminate TLS at the proxy; do not expose plain HTTP outside a trusted network. +- Preserve the `Authorization`, `Origin`, and `Mcp-Session-Id` headers. +- Apply connection and request-rate limits at the proxy. +- Store `HTTP_AUTH_TOKEN` in a secret manager and rotate it by restarting the service with a new value. +- Restrict `/health` at the network layer if infrastructure metadata is considered sensitive. +- Keep logs free of Authorization headers and tokens. + +## Explicit unauthenticated mode + +For an isolated local development network only: + +```bash +HTTP_ALLOW_UNAUTHENTICATED=true MODE=http npx agent-search-mcp +``` + +This is an explicit risk acceptance. Origin checks still apply to browser requests; set `ALLOWED_ORIGINS` when browser access is required. diff --git a/docs/index.md b/docs/index.md index 9a2918f..5b627c5 100644 --- a/docs/index.md +++ b/docs/index.md @@ -37,6 +37,10 @@ | 文档 | 说明 | |------|------| +| [../benchmarks/README.md](../benchmarks/README.md) | 探索性基准、方法限制与历史报告 | +| [geo/DISTRIBUTION_STATUS.md](geo/DISTRIBUTION_STATUS.md) | 分发渠道和公开指标状态 | +| [geo/PROMOTION_KIT.md](geo/PROMOTION_KIT.md) | 已校准口径的中英文推广素材 | +| [geo/juejin-agent-search-mcp.md](geo/juejin-agent-search-mcp.md) | 掘金长文草稿 | | `plans/YYYY-MM-DD-title.md` | 功能计划与评审 | | `reviews/` | 安全/多平台/功能评审 | diff --git a/package-lock.json b/package-lock.json index f65d7e9..f212a4d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -7,10 +7,10 @@ "": { "name": "agent-search-mcp", "version": "3.1.3", - "license": "MIT", + "license": "Apache-2.0", "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", - "cheerio": "^1.2.0", + "cheerio": "1.0.0", "pino": "^10.3.1", "yaml": "^2.7.1", "zod": "^4.4.3" @@ -23,7 +23,8 @@ "@eslint/js": "^10.0.1", "@types/node": "^26.1.1", "eslint": "^10.7.0", - "typescript": "^7.0.2", + "gpt-tokenizer": "3.4.0", + "typescript": "^6.0.3", "typescript-eslint": "^8.65.0", "vitest": "^3.1.3" }, @@ -1181,346 +1182,6 @@ "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@typescript/typescript-aix-ppc64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-aix-ppc64/-/typescript-aix-ppc64-7.0.2.tgz", - "integrity": "sha512-MTKKkWB7p/0E9xi1d1tHtZ5PiLkGEMIq88pK2CubZjOsLtYTLqhgIgi6zepFa+9GHZ6h05NMCkQxGKiPXMxXtQ==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "aix" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-darwin-arm64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-arm64/-/typescript-darwin-arm64-7.0.2.tgz", - "integrity": "sha512-gowzar9MwS/aRWp6f3a4KUqzRjAZjOsmGNCM6LcTgXum+dBfgsBVMN+AgvOCCbguXyick6LJhpBszxMebJ8syA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-darwin-x64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-darwin-x64/-/typescript-darwin-x64-7.0.2.tgz", - "integrity": "sha512-SZ9xZInqApNlNGc9s0W1VSsktYSOe9cFqNOIqmN1Gs8SmkjKZYFt017G4VwPxASInODuAdbTW7sXiFUf893RgA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-freebsd-arm64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-arm64/-/typescript-freebsd-arm64-7.0.2.tgz", - "integrity": "sha512-W5NH4y/J0plIIS5b2xvTEkU7JFxyqdMAOgf+Ilhl0vHQXKO5dZoxd+C/jEtq56c4F3wk71RB4BMRQ2XdI+bwYQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-freebsd-x64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-freebsd-x64/-/typescript-freebsd-x64-7.0.2.tgz", - "integrity": "sha512-UMGDx5sTpzNw3WiPebH7l90IWfJggEd+egHt/q6p7/Cm3zqoV7VxkGXt+3DxPIw8CcmvAB0j3sVVfbhX+M4Tpw==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "freebsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-arm": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm/-/typescript-linux-arm-7.0.2.tgz", - "integrity": "sha512-gffT3xPz9sR7j/YJExkyPntrI0P2EP9XbOyWzth2/Gs0RstK+90RBcO0ncXoXy/beYll1SXw846Nf2zdnEz0QQ==", - "cpu": [ - "arm" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-arm64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-arm64/-/typescript-linux-arm64-7.0.2.tgz", - "integrity": "sha512-Qh4eU4/y3yDjnfjjyPYihMj5/ODIlmt+Bzu17OI+fiSRDW57QmU5SiN63exPRNJPKUzcc1INa1NXdrJ+MqHjUQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-loong64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-loong64/-/typescript-linux-loong64-7.0.2.tgz", - "integrity": "sha512-uEHck9i8hoAzXPiYRib1O7miOnz23SxIeVl6F4LXox+qov1K35jHcEW6VHKvZI+pyvl7fZEP4MCU5LYvIq1GuQ==", - "cpu": [ - "loong64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-mips64el": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-mips64el/-/typescript-linux-mips64el-7.0.2.tgz", - "integrity": "sha512-R4KvAMnE43W5Qeqb0Ly56O3mWMWIAgsMyz36DCaycd5nbg/9kzm0liw3JocfRqyJY0KPmzFjbswozXyW0DnIYA==", - "cpu": [ - "mips64el" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-ppc64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-ppc64/-/typescript-linux-ppc64-7.0.2.tgz", - "integrity": "sha512-DORx5b3sd/4S7eayxm4FQv+A7CrkUIGRaHiwI8oiHTAI1fAPWhF4J0vAlkC8biAlHSVVwxMQ3tjZ2/DVbnQiiA==", - "cpu": [ - "ppc64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-riscv64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-riscv64/-/typescript-linux-riscv64-7.0.2.tgz", - "integrity": "sha512-wf0jqEDOjrPRnKwYRyyJDRo11KMbvMFrU+q4zqKyChODBzvlkbhNQfKvLxQCcwTpdDaXSHZTVuh0JoCrKCUMHQ==", - "cpu": [ - "riscv64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-s390x": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-s390x/-/typescript-linux-s390x-7.0.2.tgz", - "integrity": "sha512-IkwJc3L7yhytWd/ewjyxNDfOmswCm9GWMJT/ue/dU4aZNbwZeYAetq42VyLmsmSjvoX7z74X6ZaYCtzAr0EuGw==", - "cpu": [ - "s390x" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-linux-x64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-linux-x64/-/typescript-linux-x64-7.0.2.tgz", - "integrity": "sha512-EYdf2cNg7rgCWJnxCdJ+F3V39O8ihb37eHAu1LK8oAFizgTQbPOK7zHHXbPt8rX24COqODXeI3sIf0fCXG7H/A==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-netbsd-arm64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-arm64/-/typescript-netbsd-arm64-7.0.2.tgz", - "integrity": "sha512-+polYF4MF04aPpO5FTkHran9yUQDSXqy5GiSDKpsll5jy3l3+g9QLhpf39T+ePtefhXLOGrLl0QIjkQP6VnelA==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-netbsd-x64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-netbsd-x64/-/typescript-netbsd-x64-7.0.2.tgz", - "integrity": "sha512-8YIT0EHM/3dq10ZOVF/A7pc/YSMtbcecct4rWtexrnSCHOPcpC2KTLXfTCR6vDpnSiY12heNb1GiN/wu+T/FyA==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "netbsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-openbsd-arm64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-arm64/-/typescript-openbsd-arm64-7.0.2.tgz", - "integrity": "sha512-APT8+ClYnuYm1u9+kgGXoMj2VzWzcymwh2gNSQVySHfkRDGOTVkoWLjCmOQSaO+PoqQ57B0flRp9SA+7GnnkzQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-openbsd-x64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-openbsd-x64/-/typescript-openbsd-x64-7.0.2.tgz", - "integrity": "sha512-yX7s+Q0Dln0Dt9tEzZsAjXXR/+ytBM7AlglaqyeMPxQszJ1JhlJdZ6jLA+IzldHtflX81em7lDao1xXu+aRRkg==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "openbsd" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-sunos-x64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-sunos-x64/-/typescript-sunos-x64-7.0.2.tgz", - "integrity": "sha512-dLJDGaLZ1D4HPQn62u1n8mBDkJREwMsAkCdkwd4Ieqw+x3TUyTsqY0YiBCtE6H6OzzgGk3iuZ3vFWRS+E8/d1g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "sunos" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-win32-arm64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-arm64/-/typescript-win32-arm64-7.0.2.tgz", - "integrity": "sha512-Gyl1Vy6OsWesLzmq+EP0Fb7b4Nid5232AvcA2SFcdYreldpNtYFFofPjnt62y9hQy7VTaZp65ICJjuAQRaVcIQ==", - "cpu": [ - "arm64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=16.20.0" - } - }, - "node_modules/@typescript/typescript-win32-x64": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/@typescript/typescript-win32-x64/-/typescript-win32-x64-7.0.2.tgz", - "integrity": "sha512-0BQ3HkAHHlKLSp1qRvf3SUhGpGsDuhB/jgFw75guyqbxJqEaS0Cw/VFO8i2nHglJUzQCRtMMR/IBAKE3ETMC4g==", - "cpu": [ - "x64" - ], - "dev": true, - "license": "Apache-2.0", - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">=16.20.0" - } - }, "node_modules/@vitest/expect": { "version": "3.2.6", "resolved": "https://registry.npmjs.org/@vitest/expect/-/expect-3.2.6.tgz", @@ -1866,25 +1527,25 @@ } }, "node_modules/cheerio": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz", - "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==", + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.0.0.tgz", + "integrity": "sha512-quS9HgjQpdaXOvsZz82Oz7uxtXiy6UIsIQcpBj7HRw2M63Skasm9qlDocAM7jNuaxdhpPU7c4kJN+gA5MCu4ww==", "license": "MIT", "dependencies": { "cheerio-select": "^2.1.0", "dom-serializer": "^2.0.0", "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "encoding-sniffer": "^0.2.1", - "htmlparser2": "^10.1.0", - "parse5": "^7.3.0", - "parse5-htmlparser2-tree-adapter": "^7.1.0", + "domutils": "^3.1.0", + "encoding-sniffer": "^0.2.0", + "htmlparser2": "^9.1.0", + "parse5": "^7.1.2", + "parse5-htmlparser2-tree-adapter": "^7.0.0", "parse5-parser-stream": "^7.1.2", - "undici": "^7.19.0", + "undici": "^6.19.5", "whatwg-mimetype": "^4.0.0" }, "engines": { - "node": ">=20.18.1" + "node": ">=18.17" }, "funding": { "url": "https://github.com/cheeriojs/cheerio?sponsor=1" @@ -2788,6 +2449,13 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/gpt-tokenizer": { + "version": "3.4.0", + "resolved": "https://registry.npmjs.org/gpt-tokenizer/-/gpt-tokenizer-3.4.0.tgz", + "integrity": "sha512-wxFLnhIXTDjYebd9A9pGl3e31ZpSypbpIJSOswbgop5jLte/AsZVDvjlbEuVFlsqZixVKqbcoNmRlFDf6pz/UQ==", + "dev": true, + "license": "MIT" + }, "node_modules/has-symbols": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", @@ -2822,9 +2490,9 @@ } }, "node_modules/htmlparser2": { - "version": "10.1.0", - "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz", - "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==", + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-9.1.0.tgz", + "integrity": "sha512-5zfg6mHUoaer/97TxnGpxmbR7zJtPwIYFMZ/H5ucTlPZhKvtum05yiPK3Mgai3a0DyVxv7qYqoweaEd2nrYQzQ==", "funding": [ "https://github.com/fb55/htmlparser2?sponsor=1", { @@ -2836,20 +2504,8 @@ "dependencies": { "domelementtype": "^2.3.0", "domhandler": "^5.0.3", - "domutils": "^3.2.2", - "entities": "^7.0.1" - } - }, - "node_modules/htmlparser2/node_modules/entities": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", - "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=0.12" - }, - "funding": { - "url": "https://github.com/fb55/entities?sponsor=1" + "domutils": "^3.1.0", + "entities": "^4.5.0" } }, "node_modules/http-errors": { @@ -4052,38 +3708,17 @@ } }, "node_modules/typescript": { - "version": "7.0.2", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-7.0.2.tgz", - "integrity": "sha512-8FYau96o3NKOhbjKi/qNvG/W5jhzxkbdm5sj9AbZ/5T5sWqn3hJgLfGx27sRKZWTvyzCP8dLRBTf5tBTSRVUNA==", + "version": "6.0.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", + "integrity": "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw==", "dev": true, "license": "Apache-2.0", "bin": { - "tsc": "bin/tsc" + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" }, "engines": { - "node": ">=16.20.0" - }, - "optionalDependencies": { - "@typescript/typescript-aix-ppc64": "7.0.2", - "@typescript/typescript-darwin-arm64": "7.0.2", - "@typescript/typescript-darwin-x64": "7.0.2", - "@typescript/typescript-freebsd-arm64": "7.0.2", - "@typescript/typescript-freebsd-x64": "7.0.2", - "@typescript/typescript-linux-arm": "7.0.2", - "@typescript/typescript-linux-arm64": "7.0.2", - "@typescript/typescript-linux-loong64": "7.0.2", - "@typescript/typescript-linux-mips64el": "7.0.2", - "@typescript/typescript-linux-ppc64": "7.0.2", - "@typescript/typescript-linux-riscv64": "7.0.2", - "@typescript/typescript-linux-s390x": "7.0.2", - "@typescript/typescript-linux-x64": "7.0.2", - "@typescript/typescript-netbsd-arm64": "7.0.2", - "@typescript/typescript-netbsd-x64": "7.0.2", - "@typescript/typescript-openbsd-arm64": "7.0.2", - "@typescript/typescript-openbsd-x64": "7.0.2", - "@typescript/typescript-sunos-x64": "7.0.2", - "@typescript/typescript-win32-arm64": "7.0.2", - "@typescript/typescript-win32-x64": "7.0.2" + "node": ">=14.17" } }, "node_modules/typescript-eslint": { @@ -4291,12 +3926,12 @@ } }, "node_modules/undici": { - "version": "7.28.0", - "resolved": "https://registry.npmjs.org/undici/-/undici-7.28.0.tgz", - "integrity": "sha512-cRZYrTDwWznlnRiPjggAGxZXanty6M8RV1ff8Wm4LWXBp7/IG8v5DnOm74DtUBp9OONpK75YlPnIjQqX0dBDtA==", + "version": "6.28.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-6.28.0.tgz", + "integrity": "sha512-LIY910g9TI13YS95lrMFrs8Rm/u/irgHeTWoKCoteeJ04CUJ92eEfj0rVn+7VKMPBpUPiUoBKfhNyLI23EE/KA==", "license": "MIT", "engines": { - "node": ">=20.18.1" + "node": ">=18.17" } }, "node_modules/undici-types": { diff --git a/package.json b/package.json index 613946a..f9f38cc 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "agent-search-mcp", "mcpName": "io.github.lennney/agent-search-mcp", "version": "3.1.3", - "description": "Zero-config MCP web search for AI agents: 12 engines (8 free, no API key), Chinese search, waterfall multi-source verification, content extraction, news search & Streamable HTTP. Works with Claude Code, Cursor, Codex.", + "description": "Zero-config MCP web search for AI agents: 12 adapters (8 zero-key), native Chinese search, multi-source aggregation, extraction, news & Streamable HTTP. Works with Claude Code, Cursor, Codex.", "type": "module", "main": "./dist/index.js", "bin": { @@ -65,9 +65,12 @@ "semantic-rerank" ], "scripts": { - "build": "tsc && mkdir -p dist/aggregation && cp src/aggregation/semantic_bridge.py dist/aggregation/semantic_bridge.py && node -e \"require('fs').chmodSync('dist/index.js', '755')\"", + "build": "tsc && node scripts/build.mjs", "test": "vitest run", "test:watch": "vitest", + "benchmark": "node benchmarks/run.mjs --fixture benchmarks/fixtures/format-regression.json", + "benchmark:capture": "npm run build && node benchmarks/run.mjs --capture benchmarks/fixtures/live-latest.json", + "benchmark:verify": "npm run build && node benchmarks/run.mjs --fixture benchmarks/fixtures/format-regression.json --check", "lint": "eslint src/", "lint:fix": "eslint src/ --fix", "watch": "tsc --watch", @@ -82,7 +85,7 @@ }, "dependencies": { "@modelcontextprotocol/sdk": "^1.29.0", - "cheerio": "^1.2.0", + "cheerio": "1.0.0", "pino": "^10.3.1", "yaml": "^2.7.1", "zod": "^4.4.3" @@ -91,7 +94,8 @@ "@eslint/js": "^10.0.1", "@types/node": "^26.1.1", "eslint": "^10.7.0", - "typescript": "^7.0.2", + "gpt-tokenizer": "3.4.0", + "typescript": "^6.0.3", "typescript-eslint": "^8.65.0", "vitest": "^3.1.3" } diff --git a/scripts/build.mjs b/scripts/build.mjs new file mode 100644 index 0000000..3882748 --- /dev/null +++ b/scripts/build.mjs @@ -0,0 +1,12 @@ +import { chmodSync, copyFileSync, mkdirSync } from 'node:fs'; +import { resolve } from 'node:path'; + +const root = process.cwd(); +const aggregationDir = resolve(root, 'dist', 'aggregation'); +const semanticBridgeSource = resolve(root, 'src', 'aggregation', 'semantic_bridge.py'); +const semanticBridgeTarget = resolve(aggregationDir, 'semantic_bridge.py'); +const executable = resolve(root, 'dist', 'index.js'); + +mkdirSync(aggregationDir, { recursive: true }); +copyFileSync(semanticBridgeSource, semanticBridgeTarget); +chmodSync(executable, 0o755); diff --git a/server.json b/server.json index 15d3771..5e54bd1 100644 --- a/server.json +++ b/server.json @@ -1,7 +1,7 @@ { "$schema": "https://static.modelcontextprotocol.io/schemas/2025-12-11/server.schema.json", "name": "io.github.lennney/agent-search-mcp", - "description": "11-engine MCP search server for AI agents. Free, zero API keys needed. Waterfall, verify, extract.", + "description": "MCP web search with 12 adapters, 8 zero-key sources, native Chinese search, aggregation, news, and extraction.", "repository": { "url": "https://github.com/lennney/agent-search-mcp", "source": "github" diff --git a/src/aggregation/dedup.ts b/src/aggregation/dedup.ts index 1ccd549..13e7a98 100644 --- a/src/aggregation/dedup.ts +++ b/src/aggregation/dedup.ts @@ -50,13 +50,16 @@ export function dedupByUrl(results: SearchResult[]): { results: SearchResult[]; frequencies.set(key, (frequencies.get(key) || 0) + 1); if (!seen.has(key)) { - seen.set(key, r); + seen.set(key, { ...r, engines: [...new Set(r.engines || [r.source])] }); } else { // From ddgs: keep the item with longer body (richer content) const existing = seen.get(key)!; - if ((r.snippet?.length || 0) > (existing.snippet?.length || 0)) { - seen.set(key, r); - } + const richer = (r.snippet?.length || 0) > (existing.snippet?.length || 0) ? r : existing; + const engines = [...new Set([ + ...(existing.engines || [existing.source]), + ...(r.engines || [r.source]), + ].filter(Boolean))]; + seen.set(key, { ...richer, engines }); } } diff --git a/src/aggregation/format.ts b/src/aggregation/format.ts index f948f54..6cd61c9 100644 --- a/src/aggregation/format.ts +++ b/src/aggregation/format.ts @@ -4,7 +4,6 @@ import type { SecurityProcessedResult } from '../infrastructure/security.js'; const TITLE_MAX = 100; const TITLE_MAX_CN = 150; const DEFAULT_SNIPPET_MAX = 200; -const DEFAULT_SNIPPET_MAX_CN = 300; const CJK_RE = /[\u4E00-\u9FFF\u3400-\u4DBF\uF900-\uFAFF]/; export function isChinese(text: string): boolean { @@ -58,8 +57,10 @@ export interface FormatOptions { snippetMax?: number; /** Max full results before remaining are compacted (compact mode only, default: 3) */ maxFullResults?: number; - /** Minimum confidence threshold for filtering (compact mode only, 0.0-3.0, default: 0 = no filtering) */ + /** Minimum source-reliability confidence (0.0-1.0, default: 0 = no filtering) */ minConfidence?: number; + /** Minimum number of independent sources (default: 1) */ + minSourceCount?: number; } interface FormattedResult { @@ -67,6 +68,9 @@ interface FormattedResult { url: string; snippet?: string; confidence?: number; + relevance?: number; + source_count?: number; + sources?: string[]; security?: { injection_detected: boolean; url_safe: boolean; @@ -103,6 +107,7 @@ export function formatResults(results: ScoredResult[], options?: FormatOptions): const snippetMax = clampSnippet(options?.snippetMax); const maxFullResults = options?.maxFullResults; const minConfidence = options?.minConfidence; + const minSourceCount = options?.minSourceCount; const secured = results.map(r => processResultSecurity(r)); @@ -113,6 +118,11 @@ export function formatResults(results: ScoredResult[], options?: FormatOptions): filteredResults = secured.filter(r => r.confidence >= minConfidence); filteredCount = secured.length - filteredResults.length; } + if (style === 'compact' && minSourceCount !== undefined && minSourceCount > 1) { + const beforeSourceFilter = filteredResults.length; + filteredResults = filteredResults.filter(r => r.source_count >= minSourceCount); + filteredCount += beforeSourceFilter - filteredResults.length; + } // Progressive disclosure (compact mode only) let compactedCount = 0; @@ -122,6 +132,9 @@ export function formatResults(results: ScoredResult[], options?: FormatOptions): url: r.url, snippet: truncateAtSentence(r.snippet, isChinese(r.snippet) ? snippetMax.cn : snippetMax.en), confidence: style === 'compact' ? Math.round(r.confidence * 100) / 100 : r.confidence, + relevance: style === 'compact' ? Math.round(r.relevance * 100) / 100 : r.relevance, + source_count: r.source_count, + sources: r.engines || [r.source], ...(r.security.injectionDetected || !r.security.urlSafe ? { security: { injection_detected: r.security.injectionDetected, @@ -166,7 +179,7 @@ export function formatResults(results: ScoredResult[], options?: FormatOptions): filtered_total?: number; } = { total: results.length, - high_confidence: results.filter(r => r.confidence >= 2).length, + high_confidence: results.filter(r => r.confidence >= 0.8).length, engines: [...new Set(results.flatMap(r => r.engines || [r.source]))], }; diff --git a/src/aggregation/scorer.ts b/src/aggregation/scorer.ts index a05f2f7..850dac4 100644 --- a/src/aggregation/scorer.ts +++ b/src/aggregation/scorer.ts @@ -64,7 +64,13 @@ function getDomainBoost(url: string): number { } export interface ScoredResult extends SearchResult { + /** Source-reliability confidence, independent of query relevance (0-1). */ confidence: number; + /** Query relevance score (0-1). */ + relevance: number; + /** Number of independent adapters that returned this result. */ + source_count: number; + /** @deprecated Use relevance. Retained for backward compatibility. */ score: number; } @@ -80,18 +86,23 @@ export function scoreAndRank( ): ScoredResult[] { const tokens = query.toLowerCase().split(/\W+/).filter(t => t.length >= 3); - // Calculate max possible weight for normalization - const maxWeightSum = Math.max(...Object.values(weights), 0.5) * Math.max(tokens.length, 1); - return results .map(r => { const normalizedUrl = normalizeUrl(r.url); const freq = frequencies?.get(normalizedUrl) || 1; + const engines = [...new Set((r.engines || [r.source]).filter(Boolean))]; + // Frequency can include duplicate rows from one adapter; source_count is + // intentionally limited to distinct adapter provenance. + const sourceCount = Math.max(engines.length, 1); + const relevance = calculateRelevance(r, tokens, weights, freq); return { ...r, - confidence: calculateWeightedConfidence(r, weights, maxWeightSum), - score: calculateScore(r, tokens, weights, freq), + engines, + confidence: calculateWeightedConfidence(r, weights, sourceCount), + relevance, + source_count: sourceCount, + score: relevance, }; }) .sort((a, b) => { @@ -103,8 +114,8 @@ export function scoreAndRank( } /** - * Calculate weighted confidence score (0-1) based on engine weights. - * Instead of raw engine count, uses sum of weights / max possible weight. + * Calculate confidence (0-1) from source reliability and corroboration only. + * Query-token matches intentionally do not affect this signal. * * Example: Brave (0.95) + Exa (0.92) = (0.95+0.92)/max_possible * vs Sogou (0.80) + Baidu (0.75) = (0.80+0.75)/max_possible @@ -113,27 +124,16 @@ export function scoreAndRank( function calculateWeightedConfidence( result: SearchResult, weights: Record, - maxWeightSum: number + sourceCount: number ): number { - const engines = result.engines || []; + const engines = [...new Set((result.engines || [result.source]).filter(Boolean))]; if (engines.length === 0) { - // No engine info, use source weight as fallback - const sourceWeight = weights[result.source] || 0.5; - return sourceWeight * 0.5; // Lower confidence for unknown source + return 0.5; } - - // Sum weights for engines that returned this result - const weightSum = engines.reduce((sum, engine) => { - return sum + (weights[engine] || 0.5); - }, 0); - - // Normalize: divide by max possible weight sum (considering count) - const normalizedConfidence = Math.min(weightSum / (maxWeightSum * engines.length), 1.0); - - // Apply count bonus (more engines still matters, but with diminishing returns) - const countBonus = Math.min(engines.length * 0.1, 0.3); - - return Math.min(normalizedConfidence + countBonus, 1.0); + + const reliability = engines.reduce((sum, engine) => sum + (weights[engine] || 0.5), 0) / engines.length; + const corroborationBonus = Math.min(Math.max(sourceCount - 1, 0) * 0.08, 0.2); + return Math.min(reliability + corroborationBonus, 1.0); } function normalizeUrl(url: string): string { @@ -157,7 +157,7 @@ function normalizeUrl(url: string): string { * * Then multiply by frequency bonus and engine weight. */ -function calculateScore( +function calculateRelevance( result: SearchResult, tokens: string[], weights: Record, diff --git a/src/aggregation/semantic.ts b/src/aggregation/semantic.ts index ef7e1e1..7c1bbda 100644 --- a/src/aggregation/semantic.ts +++ b/src/aggregation/semantic.ts @@ -23,7 +23,7 @@ export interface SemanticOptions { // ── Bridge process singleton ──────────────────────────────────────────────── let _process: ChildProcess | null = null; -let _pending = new Map void; reject: (e: Error) => void; timer: NodeJS.Timeout }>(); +const _pending = new Map void; reject: (e: Error) => void; timer: NodeJS.Timeout }>(); let _requestId = 0; let _availability: boolean | null = null; // null = unchecked let _spawnLock = false; // prevents concurrent spawn attempts diff --git a/src/cli.ts b/src/cli.ts index a63e86a..73b8a13 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -26,7 +26,7 @@ export interface CliArgs { } const VALID_COMMANDS = ['search', 'extract', 'serve']; -const VALID_ENGINES: SearchProvider[] = ['duckduckgo', 'sogou', 'bing', 'baidu', 'brave', 'tavily', 'exa', 'youcom']; +const VALID_ENGINES: SearchProvider[] = ['duckduckgo', 'sogou', 'bing', 'baidu', 'wikipedia', 'startpage', 'yandex', 'mojeek', 'brave', 'tavily', 'exa', 'youcom']; export function parseArgs(argv: string[]): CliArgs { const args = argv.slice(2); // skip node and script path @@ -98,7 +98,7 @@ Usage: Search Options: --count Number of results (1-50, default: 10) - --engines Comma-separated engines (duckduckgo,sogou,bing,baidu,brave,tavily,exa,youcom) + --engines Comma-separated engines (all 12 adapters supported) --json Output as JSON --proxy HTTP proxy URL (e.g., http://127.0.0.1:7890) @@ -192,6 +192,8 @@ async function main(): Promise { port, enableCors: config.enableCors, corsOrigin: config.corsOrigin, + allowedOrigins: config.allowedOrigins, + authToken: config.httpAuthToken, }); await server.listen(); diff --git a/src/index.ts b/src/index.ts index daa4357..bbb2c6d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -23,7 +23,7 @@ async function main() { const server = new McpServer( { name: 'agent-search-mcp', - version: '3.1.1', + version: '3.1.3', }, { capabilities: { @@ -60,10 +60,17 @@ async function main() { } if (config.mode === 'http' || config.mode === 'both') { + if (!config.httpAuthToken && !config.httpAllowUnauthenticated) { + throw new Error( + 'HTTP mode requires HTTP_AUTH_TOKEN. Set HTTP_ALLOW_UNAUTHENTICATED=true only on a trusted local network.' + ); + } const httpServer = createHttpServer(server, { port: config.port, enableCors: config.enableCors, corsOrigin: config.corsOrigin, + allowedOrigins: config.allowedOrigins, + authToken: config.httpAuthToken, }); await httpServer.listen(); console.error('✅ agent-search-mcp ready (HTTP)'); diff --git a/src/infrastructure/config.ts b/src/infrastructure/config.ts index 8c6fdad..cc0bf48 100644 --- a/src/infrastructure/config.ts +++ b/src/infrastructure/config.ts @@ -3,6 +3,9 @@ export interface Config { port: number; enableCors: boolean; corsOrigin: string; + allowedOrigins: string[]; + httpAuthToken: string; + httpAllowUnauthenticated: boolean; useProxy: boolean; proxyUrl: string; defaultEngine: string; @@ -15,6 +18,7 @@ export interface Config { snippetLength: number; maxFullResults: number; minConfidence: number; + minSourceCount: number; semanticDedup: boolean; dedupThreshold: number; dedupModel: string; @@ -29,12 +33,20 @@ export function loadConfig(): Config { const rawPort = parseInt(process.env.PORT || '3000', 10); const port = Number.isFinite(rawPort) && rawPort > 0 ? rawPort : 3000; + const legacyMinConfidence = parseFloat(process.env.MIN_CONFIDENCE || '0') || 0; + const explicitMinSourceCount = parseInt(process.env.MIN_SOURCE_COUNT || '', 10); return { mode, port, enableCors: process.env.ENABLE_CORS === 'true', corsOrigin: process.env.CORS_ORIGIN || '*', + allowedOrigins: (process.env.ALLOWED_ORIGINS || process.env.CORS_ORIGIN || '') + .split(',') + .map(origin => origin.trim()) + .filter(Boolean), + httpAuthToken: process.env.HTTP_AUTH_TOKEN || '', + httpAllowUnauthenticated: process.env.HTTP_ALLOW_UNAUTHENTICATED === 'true', useProxy: process.env.USE_PROXY === 'true', proxyUrl: process.env.PROXY_URL || 'http://127.0.0.1:7890', defaultEngine: process.env.DEFAULT_ENGINE || 'duckduckgo', @@ -52,7 +64,10 @@ export function loadConfig(): Config { outputStyle: process.env.OUTPUT_STYLE === 'compact' ? 'compact' : 'normal', snippetLength: parseInt(process.env.SNIPPET_LENGTH || '200', 10) || 200, maxFullResults: parseInt(process.env.MAX_FULL_RESULTS || '3', 10) || 3, - minConfidence: parseFloat(process.env.MIN_CONFIDENCE || '0') || 0, + minConfidence: legacyMinConfidence <= 1 ? Math.max(legacyMinConfidence, 0) : 0, + minSourceCount: Math.min(12, Number.isFinite(explicitMinSourceCount) + ? Math.max(explicitMinSourceCount, 1) + : legacyMinConfidence > 1 ? Math.ceil(legacyMinConfidence) : 1), semanticDedup: process.env.SEMANTIC_DEDUP === 'true', dedupThreshold: parseFloat(process.env.DEDUP_THRESHOLD || '0.85') || 0.85, dedupModel: process.env.DEDUP_MODEL || 'minishlab/M2V_base_output', diff --git a/src/infrastructure/health.ts b/src/infrastructure/health.ts index 5a6d91a..9da6bd7 100644 --- a/src/infrastructure/health.ts +++ b/src/infrastructure/health.ts @@ -1,4 +1,5 @@ import { SearchCache } from './cache.js'; +import { logger } from './logger.js'; export interface ServerMetricsData { /** Server uptime in seconds */ @@ -117,7 +118,7 @@ export class HealthTracker { h.circuitState = 'closed'; h.circuitOpenedAt = null; h.circuitCooldownMs = HealthTracker.INITIAL_COOLDOWN_MS; - console.log(`[Health] Circuit CLOSED for ${provider} (recovered, errors: ${h.errorCount})`); + logger.info({ provider, errors: h.errorCount }, 'Health circuit closed after recovery'); } h.isHealthy = this.calculateHealth(h); @@ -132,7 +133,7 @@ export class HealthTracker { if (h.errorCount >= HealthTracker.FAILURE_THRESHOLD && h.circuitState === 'closed') { h.circuitState = 'open'; h.circuitOpenedAt = Date.now(); - console.log(`[Health] Circuit OPENED for ${provider} (errors: ${h.errorCount})`); + logger.warn({ provider, errors: h.errorCount }, 'Health circuit opened'); } // If half-open and failed again, re-open with longer cooldown @@ -140,7 +141,10 @@ export class HealthTracker { h.circuitState = 'open'; h.circuitOpenedAt = Date.now(); h.circuitCooldownMs = Math.min(h.circuitCooldownMs * 2, HealthTracker.MAX_COOLDOWN_MS); - console.log(`[Health] Circuit RE-OPENED for ${provider} (cooldown: ${h.circuitCooldownMs}ms)`); + logger.warn( + { provider, cooldownMs: h.circuitCooldownMs }, + 'Health circuit re-opened after failed probe' + ); } h.isHealthy = this.calculateHealth(h); @@ -159,7 +163,7 @@ export class HealthTracker { const elapsed = Date.now() - h.circuitOpenedAt; if (elapsed >= h.circuitCooldownMs) { h.circuitState = 'half-open'; - console.log(`[Health] Circuit HALF-OPEN for ${provider} (testing recovery)`); + logger.info({ provider }, 'Health circuit half-open; testing recovery'); return true; // Allow one test request } return false; // Still in cooldown diff --git a/src/infrastructure/http.ts b/src/infrastructure/http.ts index 89e98cc..b7b3b5e 100644 --- a/src/infrastructure/http.ts +++ b/src/infrastructure/http.ts @@ -1,4 +1,5 @@ import * as http from 'node:http'; +import { timingSafeEqual } from 'node:crypto'; import { StreamableHTTPServerTransport } from '@modelcontextprotocol/sdk/server/streamableHttp.js'; import type { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; @@ -6,6 +7,8 @@ export interface HttpServerOptions { port: number; enableCors: boolean; corsOrigin: string; + allowedOrigins?: string[]; + authToken?: string; } export interface HttpServer { @@ -26,7 +29,8 @@ export interface HttpServer { * - Only health check endpoint is available */ export function createHttpServer(mcpServer: McpServer | null, options: HttpServerOptions): HttpServer { - const { port, enableCors, corsOrigin } = options; + const { port, enableCors, corsOrigin, authToken = '' } = options; + const allowedOrigins = options.allowedOrigins ?? (corsOrigin ? [corsOrigin] : []); let transport: StreamableHTTPServerTransport | null = null; @@ -41,10 +45,19 @@ export function createHttpServer(mcpServer: McpServer | null, options: HttpServe req.on('error', () => { /* swallow */ }); res.on('error', () => { /* swallow */ }); - // CORS headers - if (enableCors) { - res.setHeader('Access-Control-Allow-Origin', corsOrigin); - res.setHeader('Access-Control-Allow-Headers', 'Content-Type, Mcp-Session-Id'); + const requestOrigin = req.headers.origin; + const originAllowed = !requestOrigin || allowedOrigins.includes('*') || allowedOrigins.includes(requestOrigin); + if (!originAllowed) { + res.writeHead(403, { 'Content-Type': 'application/json' }); + res.end(JSON.stringify({ error: 'Origin not allowed' })); + return; + } + + // CORS headers are emitted only for an allowed browser origin. + if (enableCors && requestOrigin) { + res.setHeader('Access-Control-Allow-Origin', allowedOrigins.includes('*') ? '*' : requestOrigin); + res.setHeader('Vary', 'Origin'); + res.setHeader('Access-Control-Allow-Headers', 'Authorization, Content-Type, Mcp-Session-Id'); res.setHeader('Access-Control-Allow-Methods', 'GET, POST, DELETE, OPTIONS'); res.setHeader('Access-Control-Expose-Headers', 'Mcp-Session-Id'); } @@ -59,12 +72,22 @@ export function createHttpServer(mcpServer: McpServer | null, options: HttpServe // Health check if (req.method === 'GET' && req.url === '/health') { res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ status: 'ok', version: '3.1.1' })); + res.end(JSON.stringify({ status: 'ok', version: '3.1.3' })); + return; + } + + const isMcpRoute = req.url === '/mcp' || req.url?.startsWith('/mcp?'); + if (isMcpRoute && authToken && !hasValidBearerToken(req.headers.authorization, authToken)) { + res.writeHead(401, { + 'Content-Type': 'application/json', + 'WWW-Authenticate': 'Bearer', + }); + res.end(JSON.stringify({ error: 'Unauthorized' })); return; } // MCP Streamable HTTP — route GET/POST/DELETE /mcp to transport - if (transport && (req.url === '/mcp' || req.url?.startsWith('/mcp?'))) { + if (transport && isMcpRoute) { try { await transport.handleRequest(req, res); } catch (err) { @@ -116,8 +139,18 @@ export function createHttpServer(mcpServer: McpServer | null, options: HttpServe if (err) reject(err); else resolve(); }); + // Node.js 18 otherwise waits for the keep-alive timeout before closing. + httpServer.closeIdleConnections(); }); }, getPort: () => actualPort, }; -} \ No newline at end of file +} + +function hasValidBearerToken(authorization: string | undefined, expectedToken: string): boolean { + const match = authorization?.match(/^Bearer\s+(.+)$/i); + if (!match) return false; + const supplied = Buffer.from(match[1]); + const expected = Buffer.from(expectedToken); + return supplied.length === expected.length && timingSafeEqual(supplied, expected); +} diff --git a/src/infrastructure/security.ts b/src/infrastructure/security.ts index dbfe79e..be2739d 100644 --- a/src/infrastructure/security.ts +++ b/src/infrastructure/security.ts @@ -189,6 +189,10 @@ export interface SecurityProcessedResult { url: string; snippet: string; confidence: number; + relevance: number; + source_count: number; + source: string; + engines?: string[]; security: { injectionDetected: boolean; urlSafe: boolean; @@ -206,6 +210,10 @@ export function processResultSecurity(result: { url: string; snippet: string; confidence: number; + relevance?: number; + source_count?: number; + source?: string; + engines?: string[]; }): SecurityProcessedResult { // Check snippet for injection const injectionResult = checkSnippetInjection(result.snippet); @@ -224,6 +232,10 @@ export function processResultSecurity(result: { url: result.url, snippet: injectionResult.clean ? result.snippet : injectionResult.snippet, confidence: result.confidence, + relevance: result.relevance ?? 0, + source_count: result.source_count ?? 1, + source: result.source ?? 'unknown', + engines: result.engines, security: { injectionDetected: allThreats.length > 0, urlSafe: urlResult.safe, diff --git a/src/tools/capabilities.ts b/src/tools/capabilities.ts index 2ac8412..3a38330 100644 --- a/src/tools/capabilities.ts +++ b/src/tools/capabilities.ts @@ -10,13 +10,13 @@ export function registerCapabilities(server: McpServer) { **Agent Search MCP** — Free & open-source multi-engine MCP search server. - GitHub: https://github.com/lennney/agent-search-mcp ⭐ - npm: npx agent-search-mcp / npm install -g agent-search-mcp -- Version: 3.1.1 | MIT | 5 deps | 448+ tests +- Version: 3.1.3 | Apache-2.0 | 5 runtime deps | 498+ tests ## Quick Usage free_search(query) — search the web for free ## High Quality -free_search_advanced(query, min_confidence=2) — verified results only +free_search_advanced(query) — filtering, waterfall search, and optional enrichment ## Smart Answer search_with_synthesis(query) — deep search with waterfall verification + prompt hint for LLM synthesis @@ -30,11 +30,9 @@ free_search_advanced(query, language="zh") — Chinese sources ## Content Extraction free_extract(url) — get full page as markdown -## Confidence Scores -Each result has confidence 1-3 based on multi-source verification. -- 1: Single source -- 2: Verified by 2 sources (recommended) -- 3: Highly verified by 3+ sources +## Result Signals +Each result exposes relevance (query match), confidence (source reliability and corroboration), and source_count (independent engines). +Use these as ranking and verification aids; inspect result URLs before treating a claim as verified. ## Engines - DuckDuckGo (free) @@ -44,6 +42,7 @@ Each result has confidence 1-3 based on multi-source verification. - Brave Search (paid, 2000 free/month) - Tavily (paid, 1000 free/month) - Exa (paid) +- You.com (paid) - Yandex (free, Russian) - Mojeek (free, privacy-focused) - Wikipedia (free) diff --git a/src/tools/fetch-tools.ts b/src/tools/fetch-tools.ts index c2b339f..0f7c9ef 100644 --- a/src/tools/fetch-tools.ts +++ b/src/tools/fetch-tools.ts @@ -2,6 +2,26 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; import { logger } from '../infrastructure/index.js'; +function parseCsdnArticleUrl(url: string): URL { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw new Error('Invalid CSDN article URL: expected an HTTPS blog.csdn.net URL'); + } + + const isAllowed = + parsed.protocol === 'https:' && + parsed.hostname === 'blog.csdn.net' && + (parsed.port === '' || parsed.port === '443'); + + if (!isAllowed) { + throw new Error('Invalid CSDN article URL: expected an HTTPS blog.csdn.net URL'); + } + + return parsed; +} + /** * Extract GitHub README content from a repository URL. */ @@ -60,10 +80,12 @@ export async function fetchGithubReadme(url: string): Promise { */ export async function fetchCsdnArticle(url: string): Promise { try { - const response = await fetch(url, { + const safeUrl = parseCsdnArticleUrl(url); + const response = await fetch(safeUrl.href, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36', }, + redirect: 'error', signal: AbortSignal.timeout(10000), }); @@ -186,7 +208,7 @@ export function setupFetchCsdnArticle(server: McpServer): void { 'Not recommended for: Other Chinese sites — use free_extract instead.\n\n' + '@readOnly true @idempotent true — makes outbound HTTP requests to the CSDN article URL.', inputSchema: { - url: z.string().url('Must be a valid URL').describe('CSDN article URL'), + url: z.string().url('Must be a valid URL').describe('HTTPS article URL on blog.csdn.net'), }, annotations: { readOnlyHint: true, idempotentHint: true }, }, diff --git a/src/tools/free-search-advanced.ts b/src/tools/free-search-advanced.ts index 18de38f..b808b91 100644 --- a/src/tools/free-search-advanced.ts +++ b/src/tools/free-search-advanced.ts @@ -17,8 +17,10 @@ Not recommended for: Simple queries — use free_search instead. inputSchema: { query: z.string().describe('Search query'), count: z.number().optional().default(5).describe('Number of results (1-20)'), - min_confidence: z.number().min(1).max(3).optional().default(1) - .describe('Only return results verified by N+ sources'), + min_confidence: z.number().min(0).max(3).optional().default(0) + .describe('Minimum source-reliability confidence (0-1). Legacy values 2-3 are treated as min_source_count.'), + min_source_count: z.number().int().min(1).max(12).optional().default(1) + .describe('Minimum number of independent adapters that returned the result'), time_range: z.enum(['day', 'week', 'month', 'year']).optional() .describe('Filter by recency'), language: z.enum(['auto', 'en', 'zh']).optional().default('auto') @@ -42,11 +44,13 @@ Not recommended for: Simple queries — use free_search instead. }, async (input) => { try { + const legacySourceCount = input.min_confidence > 1 ? Math.ceil(input.min_confidence) : 1; const results = await searchWithFallback({ query: input.query, count: input.count, - engines: ['duckduckgo', 'sogou', 'bing', 'baidu', 'brave', 'tavily'], - minConfidence: input.min_confidence, + engines: ['duckduckgo', 'sogou', 'bing', 'baidu', 'wikipedia', 'startpage', 'yandex', 'mojeek', 'brave', 'tavily', 'exa', 'youcom'], + minConfidence: input.min_confidence <= 1 ? input.min_confidence : 0, + minSourceCount: Math.max(input.min_source_count, legacySourceCount), language: input.language, includeDomains: input.include_domains, excludeDomains: input.exclude_domains, diff --git a/src/tools/free-search.ts b/src/tools/free-search.ts index 7fd34b8..6b684c7 100644 --- a/src/tools/free-search.ts +++ b/src/tools/free-search.ts @@ -1,6 +1,6 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js'; import { z } from 'zod'; -import { searchDuckDuckGo, isDdgsAvailable } from '../engines/duckduckgo.js'; +import { searchDuckDuckGo } from '../engines/duckduckgo.js'; import { searchSogou } from '../engines/sogou.js'; import { searchBing } from '../engines/bing.js'; import { searchBaidu } from '../engines/baidu.js'; @@ -8,6 +8,10 @@ import { BraveProvider } from '../engines/brave.js'; import { TavilyProvider } from '../engines/tavily.js'; import { searchExa } from '../engines/exa.js'; import { searchYouCom } from '../engines/youcom.js'; +import { searchWikipedia } from '../engines/wikipedia.js'; +import { searchStartpage } from '../engines/startpage.js'; +import { searchYandex } from '../engines/yandex.js'; +import { searchMojeek } from '../engines/mojeek.js'; import { getSecurityNote } from '../infrastructure/security.js'; // ── Agent instruction: DO NOT TOUCH ─────────────────────────────────── import type { SearchResult, SearchProvider, EngineError } from '../types.js'; @@ -15,8 +19,7 @@ import { dedupByUrl, dedupByTitle, filterLowQuality, scoreAndRank, formatResults import type { FormatOptions } from '../aggregation/format.js'; import { SearchCache, logger, HealthTracker, RateLimiter, loadConfig, EnginePolicy, ServerMetrics } from '../infrastructure/index.js'; -const ALL_ENGINES: SearchProvider[] = ['duckduckgo', 'sogou', 'bing', 'baidu', 'brave', 'tavily', 'exa', 'youcom']; -const FREE_ENGINES: SearchProvider[] = ['duckduckgo', 'sogou', 'bing', 'baidu']; +const FREE_ENGINES: SearchProvider[] = ['duckduckgo', 'sogou', 'bing', 'baidu', 'wikipedia', 'startpage', 'yandex', 'mojeek']; const PAID_ENGINES: SearchProvider[] = ['brave', 'tavily', 'exa', 'youcom']; // Engine weights (higher = more trusted) @@ -25,6 +28,10 @@ const ENGINE_WEIGHTS: Record = { sogou: 0.8, bing: 0.9, baidu: 0.75, + wikipedia: 0.93, + startpage: 0.86, + yandex: 0.82, + mojeek: 0.8, brave: 0.95, tavily: 0.9, exa: 0.92, @@ -46,6 +53,10 @@ const PROVIDER_MAP: Record = { sogou: 'sogou', bing: 'bing', baidu: 'baidu', + wikipedia: 'wikipedia', + startpage: 'startpage', + yandex: 'yandex', + mojeek: 'mojeek', brave: 'brave', tavily: 'tavily', exa: 'exa', @@ -95,12 +106,6 @@ async function searchEngine( // Rate limit before making the request await rateLimiter.waitForSlot(engine); - // DDG-specific: throw early if ddgs is not available, so Promise.allSettled - // records it as a rejection → partialFailures gets the correct engine name - if (engine === 'duckduckgo' && !isDdgsAvailable()) { - throw new Error('DuckDuckGo unavailable: Python ddgs library not installed. Install with: pip install ddgs'); - } - let lastError: Error | null = null; for (let attempt = 0; attempt <= maxRetries; attempt++) { @@ -120,6 +125,18 @@ async function searchEngine( case 'baidu': results = await searchBaidu(query, limit); break; + case 'wikipedia': + results = await searchWikipedia(query, limit); + break; + case 'startpage': + results = await searchStartpage(query, limit); + break; + case 'yandex': + results = await searchYandex(query, limit); + break; + case 'mojeek': + results = await searchMojeek(query, limit); + break; case 'brave': results = await new BraveProvider().search(query, limit); break; @@ -226,8 +243,9 @@ function hasApiKey(engine: SearchProvider): boolean { case 'exa': return !!process.env.EXA_API_KEY; case 'youcom': + return !!process.env.YDC_API_KEY; default: - return true; // free engines always available + return true; // zero-key engines are always eligible } } @@ -237,6 +255,7 @@ export interface SearchWithFallbackOptions { count?: number; engines?: SearchProvider[]; minConfidence?: number; + minSourceCount?: number; language?: string; includeDomains?: string[]; excludeDomains?: string[]; @@ -246,13 +265,18 @@ export interface SearchWithFallbackOptions { enrich?: boolean; enrichMax?: number; enrichMinConfidence?: number; + /** Disable query-expansion recursion for deterministic benchmark capture. */ + expandQueries?: boolean; } interface FormattedResult { title: string; url: string; - snippet: string; - confidence: number; + snippet?: string; + confidence?: number; + relevance?: number; + source_count?: number; + sources?: string[]; security?: { injection_detected: boolean; url_safe: boolean; @@ -269,6 +293,13 @@ interface SearchResponse { total: number; high_confidence: number; engines: string[]; + execution?: { + mode: 'parallel' | 'waterfall'; + engine_calls: number; + searched_engines: string[]; + phases_completed: string[]; + early_stop: boolean; + }; }; security_note: string; detected_language?: string; @@ -285,9 +316,29 @@ const pendingRequests = new Map>(); * Generate cache key for request collapsing. */ function makeCollapseKey(options: SearchWithFallbackOptions): string { - const { query, count = 10, engines = [] } = options; - const sortedEngines = [...engines].sort().join(','); - return `${query}:${count}:${sortedEngines}`; + const { + query, count = 10, engines = [], minConfidence = 0, minSourceCount = 1, + language = 'auto', includeDomains = [], excludeDomains = [], waterfall = false, + waterfallMinResults = 3, waterfallMinConfidence = 0.6, + enrich = false, enrichMax, enrichMinConfidence, expandQueries = true, + } = options; + return JSON.stringify({ + query, + count, + engines: [...engines].sort(), + minConfidence, + minSourceCount, + language, + includeDomains: [...includeDomains].sort(), + excludeDomains: [...excludeDomains].sort(), + waterfall, + waterfallMinResults, + waterfallMinConfidence, + enrich, + enrichMax, + enrichMinConfidence, + expandQueries, + }); } // ─── Core search logic (fused patterns from ddgs) ────────────────────── @@ -368,7 +419,8 @@ async function executeParallelSearch(options: SearchWithFallbackOptions): Promis query, count = 10, engines: userEngines = ['duckduckgo', 'sogou'] as SearchProvider[], - minConfidence = 1, + minConfidence = 0, + minSourceCount = 1, language, includeDomains, excludeDomains, @@ -378,7 +430,7 @@ async function executeParallelSearch(options: SearchWithFallbackOptions): Promis logger.info({ query, detectedLang, explicitLang: language }, 'Language detection'); // Check cache first - const cacheKey = cache.makeKey(query, count, userEngines); + const cacheKey = cache.makeKey(makeCollapseKey(options), count, userEngines); const cached = cache.get(cacheKey); if (cached) { logger.info({ query, count, engines: userEngines }, 'Cache hit'); @@ -397,6 +449,9 @@ async function executeParallelSearch(options: SearchWithFallbackOptions): Promis const freeToSearch = uniqueEngines.filter(e => FREE_ENGINES.includes(e)); const allFree = FREE_ENGINES.filter(e => !uniqueEngines.includes(e)); const phase1Engines = [...freeToSearch, ...allFree]; + const paidToSearch = uniqueEngines.filter( + e => PAID_ENGINES.includes(e) && hasApiKey(e) + ); // ── Step 3: Batch concurrency + early exit (from ddgs) ────────────── const BATCH_SIZE = calculateAdaptiveConcurrency(phase1Engines, count); @@ -411,8 +466,8 @@ async function executeParallelSearch(options: SearchWithFallbackOptions): Promis const batch = phase1Engines.slice(i, i + BATCH_SIZE); const batchResults = await Promise.allSettled( batch.map(async (engine) => { - const results = await searchEngine(engine, query, count); searchedEngines.push(engine); + const results = await searchEngine(engine, query, count); return { engine, results }; }) ); @@ -439,18 +494,14 @@ async function executeParallelSearch(options: SearchWithFallbackOptions): Promis // ── Step 4: Fallback to paid engines if not enough ─────────────────── if (allResults.length < count) { - const paidToSearch = uniqueEngines.filter( - e => PAID_ENGINES.includes(e) && hasApiKey(e) - ); - if (paidToSearch.length > 0) { const remaining = Math.max(count - allResults.length, 1); logger.info({ engines: paidToSearch, remaining }, 'Phase 2: paid engines'); const phase2Results = await Promise.allSettled( paidToSearch.map(async (engine) => { - const results = await searchEngine(engine, query, remaining); searchedEngines.push(engine); + const results = await searchEngine(engine, query, remaining); return { engine, results }; }) ); @@ -488,7 +539,7 @@ failures.push( // ── Steps 5e-7: Shared post-processing (semantic + filters + enrich + format) const { formatted } = await applyPostProcessing( - scored, query, minConfidence, + scored, query, minConfidence, minSourceCount, includeDomains, excludeDomains, options.enrich, options.enrichMax, options.enrichMinConfidence, ); @@ -496,8 +547,17 @@ failures.push( const response: SearchResponse = { query, engines: userEngines, - results: formatted.results as any, - meta: formatted.meta, + results: formatted.results, + meta: { + ...formatted.meta, + execution: { + mode: 'parallel', + engine_calls: searchedEngines.length, + searched_engines: [...searchedEngines], + phases_completed: searchedEngines.length > 0 ? ['free', ...(searchedEngines.some(e => PAID_ENGINES.includes(e as SearchProvider)) ? ['optional'] : [])] : [], + early_stop: searchedEngines.length < phase1Engines.length + paidToSearch.length, + }, + }, security_note: formatted.security_note, detected_language: detectedLang, ...(config.outputStyle !== 'compact' ? { @@ -532,6 +592,7 @@ async function applyPostProcessing( scored: ScoredResult[], query: string, minConfidence: number, + minSourceCount: number, includeDomains: string[] | undefined, excludeDomains: string[] | undefined, enrich: boolean | undefined, @@ -556,9 +617,12 @@ async function applyPostProcessing( } // Post-search filters - if (minConfidence > 1) { + if (minConfidence > 0) { scored = scored.filter(r => r.confidence >= minConfidence); } + if (minSourceCount > 1) { + scored = scored.filter(r => r.source_count >= minSourceCount); + } if (includeDomains && includeDomains.length > 0) { scored = scored.filter(r => { @@ -600,6 +664,7 @@ async function applyPostProcessing( snippetMax: config.snippetLength, maxFullResults: config.maxFullResults, minConfidence: config.minConfidence, + minSourceCount: config.minSourceCount, }; const formatted = formatResults(scored, fmtOptions); @@ -609,7 +674,8 @@ async function applyPostProcessing( const WATERFALL_PHASES = { phase1a: ["duckduckgo", "sogou"], phase1b: ["bing", "baidu"], - phase2: ["brave", "tavily", "exa"], + phase1c: ["wikipedia", "startpage", "yandex", "mojeek"], + phase2: ["brave", "tavily", "exa", "youcom"], } as const; async function executeWaterfallSearch(options: SearchWithFallbackOptions, depth: number = 0): Promise { @@ -631,7 +697,8 @@ async function executeWaterfallSearch(options: SearchWithFallbackOptions, depth: language, includeDomains, excludeDomains, - minConfidence = 1, + minConfidence = 0, + minSourceCount = 1, waterfallMinResults = 3, waterfallMinConfidence = 0.6, } = options; @@ -642,16 +709,18 @@ async function executeWaterfallSearch(options: SearchWithFallbackOptions, depth: const allResults: SearchResult[] = []; const allFailures: EngineError[] = []; const searchedEngines: string[] = []; + const phasesCompleted: string[] = []; async function searchBatch(engines: SearchProvider[], phaseLabel: string): Promise { + phasesCompleted.push(phaseLabel); const batchSize = calculateAdaptiveConcurrency(engines, count); for (let i = 0; i < engines.length; i += batchSize) { const batch = engines.slice(i, i + batchSize); const batchResults = await Promise.allSettled( batch.map(async (engine) => { - const results = await searchEngine(engine, query, count); searchedEngines.push(engine); + const results = await searchEngine(engine, query, count); return { engine, results }; }) ); @@ -709,19 +778,27 @@ async function executeWaterfallSearch(options: SearchWithFallbackOptions, depth: if (!basketFull) { basketFull = await searchBatch([...WATERFALL_PHASES.phase1b] as SearchProvider[], "1b"); if (basketFull) { - logger.info("Phase 1b satisfied — skipping Phase 2"); + logger.info("Phase 1b satisfied — skipping remaining phases"); + } + } + + if (!basketFull) { + basketFull = await searchBatch([...WATERFALL_PHASES.phase1c] as SearchProvider[], "1c"); + if (basketFull) { + logger.info("Phase 1c satisfied — skipping Phase 2"); } } if (!basketFull) { const paidAvailable = WATERFALL_PHASES.phase2.filter((e) => hasApiKey(e as SearchProvider)); if (paidAvailable.length > 0) { + phasesCompleted.push('2'); logger.info({ engines: paidAvailable }, "Waterfall Phase 2: paid engines"); const paidResults = await Promise.allSettled( paidAvailable.map(async (engine) => { const remaining = Math.max(count - allResults.length, 1); - const results = await searchEngine(engine as SearchProvider, query, remaining); searchedEngines.push(engine); + const results = await searchEngine(engine as SearchProvider, query, remaining); return { engine, results }; }) ); @@ -741,7 +818,7 @@ async function executeWaterfallSearch(options: SearchWithFallbackOptions, depth: } // ── Phase 3: Query Expansion (if confidence still low) ────────── - if (!basketFull) { + if (!basketFull && options.expandQueries !== false) { // 3a: Chinese query optimization — try character variants first let alternatives: string[] = []; if (hasChinese(query)) { @@ -756,6 +833,7 @@ async function executeWaterfallSearch(options: SearchWithFallbackOptions, depth: alternatives = expandQuery(query); } if (alternatives.length > 0) { + phasesCompleted.push('3'); logger.info({ alternatives }, "Phase 3: query expansion"); for (const altQuery of alternatives) { const altSearch = await executeWaterfallSearch({ @@ -764,12 +842,16 @@ async function executeWaterfallSearch(options: SearchWithFallbackOptions, depth: waterfall: true, enrich: false, }, depth + 1); + const altExecution = altSearch.meta.execution; + if (altExecution) { + searchedEngines.push(...altExecution.searched_engines); + } if (altSearch.results && altSearch.results.length > 0) { for (const r of altSearch.results) { allResults.push({ title: r.title, url: r.url, - snippet: r.snippet, + snippet: r.snippet || '', source: "expanded", engines: altSearch.meta?.engines || [], }); @@ -787,7 +869,7 @@ async function executeWaterfallSearch(options: SearchWithFallbackOptions, depth: // ── Steps 5e-7: Shared post-processing (semantic + filters + enrich + format) const { formatted } = await applyPostProcessing( - scored, query, minConfidence, + scored, query, minConfidence, minSourceCount, includeDomains, excludeDomains, options.enrich, options.enrichMax, options.enrichMinConfidence, ); @@ -796,6 +878,16 @@ async function executeWaterfallSearch(options: SearchWithFallbackOptions, depth: query, engines: searchedEngines, ...formatted, + meta: { + ...formatted.meta, + execution: { + mode: 'waterfall', + engine_calls: searchedEngines.length, + searched_engines: [...searchedEngines], + phases_completed: phasesCompleted, + early_stop: basketFull, + }, + }, detected_language: detectedLang, ...(config.outputStyle !== 'compact' ? { rate_limits: rateLimiter.getAllRateLimits(searchedEngines) } : {}), ...(allFailures.length > 0 ? { partialFailures: allFailures } : {}), @@ -826,16 +918,16 @@ export function setupFreeSearchTool(server: McpServer): void { 'Best for: Quick fact-finding, general search, when date/domain filters are not needed.\n' + 'Not recommended for: Filtered or verified-only results — use free_search_advanced. ' + 'For full page content — use free_extract.\n\n' + - 'Phase 1: DuckDuckGo + Sogou + Bing + Baidu (free, no key required).\n' + - 'Phase 2: Brave + Tavily + Exa (paid, requires BRAVE_API_KEY / TAVILY_API_KEY / EXA_API_KEY env vars).\n' + - 'Results are deduplicated, scored by confidence (1-3), and include security metadata.\n\n' + + 'Phase 1: 8 zero-key adapters (DuckDuckGo, Sogou, Bing, Baidu, Wikipedia, Startpage, Yandex, Mojeek).\n' + + 'Phase 2: Brave + Tavily + Exa + You.com (optional API keys).\n' + + 'Results are deduplicated and include separate confidence, relevance, and source-count signals.\n\n' + '@readOnly true @idempotent true — makes outbound HTTP requests to configured search engines. ' + 'Injection detection and SSRF protection active.', inputSchema: { query: z.string().min(1, 'Search query must not be empty') .describe('Search query string. Use natural language (e.g., "latest AI news 2026"). For Chinese queries, Sogou and Baidu are used automatically.'), limit: z.number().int().min(1).max(50).default(10).describe('Number of results to return (1-50). Default 10. Higher values increase token usage.'), - engines: z.array(z.enum(['duckduckgo', 'sogou', 'bing', 'baidu', 'brave', 'tavily', 'exa', 'youcom'])) + engines: z.array(z.enum(['duckduckgo', 'sogou', 'bing', 'baidu', 'wikipedia', 'startpage', 'yandex', 'mojeek', 'brave', 'tavily', 'exa', 'youcom'])) .min(1) .default(['duckduckgo', 'sogou']) .describe('Search engines to use (default: duckduckgo + sogou). Free engines work without API keys. ' + diff --git a/src/tools/search-with-synthesis.ts b/src/tools/search-with-synthesis.ts index 2ec1c9f..c68e904 100644 --- a/src/tools/search-with-synthesis.ts +++ b/src/tools/search-with-synthesis.ts @@ -42,8 +42,8 @@ export function registerSearchWithSynthesis(server: McpServer) { const results: SynthesisResult[] = rawResults.map((r) => ({ title: r.title, url: r.url, - snippet: r.snippet, - confidence: r.confidence, + snippet: r.snippet || '', + confidence: r.confidence ?? 0, source: engines[0] || 'unknown', })); @@ -69,4 +69,4 @@ export function registerSearchWithSynthesis(server: McpServer) { } } ); -} \ No newline at end of file +} diff --git a/tests/aggregation.test.ts b/tests/aggregation.test.ts index 4baba49..b294550 100644 --- a/tests/aggregation.test.ts +++ b/tests/aggregation.test.ts @@ -55,6 +55,15 @@ describe('dedupByUrl', () => { expect(frequencies.get('example.com/a')).toBe(2); expect(frequencies.get('example.com/b')).toBe(1); }); + + it('merges engine provenance for duplicate URLs', () => { + const { results } = dedupByUrl([ + { title: 'A', url: 'https://example.com/a', snippet: 'first source snippet', source: 'duckduckgo', engines: ['duckduckgo'] }, + { title: 'A', url: 'https://example.com/a', snippet: 'second source snippet is longer', source: 'wikipedia', engines: ['wikipedia'] }, + ]); + + expect(results[0].engines).toEqual(['duckduckgo', 'wikipedia']); + }); }); // ─── normalizeUrl ──────────────────────────────────────────────────────── @@ -198,8 +207,8 @@ describe('formatResults', () => { it('builds meta with total, high_confidence, and unique engines', () => { const results: ScoredResult[] = [ - { title: 'A', url: 'a', snippet: 'A'.repeat(30), source: '', engines: ['duckduckgo', 'sogou'], confidence: 2, score: 0.8 }, - { title: 'B', url: 'b', snippet: 'B'.repeat(30), source: '', engines: ['sogou'], confidence: 1, score: 0.5 }, + { title: 'A', url: 'a', snippet: 'A'.repeat(30), source: '', engines: ['duckduckgo', 'sogou'], confidence: 0.9, relevance: 0.8, source_count: 2, score: 0.8 }, + { title: 'B', url: 'b', snippet: 'B'.repeat(30), source: '', engines: ['sogou'], confidence: 0.5, relevance: 0.5, source_count: 1, score: 0.5 }, ]; const formatted = formatResults(results); expect(formatted.meta.total).toBe(2); diff --git a/tests/aggregation/format.test.ts b/tests/aggregation/format.test.ts index 9af380d..12ee15b 100644 --- a/tests/aggregation/format.test.ts +++ b/tests/aggregation/format.test.ts @@ -4,7 +4,8 @@ import { ScoredResult } from '../../src/aggregation/scorer.js'; // ─── Test helpers ────────────────────────────────────────────────────────── -function makeResult(i: number, confidence: number = 3 - i * 0.3): ScoredResult { +function makeResult(i: number, confidence: number = 0.98 - i * 0.05): ScoredResult { + const relevance = 0.9 - i * 0.05; return { title: `Test Result ${i + 1}`, url: `https://example.com/page/${i + 1}`, @@ -12,13 +13,15 @@ function makeResult(i: number, confidence: number = 3 - i * 0.3): ScoredResult { source: 'duckduckgo', engines: ['duckduckgo'], confidence, - score: 0.9 - i * 0.05, + relevance, + source_count: 1, + score: relevance, }; } function makeResults(n: number, confidences?: number[]): ScoredResult[] { return Array.from({ length: n }, (_, i) => - makeResult(i, confidences?.[i] ?? (3 - i * 0.3)) + makeResult(i, confidences?.[i] ?? (0.98 - i * 0.05)) ); } @@ -155,20 +158,20 @@ describe('formatResults — progressive disclosure', () => { describe('formatResults — confidence filtering', () => { it('filters results below minConfidence in compact mode', () => { - const confidences = [3.0, 2.5, 1.8, 1.2, 0.8, 0.5, 0.3]; + const confidences = [0.95, 0.85, 0.75, 0.65, 0.55, 0.45, 0.35]; const results = makeResults(7, confidences); const formatted = formatResults(results, { style: 'compact', - minConfidence: 1.5, + minConfidence: 0.7, maxFullResults: 10, // no progressive disclosure to isolate filtering }); - // Only results with confidence >= 1.5 should remain: 3.0, 2.5, 1.8 → 3 results + // Only results with confidence >= 0.7 should remain. expect(formatted.results).toHaveLength(3); - expect(formatted.results[0].confidence).toBeCloseTo(3.0); - expect(formatted.results[1].confidence).toBeCloseTo(2.5); - expect(formatted.results[2].confidence).toBeCloseTo(1.8); + expect(formatted.results[0].confidence).toBeCloseTo(0.95); + expect(formatted.results[1].confidence).toBeCloseTo(0.85); + expect(formatted.results[2].confidence).toBeCloseTo(0.75); expect((formatted.meta as any).filtered_count).toBe(4); }); @@ -185,26 +188,26 @@ describe('formatResults — confidence filtering', () => { }); it('does not filter in normal mode', () => { - const results = makeResults(5, [3.0, 2.0, 0.5, 0.3, 0.1]); + const results = makeResults(5, [0.9, 0.8, 0.5, 0.3, 0.1]); const formatted = formatResults(results, { style: 'normal', - minConfidence: 1.0, + minConfidence: 0.8, }); expect(formatted.results).toHaveLength(5); }); it('applies filtering BEFORE progressive disclosure', () => { - const confidences = [3.0, 2.8, 2.5, 2.0, 1.5, 1.0, 0.5, 0.3]; + const confidences = [0.98, 0.9, 0.85, 0.8, 0.75, 0.6, 0.5, 0.3]; const results = makeResults(8, confidences); const formatted = formatResults(results, { style: 'compact', - minConfidence: 1.5, + minConfidence: 0.75, maxFullResults: 3, }); - // After filter: 3.0, 2.8, 2.5, 2.0, 1.5 → 5 results + // After filter: five results remain. // Then progressive: first 3 full, last 2 compacted expect(formatted.results).toHaveLength(5); expect((formatted.meta as any).filtered_count).toBe(3); // 0.5, 0.3 removed @@ -225,7 +228,7 @@ describe('formatResults — confidence filtering', () => { const results = makeResults(3, [0.5, 0.3, 0.1]); const formatted = formatResults(results, { style: 'compact', - minConfidence: 1.0, + minConfidence: 0.8, }); expect(formatted.results).toHaveLength(0); @@ -259,7 +262,7 @@ describe('formatResults — backward compatibility', () => { const formatted = formatResults(results, { style: 'normal', maxFullResults: 3, - minConfidence: 2.0, + minConfidence: 0.8, }); // normal mode ignores progressive disclosure and filtering @@ -267,4 +270,20 @@ describe('formatResults — backward compatibility', () => { const compactedCount = formatted.results.filter((r: any) => r.compacted).length; expect(compactedCount).toBe(0); }); + + it('filters source count independently from confidence', () => { + const results = makeResults(3, [0.9, 0.9, 0.9]); + results[0].source_count = 3; + results[1].source_count = 2; + results[2].source_count = 1; + + const formatted = formatResults(results, { + style: 'compact', + minSourceCount: 2, + maxFullResults: 10, + }); + + expect(formatted.results).toHaveLength(2); + expect(formatted.results.map(result => result.source_count)).toEqual([3, 2]); + }); }); diff --git a/tests/aggregation/scorer.test.ts b/tests/aggregation/scorer.test.ts index 353df3c..e6787c8 100644 --- a/tests/aggregation/scorer.test.ts +++ b/tests/aggregation/scorer.test.ts @@ -88,6 +88,17 @@ describe('scoreAndRank', () => { 'test', weights ); expect(multi[0].confidence).toBeGreaterThan(single[0].confidence); + expect(multi[0].source_count).toBe(3); + expect(single[0].source_count).toBe(1); + }); + + it('does not treat duplicate rows from one adapter as independent sources', () => { + const frequencies = new Map([['example.com/same', 3]]); + const scored = scoreAndRank( + [makeResult({ url: 'https://example.com/same', engines: ['ddg'] })], + 'test', weights, frequencies + ); + expect(scored[0].source_count).toBe(1); }); it('empty tokens returns default score 0.3', () => { @@ -108,6 +119,8 @@ describe('checkConfidenceBasket', () => { snippet: 'snippet', source: 'ddg', confidence, + relevance: 0.5, + source_count: 1, score: 0.5, }; } diff --git a/tests/cli.test.ts b/tests/cli.test.ts index ef6431a..650faaf 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -54,6 +54,11 @@ describe('parseArgs', () => { expect(args.engines).toEqual(['bing', 'baidu', 'youcom']); }); + it('accepts all additional zero-key engines', () => { + const args = parseArgs(['node', 'cli.ts', 'search', 'query', '--engines', 'wikipedia,startpage,yandex,mojeek']); + expect(args.engines).toEqual(['wikipedia', 'startpage', 'yandex', 'mojeek']); + }); + it('parses --proxy flag', () => { const args = parseArgs(['node', 'cli.ts', 'search', 'query', '--proxy', 'http://127.0.0.1:7890']); expect(args.proxy).toBe('http://127.0.0.1:7890'); diff --git a/tests/infrastructure/config.test.ts b/tests/infrastructure/config.test.ts index d96b74d..d58ac43 100644 --- a/tests/infrastructure/config.test.ts +++ b/tests/infrastructure/config.test.ts @@ -85,6 +85,38 @@ describe('loadConfig', () => { expect(config.allowedEngines).toEqual([]); }); + it('treats MIN_CONFIDENCE within 0..1 as confidence', () => { + process.env.MIN_CONFIDENCE = '0.7'; + delete process.env.MIN_SOURCE_COUNT; + const config = loadConfig(); + expect(config.minConfidence).toBe(0.7); + expect(config.minSourceCount).toBe(1); + }); + + it('maps legacy MIN_CONFIDENCE source counts to MIN_SOURCE_COUNT', () => { + process.env.MIN_CONFIDENCE = '2'; + delete process.env.MIN_SOURCE_COUNT; + const config = loadConfig(); + expect(config.minConfidence).toBe(0); + expect(config.minSourceCount).toBe(2); + }); + + it('prefers an explicit MIN_SOURCE_COUNT', () => { + process.env.MIN_CONFIDENCE = '0.6'; + process.env.MIN_SOURCE_COUNT = '3'; + const config = loadConfig(); + expect(config.minConfidence).toBe(0.6); + expect(config.minSourceCount).toBe(3); + }); + + it('parses HTTP authentication and origin settings', () => { + process.env.HTTP_AUTH_TOKEN = 'test-token'; + process.env.ALLOWED_ORIGINS = 'https://one.example, https://two.example'; + const config = loadConfig(); + expect(config.httpAuthToken).toBe('test-token'); + expect(config.allowedOrigins).toEqual(['https://one.example', 'https://two.example']); + }); + it('parses ENABLED_TOOLS as array', () => { process.env.ENABLED_TOOLS = 'free_search,free_extract'; const config = loadConfig(); diff --git a/tests/infrastructure/health.test.ts b/tests/infrastructure/health.test.ts index 0f3507c..4f6cd91 100644 --- a/tests/infrastructure/health.test.ts +++ b/tests/infrastructure/health.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { ServerMetrics, HealthTracker } from '../../src/infrastructure/health.js'; import { SearchCache } from '../../src/infrastructure/cache.js'; @@ -128,6 +128,7 @@ describe('ServerMetrics edge cases', () => { describe('HealthTracker', () => { it('tracks provider health with circuit breaker', () => { + const stdoutSpy = vi.spyOn(console, 'log').mockImplementation(() => {}); const health = new HealthTracker(); expect(health.isHealthy('test-provider')).toBe(true); @@ -144,5 +145,7 @@ describe('HealthTracker', () => { expect(tp).toBeDefined(); expect(tp!.circuitState).toBe('open'); expect(tp!.errorCount).toBe(5); + expect(stdoutSpy).not.toHaveBeenCalled(); + stdoutSpy.mockRestore(); }); }); diff --git a/tests/infrastructure/http.test.ts b/tests/infrastructure/http.test.ts index 000fa13..36dcb64 100644 --- a/tests/infrastructure/http.test.ts +++ b/tests/infrastructure/http.test.ts @@ -21,7 +21,7 @@ describe('createHttpServer', () => { const body = await res.json(); expect(body.status).toBe('ok'); - expect(body.version).toBe('3.1.1'); + expect(body.version).toBe('3.1.3'); } finally { await server.close(); } @@ -41,17 +41,70 @@ it('GET /mcp without transport returns 404', async () => { }); it('CORS headers present when enableCors=true', async () => { - const server = createHttpServer(null, { port: 0, enableCors: true, corsOrigin: 'https://example.com' }); + const server = createHttpServer(null, { + port: 0, + enableCors: true, + corsOrigin: 'https://example.com', + allowedOrigins: ['https://example.com'], + }); await server.listen(); try { - const res = await fetch(`http://localhost:${server.getPort()}/health`); + const res = await fetch(`http://localhost:${server.getPort()}/health`, { + headers: { Origin: 'https://example.com' }, + }); expect(res.headers.get('access-control-allow-origin')).toBe('https://example.com'); } finally { await server.close(); } }); + it('rejects browser requests from an untrusted Origin', async () => { + const server = createHttpServer(null, { + port: 0, + enableCors: true, + corsOrigin: 'https://example.com', + allowedOrigins: ['https://example.com'], + }); + await server.listen(); + + try { + const res = await fetch(`http://localhost:${server.getPort()}/health`, { + headers: { Origin: 'https://evil.example' }, + }); + expect(res.status).toBe(403); + } finally { + await server.close(); + } + }); + + it('requires a valid Bearer token for MCP routes when configured', async () => { + const server = createHttpServer(null, { + port: 0, + enableCors: false, + corsOrigin: '*', + authToken: 'test-secret', + }); + await server.listen(); + + try { + const missing = await fetch(`http://localhost:${server.getPort()}/mcp`); + expect(missing.status).toBe(401); + + const wrong = await fetch(`http://localhost:${server.getPort()}/mcp`, { + headers: { Authorization: 'Bearer wrong-secret' }, + }); + expect(wrong.status).toBe(401); + + const valid = await fetch(`http://localhost:${server.getPort()}/mcp`, { + headers: { Authorization: 'bearer test-secret' }, + }); + expect(valid.status).toBe(404); + } finally { + await server.close(); + } + }); + it('returns 404 for unknown routes', async () => { const server = createHttpServer(null, { port: 0, enableCors: false, corsOrigin: '*' }); await server.listen(); diff --git a/tests/tools/fetch-tools.test.ts b/tests/tools/fetch-tools.test.ts index e75331a..1eeb468 100644 --- a/tests/tools/fetch-tools.test.ts +++ b/tests/tools/fetch-tools.test.ts @@ -59,6 +59,10 @@ describe('Fetch tools', () => { const content = await fetchCsdnArticle('https://blog.csdn.net/test/article/details/123'); expect(content).toContain('Test Article'); + expect(mockFetch).toHaveBeenCalledWith( + 'https://blog.csdn.net/test/article/details/123', + expect.objectContaining({ redirect: 'error' }) + ); }); it('throws on HTTP error', async () => { @@ -69,6 +73,17 @@ describe('Fetch tools', () => { await expect(fetchCsdnArticle('https://blog.csdn.net/test/article/details/123')).rejects.toThrow(); }); + + it.each([ + 'http://blog.csdn.net/test/article/details/123', + 'https://localhost/admin', + 'https://169.254.169.254/latest/meta-data', + 'https://blog.csdn.net.evil.example/article', + 'https://blog.csdn.net:8443/article', + ])('rejects unsafe or non-CSDN URL %s before fetching', async (url) => { + await expect(fetchCsdnArticle(url)).rejects.toThrow('Invalid CSDN article URL'); + expect(mockFetch).not.toHaveBeenCalled(); + }); }); describe('fetchJuejinArticle', () => { diff --git a/tests/tools/free-search-ddg-unavailable.test.ts b/tests/tools/free-search-ddg-unavailable.test.ts index 2fef43d..0c50355 100644 --- a/tests/tools/free-search-ddg-unavailable.test.ts +++ b/tests/tools/free-search-ddg-unavailable.test.ts @@ -2,8 +2,14 @@ import { describe, it, expect, vi, beforeAll } from 'vitest'; // Mock all engines vi.mock('../../src/engines/duckduckgo.js', () => ({ - searchDuckDuckGo: vi.fn(), - isDdgsAvailable: vi.fn(() => false), + searchDuckDuckGo: vi.fn(async () => [ + { + title: 'DuckDuckGo HTML Result', + url: 'https://duckduckgo.example/1', + snippet: 'HTML fallback result', + source: 'duckduckgo', + }, + ]), duckduckgoProvider: { id: 'duckduckgo', name: 'DuckDuckGo', isFree: true, languages: ['en'] }, })); vi.mock('../../src/engines/sogou.js', () => ({ @@ -20,12 +26,17 @@ vi.mock('../../src/engines/tavily.js', () => ({ TavilyProvider: vi.fn(() => ({ search: vi.fn() })), })); vi.mock('../../src/engines/exa.js', () => ({ searchExa: vi.fn() })); +vi.mock('../../src/engines/youcom.js', () => ({ searchYouCom: vi.fn() })); +vi.mock('../../src/engines/wikipedia.js', () => ({ searchWikipedia: vi.fn(async () => []) })); +vi.mock('../../src/engines/startpage.js', () => ({ searchStartpage: vi.fn(async () => []) })); +vi.mock('../../src/engines/yandex.js', () => ({ searchYandex: vi.fn(async () => []) })); +vi.mock('../../src/engines/mojeek.js', () => ({ searchMojeek: vi.fn(async () => []) })); vi.mock('../../src/aggregation/index.js', () => ({ dedupByUrl: vi.fn((r) => ({ results: r, frequencies: new Map() })), dedupByTitle: vi.fn((r) => r), filterLowQuality: vi.fn((r) => r), - scoreAndRank: vi.fn((r) => r.map((x) => ({ ...x, confidence: 1, score: 0.5 }))), + scoreAndRank: vi.fn((r) => r.map((x) => ({ ...x, confidence: 1, relevance: 0.5, source_count: 1, score: 0.5 }))), formatResults: vi.fn((r) => ({ results: r, meta: { total: r.length, high_confidence: r.length, engines: [] }, @@ -63,23 +74,23 @@ vi.mock('../../src/infrastructure/index.js', async (importOriginal) => { }; }); +import { searchDuckDuckGo } from '../../src/engines/duckduckgo.js'; import { searchWithFallback } from '../../src/tools/free-search.js'; -describe('DDG unavailable partialFailures', () => { - it('includes DDG unavailability in partialFailures', async () => { +describe('DDG without Python ddgs', () => { + it('lets the DDG adapter use its HTML fallback', async () => { const response = await searchWithFallback({ query: 'test query', count: 5, engines: ['duckduckgo', 'sogou'], }); - expect(response.partialFailures).toBeDefined(); - expect(response.partialFailures!.length).toBeGreaterThan(0); - const ddgFailure = response.partialFailures!.find( + expect(searchDuckDuckGo).toHaveBeenCalledWith('test query', 5); + const ddgFailure = response.partialFailures?.find( (f) => f.engine === 'duckduckgo' ); - expect(ddgFailure).toBeDefined(); - expect(ddgFailure!.message).toContain('ddgs'); + expect(ddgFailure).toBeUndefined(); + expect(response.results.some((result) => result.title === 'DuckDuckGo HTML Result')).toBe(true); }); it('uses correct engine name in failures (not "unknown")', async () => { diff --git a/tests/tools/free-search.test.ts b/tests/tools/free-search.test.ts index a6d60e5..d35bf03 100644 --- a/tests/tools/free-search.test.ts +++ b/tests/tools/free-search.test.ts @@ -15,13 +15,18 @@ vi.mock('../../src/engines/tavily.js', () => ({ TavilyProvider: vi.fn(() => ({ search: vi.fn() })), })); vi.mock('../../src/engines/exa.js', () => ({ searchExa: vi.fn() })); +vi.mock('../../src/engines/youcom.js', () => ({ searchYouCom: vi.fn() })); +vi.mock('../../src/engines/wikipedia.js', () => ({ searchWikipedia: vi.fn(async () => []) })); +vi.mock('../../src/engines/startpage.js', () => ({ searchStartpage: vi.fn(async () => []) })); +vi.mock('../../src/engines/yandex.js', () => ({ searchYandex: vi.fn(async () => []) })); +vi.mock('../../src/engines/mojeek.js', () => ({ searchMojeek: vi.fn(async () => []) })); vi.mock('../../src/aggregation/index.js', () => ({ dedupByProvider: vi.fn((r) => r), dedupByUrl: vi.fn((r) => ({ results: r, frequencies: new Map() })), dedupByTitle: vi.fn((r) => r), filterLowQuality: vi.fn((r) => r), - scoreAndRank: vi.fn((r) => r.map((x) => ({ ...x, confidence: 0.8, score: 0.6 }))), + scoreAndRank: vi.fn((r) => r.map((x) => ({ ...x, confidence: 0.8, relevance: 0.6, source_count: 1, score: 0.6 }))), formatResults: vi.fn((r) => ({ results: r.map((x) => ({ title: x.title, url: x.url, snippet: x.snippet || '', confidence: x.confidence || 0.8, @@ -68,6 +73,10 @@ import { searchDuckDuckGo } from '../../src/engines/duckduckgo.js'; import { searchSogou } from '../../src/engines/sogou.js'; import { searchBing } from '../../src/engines/bing.js'; import { searchBaidu } from '../../src/engines/baidu.js'; +import { searchWikipedia } from '../../src/engines/wikipedia.js'; +import { searchStartpage } from '../../src/engines/startpage.js'; +import { searchYandex } from '../../src/engines/yandex.js'; +import { searchMojeek } from '../../src/engines/mojeek.js'; import { detectLanguage, enrichResults } from '../../src/aggregation/index.js'; function makeResults(count: number, source: string) { @@ -114,6 +123,18 @@ describe('searchWithFallback — parallel', () => { expect(a).toBe(b); }); + it('does not collapse requests with different verification filters', async () => { + (searchDuckDuckGo as any).mockImplementation(async () => { + await new Promise(resolve => setTimeout(resolve, 10)); + return makeResults(3, 'ddg'); + }); + const [a, b] = await Promise.all([ + searchWithFallback({ query: 'filter-variant', minSourceCount: 1 }), + searchWithFallback({ query: 'filter-variant', minSourceCount: 2 }), + ]); + expect(a).not.toBe(b); + }); + it('handles engine failure gracefully', async () => { (searchBing as any).mockRejectedValue(new Error('ECONNRESET')); await expect( @@ -139,6 +160,19 @@ describe('searchWithFallback — parallel', () => { await searchWithFallback({ query: 'e', enrich: true, enrichMax: 3 }); expect(enrichResults).toHaveBeenCalled(); }); + + it('routes all four additional zero-key adapters', async () => { + await searchWithFallback({ + query: 'all-free-adapters', + count: 50, + engines: ['wikipedia', 'startpage', 'yandex', 'mojeek'], + }); + + expect(searchWikipedia).toHaveBeenCalled(); + expect(searchStartpage).toHaveBeenCalled(); + expect(searchYandex).toHaveBeenCalled(); + expect(searchMojeek).toHaveBeenCalled(); + }); }); describe('searchWithFallback — waterfall', () => {