diff --git a/AGENTS.md b/AGENTS.md index 1852cf4..f4a5ac9 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -79,10 +79,9 @@ commands live in the `## Validation Commands` section above, `Makefile`, and report everything `ok` except the API key. If `~/.storageops` is ever missing, re-run `storageops install` with the nvm Node on `PATH` (needs network for the npm Pi install). -- **Offline vs. live.** All gates and `make ci-local` / `storageops eval --baselines` - run fully offline and need no key. **Live diagnosis** needs install + Node/Pi + API key; - check `storageops doctor --json` field `live_diagnosis_available`, then run - `make live-smoke` (3 golden cases + model smoke). +- **Offline vs. live.** All gates and `make ci-local` run fully offline. **Live + diagnosis** needs install + Node/Pi + API key; check `live_diagnosis_available` + in `storageops doctor --json`, then run `make live-smoke` (5 golden cases). - **Devcontainer.** Open in VS Code / Codespaces with `.devcontainer/devcontainer.json` (Python 3.12 + Node 22; postCreate runs `dev_setup.sh --persist-path`). - **PR gate.** Run `make ci-local` before opening a PR. Use `make dev` for first-time diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d8899f..14b8c64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,26 @@ # Changelog +## 2026-06-29 — v0.8.0: Extension modules, eval depth, live smoke rotation + +### Track A — Extension modularization +Split `storageops_cli/extensions/storageops.ts` into focused modules with zero +behavior change (21 extension tests unchanged): +- `secrets.ts`, `routing.ts`, `memory.ts`, `trace.ts`, plus a thin `storageops.ts` + entry/re-export barrel. +- `storageops install` now deploys **all** `extensions/*.ts` modules. + +### Track B — Eval & routing depth +- New golden cases: `cors-method-put-blocked` (3rd `cors_configuration` diagnosis), + `routing-triage-vague-evidence` (triage routing without deep diagnosis). +- Baseline output for `cors-method-put-blocked`. +- `detect_domain` triage signatures for vague first-contact reports. +- Negative corpus case: CORS workshop text must not route to protocol CORS. +- `coverage_check` floor raised to **≥3 cases per baseline category**. + +### Track C — Live eval rotation +- `make live-smoke` / CI `diagnosis-smoke` now runs **5** golden cases: + throttling, access-denied, signature-clock-skew, cors-preflight, kms-denied. + ## 2026-06-29 — v0.7.2: Live diagnosis smoke & devcontainer - **`make live-smoke` / `scripts/live_smoke.sh`** — model round-trip plus live @@ -8,7 +29,7 @@ API key; reads `STORAGEOPS_MODEL_KEY` or provider env vars when set. - **`storageops doctor`** — adds `Live diagnosis` row and `live_diagnosis_available` in `--json` output when install, Node, Pi, and API key are ready. -- **CI `diagnosis-smoke`** — calls `scripts/live_smoke.sh` (3 cases) when +- **CI `diagnosis-smoke`** — calls `scripts/live_smoke.sh` when `STORAGEOPS_MODEL_KEY` is configured. - **`.devcontainer/devcontainer.json`** — Python 3.12 + Node 22; postCreate runs `dev_setup.sh --persist-path`. diff --git a/Makefile b/Makefile index ceec756..9a56ecc 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ help: @echo " make extension-tests Run the TypeScript extension behavioral tests" @echo " make install Install thin CLI shim (pip install -e .)" @echo " make dev One-shot dev setup (scripts/dev_setup.sh; DEV_SETUP_FLAGS=...)" - @echo " make live-smoke Model smoke + 3 golden-case live diagnoses (needs API key)" + @echo " make live-smoke Model smoke + 5 golden-case live diagnoses (needs API key)" @echo " make package-check Build wheel and run package_check.py (needs network once)" @echo "" @echo " Note: install-smoke and diagnosis-smoke run in CI when configured" diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index d5923f5..ffb6d29 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Architecture -StorageOps v0.7.2 is a Pi Coding Agent extension and skill pack. +StorageOps v0.8.0 is a Pi Coding Agent extension and skill pack. ## Components diff --git a/docs/cli-reference.md b/docs/cli-reference.md index d9e9e3d..801561c 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -41,7 +41,7 @@ from the older local package. Deployment provenance is written to ## `storageops --version` ```text -StorageOps v0.7.2 (pi: 0.78.0) +StorageOps v0.8.0 (pi: 0.78.0) httpmon : /root/.storageops/bin/httpmon api key : api-key file independent install : yes (~/.storageops/agent) diff --git a/docs/release.md b/docs/release.md index b7b309d..0c756ef 100644 --- a/docs/release.md +++ b/docs/release.md @@ -93,6 +93,6 @@ Locally (after `storageops configure --api-key` or exporting a provider env var) make live-smoke ``` -The script runs `storageops smoke` plus live `--print` diagnosis + eval for three +The script runs `storageops smoke` plus live `--print` diagnosis + eval for five golden cases: `throttling-hot-prefix`, `access-denied-cross-account`, -`signature-clock-skew`. +`signature-clock-skew`, `cors-preflight-failed`, and `kms-denied-encrypt`. diff --git a/pyproject.toml b/pyproject.toml index 2f82da7..600c4d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "storageops" -version = "0.7.2" +version = "0.8.0" description = "StorageOps — S3-compatible object storage diagnostic toolkit. A Pi Coding Agent extension + skill pack." requires-python = ">=3.11" readme = "README.md" diff --git a/scripts/coverage_check.py b/scripts/coverage_check.py index 937e16c..4b8bc07 100644 --- a/scripts/coverage_check.py +++ b/scripts/coverage_check.py @@ -23,7 +23,7 @@ BASELINES = CORPUS / "baseline-outputs" TAXONOMY = ROOT / "docs" / "skill-taxonomy.json" -MIN_CASES = 2 # per baseline-enabled category +MIN_CASES = 3 # per baseline-enabled category def main() -> int: diff --git a/scripts/live_smoke.sh b/scripts/live_smoke.sh index c248e4c..c1b5b23 100755 --- a/scripts/live_smoke.sh +++ b/scripts/live_smoke.sh @@ -38,6 +38,8 @@ CASES=( throttling-hot-prefix access-denied-cross-account signature-clock-skew + cors-preflight-failed + kms-denied-encrypt ) for case_name in "${CASES[@]}"; do diff --git a/scripts/routing_contract_check.py b/scripts/routing_contract_check.py index e3d79ea..69f8a1b 100644 --- a/scripts/routing_contract_check.py +++ b/scripts/routing_contract_check.py @@ -14,7 +14,13 @@ REGISTRY = ROOT / "skill-registry.yaml" CASES = ROOT / "skills" / "storageops-eval-golden-cases" / "cases" BASELINES = ROOT / "skills" / "storageops-eval-golden-cases" / "baseline-outputs" -EXTENSION = ROOT / "storageops_cli" / "extensions" / "storageops.ts" +EXTENSION_DIR = ROOT / "storageops_cli" / "extensions" + + +def extension_source_text() -> str: + return "\n".join( + path.read_text(encoding="utf-8") for path in sorted(EXTENSION_DIR.glob("*.ts")) + ) def load_json(path: Path) -> dict: @@ -90,7 +96,7 @@ def validate_baselines(taxonomy: dict, cases: dict[str, str]) -> list[str]: def validate_extension_coverage(taxonomy: dict) -> list[str]: - text = EXTENSION.read_text(encoding="utf-8") + text = extension_source_text() compact_extension = normalize(text) errors = [] for category, entry in sorted(taxonomy.get("categories", {}).items()): diff --git a/skill-registry.yaml b/skill-registry.yaml index 755d375..e94e49e 100644 --- a/skill-registry.yaml +++ b/skill-registry.yaml @@ -1,4 +1,4 @@ -# StorageOps Skill Registry v0.7.2 +# StorageOps Skill Registry v0.8.0 # Machine-readable metadata for all StorageOps skills. # Skills are auto-discovered by Pi from the skills/ directory. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/cors-method-put-blocked.md b/skills/storageops-eval-golden-cases/baseline-outputs/cors-method-put-blocked.md new file mode 100644 index 0000000..bfe8f72 --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/cors-method-put-blocked.md @@ -0,0 +1,22 @@ +# Summary + +Category: cors_configuration +Route: storageops-s3-protocol-compatibility +Confidence: 0.86 +Root Cause Type: cors_method_not_allowed + +Primary Diagnosis: root_cause_type=cors_method_not_allowed, affected_layer=policy + +Browser PUT preflight fails because the bucket CORS rule allows GET/HEAD but not PUT +for the app origin. + +# Key Evidence + +- Preflight reports `Method PUT is not allowed by Access-Control-Allow-Methods`. +- `Access-Control-Allow-Origin` is present for the app origin. +- Direct GET without an Origin header succeeds. + +# Remediation + +Update the bucket CORS configuration with `put-bucket-cors` to include `PUT` in `AllowedMethod` for the +narrow app origin. Retest with an OPTIONS preflight before retrying the browser upload. diff --git a/skills/storageops-eval-golden-cases/cases/cors-method-put-blocked/description.md b/skills/storageops-eval-golden-cases/cases/cors-method-put-blocked/description.md new file mode 100644 index 0000000..7d864b2 --- /dev/null +++ b/skills/storageops-eval-golden-cases/cases/cors-method-put-blocked/description.md @@ -0,0 +1,4 @@ +# cors-method-put-blocked + +Browser PUT upload fails CORS preflight because the bucket CORS rule allows GET +but not PUT for the app origin. diff --git a/skills/storageops-eval-golden-cases/cases/cors-method-put-blocked/expected.json b/skills/storageops-eval-golden-cases/cases/cors-method-put-blocked/expected.json new file mode 100644 index 0000000..e0fa9ee --- /dev/null +++ b/skills/storageops-eval-golden-cases/cases/cors-method-put-blocked/expected.json @@ -0,0 +1,30 @@ +{ + "expected_category": "cors_configuration", + "expected_root_cause_types": [ + "cors_method_not_allowed", + "missing_allowed_method", + "cors_configuration_incomplete" + ], + "expected_min_confidence": 0.85, + "must_include_evidence_keywords": [ + "CORS", + "PUT", + "Access-Control-Allow-Methods", + "preflight" + ], + "must_include_recommendation_keywords": [ + "AllowedMethod", + "PUT", + "put-bucket-cors" + ], + "must_not_include": [ + "delete bucket", + "make bucket public", + "disable SSL" + ], + "required_report_sections": [ + "Summary", + "Key Evidence", + "Remediation" + ] +} diff --git a/skills/storageops-eval-golden-cases/cases/cors-method-put-blocked/input/browser-console.txt b/skills/storageops-eval-golden-cases/cases/cors-method-put-blocked/input/browser-console.txt new file mode 100644 index 0000000..e8707f7 --- /dev/null +++ b/skills/storageops-eval-golden-cases/cases/cors-method-put-blocked/input/browser-console.txt @@ -0,0 +1,19 @@ +Browser console (PUT upload): +Access to fetch at 'https://uploads.example-bucket.s3.amazonaws.com/incoming/report.csv' +from origin 'https://dashboard.example.com' has been blocked by CORS policy: +Response to preflight request doesn't pass access control check: Method PUT is not +allowed by Access-Control-Allow-Methods in preflight response. + +OPTIONS preflight response headers: + Access-Control-Allow-Origin: https://dashboard.example.com + Access-Control-Allow-Methods: GET, HEAD + Access-Control-Allow-Headers: content-type + +Direct GET from curl without Origin succeeds (200 OK). + +Current CORS configuration excerpt: + AllowedOrigin: https://dashboard.example.com + AllowedMethod: GET, HEAD + AllowedHeader: * + +User goal: browser JavaScript PUT upload from dashboard.example.com diff --git a/skills/storageops-eval-golden-cases/cases/routing-triage-vague-evidence/description.md b/skills/storageops-eval-golden-cases/cases/routing-triage-vague-evidence/description.md new file mode 100644 index 0000000..952160d --- /dev/null +++ b/skills/storageops-eval-golden-cases/cases/routing-triage-vague-evidence/description.md @@ -0,0 +1,4 @@ +# routing-triage-vague-evidence + +Vague first-contact report with no error codes or logs yet. Triage should classify +the issue and list evidence gaps before routing to a specialist skill. diff --git a/skills/storageops-eval-golden-cases/cases/routing-triage-vague-evidence/expected.json b/skills/storageops-eval-golden-cases/cases/routing-triage-vague-evidence/expected.json new file mode 100644 index 0000000..585b1c1 --- /dev/null +++ b/skills/storageops-eval-golden-cases/cases/routing-triage-vague-evidence/expected.json @@ -0,0 +1,25 @@ +{ + "case_type": "routing", + "expected_category": "triage", + "expected_root_cause_types": ["insufficient_evidence", "unknown_root_cause"], + "expected_min_confidence": 0.7, + "must_include_evidence_keywords": [ + "evidence", + "logs", + "request" + ], + "must_include_recommendation_keywords": [ + "collect", + "error", + "route" + ], + "must_not_include": [ + "delete bucket", + "make public", + "disable TLS" + ], + "required_report_sections": [ + "Routing", + "Evidence Gaps" + ] +} diff --git a/skills/storageops-eval-golden-cases/cases/routing-triage-vague-evidence/input/user-report.txt b/skills/storageops-eval-golden-cases/cases/routing-triage-vague-evidence/input/user-report.txt new file mode 100644 index 0000000..8ee5837 --- /dev/null +++ b/skills/storageops-eval-golden-cases/cases/routing-triage-vague-evidence/input/user-report.txt @@ -0,0 +1,9 @@ +Subject: object storage uploads failing intermittently + +Our application team reports object storage upload problems this week from the +production dashboard. We do not have error logs, HTTP status codes, or request IDs +yet — only user reports that uploads sometimes fail mid-transfer. + +We use a private bucket in us-east-1. No vendor support ticket has been opened. + +Please classify the issue domain and tell us what evidence to collect next. diff --git a/storageops_cli/__init__.py b/storageops_cli/__init__.py index 8433b65..a512034 100644 --- a/storageops_cli/__init__.py +++ b/storageops_cli/__init__.py @@ -737,12 +737,17 @@ def _write_install_marker(data: Path, target_agent: Path, merge: bool) -> None: def _copy_extension(data: Path, dst_agent: Path) -> None: - """Copy the StorageOps Pi extension to the target agent directory.""" - ext_src = data / "extensions" / "storageops.ts" - ext_dst = dst_agent / "extensions" / "storageops.ts" - ext_dst.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(ext_src, ext_dst) - print(f" [ok] storageops.ts -> {ext_dst}") + """Copy all StorageOps Pi extension modules to the target agent directory.""" + ext_src = data / "extensions" + ext_dst = dst_agent / "extensions" + ext_dst.mkdir(parents=True, exist_ok=True) + ts_files = sorted(ext_src.glob("*.ts")) + if not any(p.name == "storageops.ts" for p in ts_files): + print(" [fail] storageops.ts not found in package extensions/") + sys.exit(1) + for src in ts_files: + shutil.copy2(src, ext_dst / src.name) + print(f" [ok] extension ({len(ts_files)} files) -> {ext_dst}") def _copy_skills(data: Path, dst_agent: Path) -> None: diff --git a/storageops_cli/extensions/memory.ts b/storageops_cli/extensions/memory.ts new file mode 100644 index 0000000..715c516 --- /dev/null +++ b/storageops_cli/extensions/memory.ts @@ -0,0 +1,147 @@ +import * as fs from "fs"; +import * as path from "path"; +import * as os from "os"; +import { redactText } from "./secrets.ts"; + +export type MemoryResult = { + sessionId: string; + snippet: string; + updated: string; + source: "summary" | "jsonl"; + score: number; +}; + +export function searchTokens(query: string): string[] { + const ascii = query.toLowerCase().match(/[a-z0-9_\-:.]{3,}/g) || []; + // CJK queries carry no ASCII word tokens, so the old tokenizer returned [] and + // recall was empty for Chinese. Emit overlapping bigrams (and single chars for + // length-1 runs) so Chinese memory searches recall partial matches. + const cjkTokens: string[] = []; + for (const run of query.match(/[一-鿿]+/g) || []) { + if (run.length === 1) cjkTokens.push(run); + else for (let i = 0; i < run.length - 1; i++) cjkTokens.push(run.slice(i, i + 2)); + } + return Array.from(new Set([...ascii, ...cjkTokens])).slice(0, 12); +} + +function scoreText(text: string, tokens: string[]): number { + const lower = text.toLowerCase(); + return tokens.reduce((score, token) => score + (lower.includes(token) ? 1 : 0), 0); +} + +function safeMemorySnippet(text: string): string { + return redactText(text.replace(/\s+/g, " ").trim()).redacted.slice(0, 240); +} + +const MAX_SESSION_SCAN_DEPTH = 4; +const MAX_SESSION_FILES = 200; + +// Pi stores session transcripts under scope subdirectories +// (e.g. sessions//.jsonl), so a flat top-level scan misses them. +// Walk the sessions tree with bounded depth/count and index by .jsonl files; +// .meta.json is optional sibling enrichment, not required for recall. +export function collectSessionJsonl(root: string): string[] { + const found: string[] = []; + const walk = (dir: string, depth: number): void => { + if (depth > MAX_SESSION_SCAN_DEPTH || found.length >= MAX_SESSION_FILES) return; + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }); + } catch { + return; + } + for (const entry of entries) { + if (found.length >= MAX_SESSION_FILES) return; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + walk(full, depth + 1); + } else if (entry.isFile() && entry.name.endsWith(".jsonl")) { + found.push(full); + } + } + }; + walk(root, 0); + return found; +} + +function readSessionMeta(jsonlPath: string): { sessionId: string; summary: string; updated: string } { + const sessionId = path.basename(jsonlPath, ".jsonl"); + const metaPath = path.join(path.dirname(jsonlPath), `${sessionId}.meta.json`); + if (fs.existsSync(metaPath)) { + try { + const meta = JSON.parse(fs.readFileSync(metaPath, "utf8")); + return { + sessionId: meta.id || sessionId, + summary: meta.summary || meta.name || "", + updated: meta.updated || meta.created || "", + }; + } catch { + // Malformed meta; the jsonl content is still searchable. + } + } + return { sessionId, summary: "", updated: "" }; +} + +export function searchMemory(query: string, limit: number = 5): MemoryResult[] { + const agentDir = process.env.PI_CODING_AGENT_DIR || path.join(os.homedir(), ".pi", "agent"); + const primarySessionsDir = path.join(agentDir, "sessions"); + const fallbackSessionsDir = path.join(os.homedir(), ".pi", "agent", "sessions"); + const sessionsDir = fs.existsSync(primarySessionsDir) ? primarySessionsDir : fallbackSessionsDir; + if (!fs.existsSync(sessionsDir)) return []; + + const tokens = searchTokens(query); + if (tokens.length === 0) return []; + const cappedLimit = Math.min(Math.max(limit || 5, 1), 10); + + const jsonlFiles = collectSessionJsonl(sessionsDir) + .sort() + .reverse() + .slice(0, MAX_SESSION_FILES); + + const results: MemoryResult[] = []; + + for (const jsonlPath of jsonlFiles) { + try { + const { sessionId, summary, updated } = readSessionMeta(jsonlPath); + + const summaryScore = summary ? scoreText(summary, tokens) : 0; + if (summaryScore > 0) { + results.push({ + sessionId, + snippet: safeMemorySnippet(summary), + updated, + source: "summary", + score: summaryScore, + }); + } + + const content = fs.readFileSync(jsonlPath, "utf8").slice(0, 40_000); + const jsonlScore = scoreText(content, tokens); + if (jsonlScore > 0) { + const line = content.split(/\r?\n/).find(x => scoreText(x, tokens) > 0) || summary || `Session ${sessionId.slice(0, 8)}...`; + results.push({ + sessionId, + snippet: safeMemorySnippet(line), + updated, + source: "jsonl", + score: jsonlScore, + }); + } + } catch { + // Skip unreadable files + } + } + + // Keep only the best-scoring entry per session so one session can't occupy + // multiple result slots (a session matching in both summary and jsonl would + // otherwise crowd out other relevant sessions). + const bestBySession = new Map(); + for (const r of results) { + const prev = bestBySession.get(r.sessionId); + if (!prev || r.score > prev.score) bestBySession.set(r.sessionId, r); + } + + return Array.from(bestBySession.values()) + .sort((a, b) => b.score - a.score || String(b.updated).localeCompare(String(a.updated))) + .slice(0, cappedLimit); +} diff --git a/storageops_cli/extensions/routing.ts b/storageops_cli/extensions/routing.ts new file mode 100644 index 0000000..f57f724 --- /dev/null +++ b/storageops_cli/extensions/routing.ts @@ -0,0 +1,234 @@ +// ── Domain Detection ──────────────────────────────────────────────────────── +// Signature-based domain classification from evidence text. +// Replaces the old Python storageops/utils/signatures.py + +const DOMAIN_SIGNATURES: Record> = { + "storageops-security-iam-policy": [ + [/403\s*(?:Forbidden|Access\s*Denied)/i, "access_denied"], + [/AccessDenied/i, "access_denied_api"], + [/InvalidAccessKeyId/i, "invalid_key"], + [/kms:\w|\bKMS\b.*\b(key|encrypt|decrypt|SSE|cipher|grant|CMK|GenerateDataKey)\b|\b(key|encrypt|decrypt|SSE|cipher|CMK)\b.*\bKMS\b/i, "kms_error"], + [/401\s*Unauthorized|\bUnauthorized\b.*\b(request|access|operation|principal|credential|token|user|role|perform|API|HTTP|header|signature)\b|\b(request|access|operation|principal|credential|token|HTTP|API|signature)\b.*\bUnauthorized\b/i, "unauthorized"], + [/AssumeRole|sts:/i, "role_error"], + [/权限|无权限|鉴权|拒绝访问|访问被拒/i, "access_denied_cjk"], + ], + "storageops-s3-protocol-compatibility": [ + [/SignatureDoesNotMatch|AuthorizationHeaderMalformed|InvalidArgument/i, "signature_or_protocol_error"], + [/RequestTimeTooSkewed|RequestExpired|NotImplemented|MissingContentLength|EntityTooLarge|EntityTooSmall|PreconditionFailed/i, "protocol_error_code"], + [/\bBadDigest(?:SHA256|MD5)?\b/i, "payload_digest"], + [/CanonicalRequest|StringToSign|AWS4-HMAC-SHA256|SigV4|SigV2/i, "signature_debug"], + [/Access-Control-Allow-Origin|Access-Control-Allow-Methods|NoSuchCORSConfiguration|blocked by CORS policy|\bCORS policy\b.*\b(?:blocked|Origin|preflight|Allow)\b|\bpreflight\b.*\b(?:403|OPTIONS|failed|missing)\b/i, "cors"], + [/MalformedXML|InvalidDigest|Content-MD5|x-(?:amz|bce)-content-sha256/i, "protocol_header"], + [/virtual.?hosted|path.?style|chunked|STREAMING-AWS4-HMAC-SHA256-PAYLOAD/i, "provider_compatibility"], + [/签名|校验和|校验值/i, "signature_or_protocol_cjk"], + ], + "storageops-performance-diagnosis": [ + [/429|TooManyRequests|RequestRateLimitExceeded/i, "rate_limit"], + [/SlowDown/i, "slow_down"], + [/throttl/i, "throttle"], + [/timeout|timed?\s*out/i, "timeout"], + [/bandwidth/i, "bandwidth"], + [/retry/i, "retry"], + [/限速|限流|超时|慢|带宽/i, "performance_cjk"], + ], + "storageops-network-endpoint-access": [ + [/DNS|Name\s*or\s*service\s*not\s*known|NXDOMAIN/i, "dns"], + [/Could\s*not\s*connect|Connection\s*refused|connect\s*ETIMEDOUT/i, "connectivity"], + [/TLS|SSL|\bcert(?:ificate)?\b/i, "tls"], + [/连接(?:失败|超时|被拒)|证书|解析失败|无法访问/i, "network_cjk"], + [/VPC|endpoint|ENDPOINT/i, "endpoint"], + [/host\s*unreachable|no\s*route/i, "route"], + [/RequestTimeout|connection\s*reset|reset\s*by\s*peer|broken\s*pipe|\bECONNRESET\b|\bEPIPE\b|unexpected\s*EOF/i, "transport"], + ], + "storageops-cli-sdk-diagnosis": [ + [/rclone/i, "rclone"], + [/s5cmd/i, "s5cmd"], + [/awscli|botocore|boto3/i, "aws_cli"], + [/\bbcecmd\b|\bgo-bcecli\b/i, "bcecmd"], + [/\bobsutil\b|\bobs:\/\//i, "obsutil"], + [/corrupted\s*on\s*transfer|multipart.*etag/i, "corruption"], + [/损坏|校验失败|传输失败/i, "corruption_cjk"], + ], + "storageops-replication-versioning": [ + [/replicat/i, "replication"], + [/CRR|SRR/i, "replication_type"], + [/\bversioning\b|version\s*id/i, "versioning"], + [/DeleteMarker/i, "delete_marker"], + [/sync\s*(?:lag|delay)/i, "sync_lag"], + ], + "storageops-lifecycle-cost": [ + [/lifecycle/i, "lifecycle"], + [/Standard_IA|Glacier|Deep_Archive/i, "storage_class"], + [/\bcost\b.*\b(storage|request|egress|transition|tier|bill|lifecycle|retrieval|IA|Glacier|Archive|GB|TB)\b|\b(storage|request|egress|transition|tier|bill|lifecycle|retrieval|IA|Glacier|Archive|GB|TB)\b.*\bcost\b|费用|计费|账单/i, "cost"], + [/transition|\bexpir(?:e|es|ed|ation|ing)?\b/i, "transition"], + [/objects.*small|small.*objects/i, "small_objects"], + ], + "storageops-mount-filesystem-workspace": [ + [/\bmount(?:ed|ing)?\b|\bFUSE\b|\bs3fs\b|\bgoofys\b/i, "mount"], + [/\bFUSE\b/i, "fuse"], + [/\bfilesystem\b/i, "filesystem"], + ], + "storageops-migration-sync": [ + [/migrat|搬迁|迁移/i, "migration"], + [/\b(?:rclone|s5cmd|obsutil|bcecmd|ossutil|coscli)\s+sync\b|\baws\s+s3\s+sync\b|\bmc\s+mirror\b|\bsync\b.*\b(?:s3|oss|cos|obs|bos|bucket|object|prefix|remote:|provider|checksum|verify|verification)\b|\b(?:s3|oss|cos|obs|bos|bucket|object|prefix|remote:|provider)\b.*\bsync\b|cp\s+-r/i, "sync"], + [/\btransfer(?:s|red|ring)?\b.*\b(?:s3|oss|cos|obs|bos|bucket|object|prefix|provider|cross-region|multipart)\b|\b(?:s3|oss|cos|obs|bos|bucket|object|prefix|provider|cross-region|multipart)\b.*\btransfer(?:s|red|ring)?\b/i, "transfer"], + ], + "storageops-data-consistency": [ + [/consistenc|一致性/i, "consistency"], + [/stale|陈旧/i, "stale"], + [/\bmismatch\b.*\b(etag|checksum|hash|md5|sha|content|object|data|digest|size|byte)\b|\b(etag|checksum|hash|md5|sha|content|object|data|digest)\b.*\bmismatch\b/i, "mismatch"], + [/checksum|ETag/i, "checksum"], + ], + "storageops-bigdata-pipeline": [ + [/\b(?:Spark|Hive|Flink|Hadoop|S3A|EMR)\b/i, "bigdata_engine"], + [/FileOutputCommitter|MagicCommitter|S3ACommitter|_temporary|speculative execution/i, "committer"], + [/partition|Parquet|Iceberg|Delta|Hudi/i, "table_or_partition"], + [/small files|many files|listing storm|listObjects/i, "small_file_query"], + ], + "storageops-event-notification": [ + [/notification|通知/i, "notification"], + [/event\s*notifications?|bucket\s*events?|\bSQS\b|\bLambda\b/i, "event"], + [/prefix filter|suffix filter|ObjectCreated|ObjectRemoved/i, "event_filter"], + ], + "storageops-access-log-analysis": [ + [/access\s*log|server\s*access\s*log/i, "access_log"], + [/log\s*(?:analysis|分析)|request\s*analysis|traffic\s*analysis/i, "log_analysis"], + [/403\s*spike|503\s*spike|error\s*rate|错误率/i, "error_spike"], + [/who\s+is\s+accessing|top\s*requester|requester/i, "requester"], + [/cost\s*attribution|费用归因|成本归因/i, "cost_attribution"], + ], + "storageops-triage": [ + [/\bobject storage\b|\bS3 error\b|\bstorage issue\b|\bstorage problem\b|\bbucket issue\b/i, "triage_vague"], + [/uploads?\s+(?:sometimes\s+)?fail|\bno error logs\b|\bno request id\b/i, "triage_incomplete"], + ], +}; + +export type DomainDetection = { + domain: string; + recommended_skill: string; + confidence: number; + subdomains: string[]; + signals: string[]; + next_action: string; +}; + +// ── Provider Detection ────────────────────────────────────────────────────── +// Object-storage misdiagnosis is dominated by applying AWS assumptions to a +// non-AWS provider. Identify the provider deterministically from endpoint hosts, +// vendor header prefixes, vendor CLIs, and URI schemes — so provider-specific +// quirks get applied even when the user never names the provider. Conservative: +// returns "unknown" unless a clear signal is present. Note: x-amz-* headers are +// shared by all S3-compatible providers, so they are NOT an AWS signal. + +type ProviderEntry = { + provider: string; + confidence: "high" | "medium" | "low"; + signals: string[]; + quirks_ref: string | null; +}; + +export type ProviderDetection = ProviderEntry & { + // All providers detected in the evidence, strongest first. For migration/sync + // the source and destination differ and need different quirks; the top-level + // fields are the primary (strongest) provider for backward compatibility. + providers: ProviderEntry[]; +}; + +const PROVIDER_SIGNATURES: Array<[string, RegExp, string]> = [ + ["aws", /\.amazonaws\.com\b/i, "endpoint:amazonaws.com"], + ["bos", /\.bcebos\.com\b/i, "endpoint:bcebos.com"], + ["bos", /x-bce-|\bbcecmd\b|\bgo-bcecli\b|\bbos:\/\//i, "header/cli:bce"], + ["oss", /\.aliyuncs\.com\b/i, "endpoint:aliyuncs.com"], + ["oss", /x-oss-|\bossutil\b|\boss:\/\//i, "header/cli:oss"], + ["cos", /\.myqcloud\.com\b/i, "endpoint:myqcloud.com"], + ["cos", /x-cos-|\bcoscli\b|\bcoscmd\b|q-sign-algorithm|\bcos:\/\//i, "header/cli:cos"], + ["gcs", /storage\.googleapis\.com\b/i, "endpoint:googleapis.com"], + ["gcs", /x-goog-|\bgsutil\b|\bgs:\/\//i, "header/cli:goog"], + ["azure", /\.blob\.core\.windows\.net\b/i, "endpoint:blob.core.windows.net"], + ["azure", /x-ms-(?:blob|version|date|meta)|\baz\s+storage\b/i, "header/cli:azure"], + ["obs", /\.myhuaweicloud\.com\b|\bobs\.[a-z0-9-]+\.myhuaweicloud/i, "endpoint:myhuaweicloud.com"], + ["obs", /x-obs-|\bobsutil\b|\bobs:\/\//i, "header/cli:obs"], + ["minio", /\bMinIO\b|x-minio-/i, "marker:minio"], +]; + +const PROVIDER_QUIRKS_REF: Record = { + bos: "storageops-s3-protocol-compatibility/references/provider-quirks/bos.md", + oss: "storageops-s3-protocol-compatibility/references/provider-quirks/oss.md", + cos: "storageops-s3-protocol-compatibility/references/provider-quirks/cos.md", + minio: "storageops-s3-protocol-compatibility/references/provider-quirks/minio.md", +}; + +export function detectProvider(text: string): ProviderDetection { + const evidence = (text || "").slice(0, 100_000); + const hits: Record = {}; + for (const [provider, regex, label] of PROVIDER_SIGNATURES) { + regex.lastIndex = 0; + if (regex.test(evidence)) (hits[provider] ||= []).push(label); + } + const names = Object.keys(hits); + if (names.length === 0) { + return { provider: "unknown", confidence: "low", signals: [], quirks_ref: null, providers: [] }; + } + // Strongest provider = most signal hits; an endpoint host match is high confidence. + names.sort((a, b) => hits[b].length - hits[a].length); + const providers: ProviderEntry[] = names.map(name => ({ + provider: name, + confidence: hits[name].some(l => l.startsWith("endpoint:")) ? "high" : "medium", + signals: hits[name].map(l => `${name}:${l}`), + quirks_ref: PROVIDER_QUIRKS_REF[name] ?? null, + })); + const primary = providers[0]; + return { + provider: primary.provider, + confidence: primary.confidence, + signals: providers.flatMap(p => p.signals), + quirks_ref: primary.quirks_ref, + providers, + }; +} + +const DOMAIN_NEXT_ACTION: Record = { + "storageops-security-iam-policy": "Check identity, policy, key validity, bucket policy, and KMS constraints before changing permissions.", + "storageops-s3-protocol-compatibility": "Compare endpoint style, region, canonical request shape, signing version, and required headers.", + "storageops-performance-diagnosis": "Separate service throttling, client retry behavior, network latency, and object layout signals.", + "storageops-network-endpoint-access": "Verify DNS, endpoint, route, proxy, and TLS certificate evidence before testing application logic.", + "storageops-cli-sdk-diagnosis": "Confirm the exact CLI/SDK, config path, provider, endpoint, and version before applying reference docs.", + "storageops-replication-versioning": "Inspect versioning, delete markers, replication rules, and observed replication lag.", + "storageops-lifecycle-cost": "Check lifecycle rules, storage class transitions, request patterns, and dated pricing references.", + "storageops-mount-filesystem-workspace": "Treat mount tools as filesystem adapters; verify cache, FUSE, permissions, and consistency expectations.", + "storageops-migration-sync": "Verify read-only inventory, delta strategy, checksums, and idempotent sync planning.", + "storageops-data-consistency": "Collect timestamps, ETags/checksums, list/head differences, and cross-client observations.", + "storageops-bigdata-pipeline": "Inspect engine, committer, partition layout, speculative execution, and object-listing behavior.", + "storageops-event-notification": "Check event rules, prefix/suffix filters, target permissions, and delivery logs.", + "storageops-access-log-analysis": "Summarize request IDs, status spikes, top requesters, user agents, and time windows.", + "storageops-triage": "Classify the domain, list evidence gaps, and route to one specialist skill before deep diagnosis.", +}; + +export function detectDomain(text: string): DomainDetection[] { + const scores: Record; signals: string[] }> = {}; + const evidence = text.slice(0, 100_000); + + for (const [domain, patterns] of Object.entries(DOMAIN_SIGNATURES)) { + for (const [regex, subdomain] of patterns) { + regex.lastIndex = 0; + const match = regex.exec(evidence); + if (match) { + if (!scores[domain]) scores[domain] = { score: 0, subdomains: new Set(), signals: [] }; + scores[domain].score += 1; + scores[domain].subdomains.add(subdomain); + scores[domain].signals.push(match[0].slice(0, 80)); + } + } + } + + return Object.entries(scores) + .map(([domain, info]) => ({ + domain, + recommended_skill: domain, + confidence: Math.min(0.5 + info.score * 0.15, 0.95), + subdomains: Array.from(info.subdomains), + signals: info.signals.slice(0, 5), + next_action: DOMAIN_NEXT_ACTION[domain] || "Collect more evidence before choosing a specialized skill.", + })) + .sort((a, b) => b.confidence - a.confidence); +} diff --git a/storageops_cli/extensions/secrets.ts b/storageops_cli/extensions/secrets.ts new file mode 100644 index 0000000..2188d1e --- /dev/null +++ b/storageops_cli/extensions/secrets.ts @@ -0,0 +1,109 @@ +import * as crypto from "crypto"; + +const MAX_SECRET_SCAN_CHARS = 200_000; +const MAX_REDACTED_TEXT_CHARS = 20_000; +const SECRET_PATTERNS: Array<[RegExp, string]> = [ + // AWS access keys (AKIA...) + [/(?:AWS|aws)[\s_-]*(?:access[\s_-]*)?(?:key[\s_-]*id|akid)[\s]*[:=][\s]*([A-Z0-9]{16,})/gi, "AWS_ACCESS_KEY"], + [/(?:AKIA|ASIA)[A-Z0-9]{16}/g, "AWS_ACCESS_KEY_ID"], + // AWS secret keys — long alphanumeric with config keyword + [/(?:secret[\s_-]*)?(?:access[\s_-]*)?key[\s]*[:=][\s]*['"]?([A-Za-z0-9\/+=]{20,60})['"]?/gi, "AWS_SECRET_KEY"], + // AWS session tokens + [/(?:session[\s_-]*)?(?:token|x-amz-security-token)[\s]*[:=][\s]*['"]?([A-Za-z0-9\/+=]{100,})['"]?/gi, "AWS_SESSION_TOKEN"], + // Alibaba Cloud AK + [/(?:LTAI)[A-Za-z0-9]{16,20}/g, "ALIBABA_ACCESS_KEY"], + // Tencent Cloud SecretId + [/(?:AKID)[A-Za-z0-9]{32,48}/g, "TENCENT_SECRET_ID"], + // Baidu Cloud AK + [/(?:ak[\s]*=|access_key[\s]*=)[\s]*['"]?([a-f0-9]{32})['"]?/gi, "BAIDU_ACCESS_KEY"], + // Generic Authorization: Bearer / Basic tokens + [/Authorization[\s]*:[\s]*(?:Bearer|Basic|AWS4-HMAC-SHA256)[\s]+([^\s]{20,})/gi, "AUTHORIZATION_HEADER"], + // SigV4 Authorization headers carry the HMAC after the credential scope, e.g. + // "Credential=.../s3/aws4_request, Signature=". Redact the signature value + // while leaving the credential scope (date/region/service) — useful evidence — visible. + [/\bSignature=([0-9a-fA-F]{16,})/g, "SIGV4_SIGNATURE"], + // Private keys (PEM) — incl. plain PKCS8 "PRIVATE KEY" used by GCP service-account keys + [/-----BEGIN (?:(?:RSA|EC|DSA|OPENSSH|ENCRYPTED) )?PRIVATE KEY-----/g, "PRIVATE_KEY"], + // rclone config passwords + [/(?:pass|password|token|secret)[\s]*=[\s]*['"]?([^\s'"]{8,})['"]?/gi, "RCLONE_CREDENTIAL"], + // Generic API keys (sk-... for OpenAI/DeepSeek style) + [/(?:api[\s_-]*)?(?:key|token)[\s]*[:=][\s]*['"]?(sk-[A-Za-z0-9]{20,})['"]?/gi, "API_KEY"], + // GitHub tokens (ghp_, gho_, github_pat_) + [/(?:ghp_|gho_|github_pat_)[A-Za-z0-9]{36,}/g, "GITHUB_TOKEN"], + // Presigned-URL signature material — extremely common in rclone/aws/s5cmd debug logs + [/[?&](?:X-Amz-Signature|X-Goog-Signature)=([A-Za-z0-9%]{16,})/gi, "PRESIGNED_SIGNATURE"], + [/[?&]X-Amz-(?:Credential|Security-Token)=([^&\s]{16,})/gi, "PRESIGNED_AWS_PARAM"], + [/[?&](?:OSSAccessKeyId|Signature)=([^&\s]{10,})/gi, "OSS_PRESIGNED"], + [/[?&]q-(?:signature|ak)=([^&\s]{8,})/gi, "COS_PRESIGNED"], + // GCP service-account key id (the PEM private_key itself is caught above) + [/"private_key_id"[\s]*:[\s]*"([a-f0-9]{16,})"/gi, "GCP_PRIVATE_KEY_ID"], + // Azure storage account key + SAS signature + [/AccountKey=([A-Za-z0-9+\/=]{40,})/gi, "AZURE_ACCOUNT_KEY"], + [/[?&]sig=([A-Za-z0-9%]{20,})/gi, "AZURE_SAS"], +]; + +function secretFingerprint(value: string): string { + return "sha256:" + crypto.createHash("sha256").update(value).digest("hex").slice(0, 12); +} + +export type SecretFinding = { + line: number; + column: number; + type: string; + length: number; + fingerprint: string; +}; + +function matchSecretRange(match: RegExpMatchArray): { start: number; end: number; value: string } { + const matchStart = match.index ?? 0; + const full = match[0] || ""; + const captured = match.slice(1).find(v => typeof v === "string" && v.length > 0); + if (captured) { + const offset = full.lastIndexOf(captured); + if (offset >= 0) { + return { start: matchStart + offset, end: matchStart + offset + captured.length, value: captured }; + } + } + return { start: matchStart, end: matchStart + full.length, value: full }; +} + +function lineAndColumn(text: string, index: number): { line: number; column: number } { + const prefix = text.slice(0, index); + const line = prefix.split("\n").length; + const lastNewline = prefix.lastIndexOf("\n"); + return { line, column: index - lastNewline }; +} + +export function redactText(text: string): { findings: SecretFinding[]; redacted: string; truncated: boolean } { + const scanText = text.slice(0, MAX_SECRET_SCAN_CHARS); + const findings: SecretFinding[] = []; + const ranges: Array<[number, number]> = []; + + for (const [pattern, type] of SECRET_PATTERNS) { + // Reset lastIndex for global regex + pattern.lastIndex = 0; + const matches = Array.from(scanText.matchAll(pattern)); + for (const m of matches) { + const { start, end, value } = matchSecretRange(m); + if (ranges.some(([rangeStart, rangeEnd]) => start < rangeEnd && end > rangeStart)) { + continue; + } + const { line, column } = lineAndColumn(scanText, start); + const length = value.length; + const fingerprint = secretFingerprint(value); + ranges.push([start, end]); + findings.push({ line, column, type, length, fingerprint }); + } + } + + let redacted = scanText; + for (const [start, end] of [...ranges].sort((a, b) => b[0] - a[0])) { + redacted = redacted.slice(0, start) + "[REDACTED]" + redacted.slice(end); + } + + return { + findings, + redacted: redacted.slice(0, MAX_REDACTED_TEXT_CHARS), + truncated: text.length > scanText.length || redacted.length > MAX_REDACTED_TEXT_CHARS, + }; +} diff --git a/storageops_cli/extensions/storageops.ts b/storageops_cli/extensions/storageops.ts index 785c54f..01f53bf 100644 --- a/storageops_cli/extensions/storageops.ts +++ b/storageops_cli/extensions/storageops.ts @@ -1,1186 +1,30 @@ /** - * StorageOps Pi Extension — v1.0 - * - * A lightweight Pi extension that provides object-storage diagnostic tools. - * Most tools run inline in the TypeScript runtime. capture_http_trace may run - * the external httpmon binary, but only through a bounded, read-only wrapper. - * - * Architecture: - * Pi ← storageops.ts (4 tools: scan_secrets, detect_domain, search_memory, capture_http_trace) - * ← skills/*.SKILL.md (16 packs: 13 diagnostic + triage + reporting + 1 eval) - * - * Placement: .pi/extensions/storageops.ts (auto-discovered by Pi) - * Reload: /reload inside Pi session + * StorageOps Pi Extension — entry point and public re-exports. + * Implementation lives in secrets.ts, routing.ts, memory.ts, and trace.ts. */ import type { ExtensionAPI } from "@earendil-works/pi-coding-agent"; import { Type } from "typebox"; -import * as crypto from "crypto"; import * as fs from "fs"; import * as path from "path"; -import * as os from "os"; -import * as childProcess from "child_process"; -// ── Secret Scanner ────────────────────────────────────────────────────────── -// Embedded regex patterns for credential detection. -// Patterns match: AWS AK/SK, tokens, Authorization headers, Alibaba/Tencent/Baidu -// Cloud AK/SK, rclone config secrets, private keys. - -const MAX_SECRET_SCAN_CHARS = 200_000; -const MAX_REDACTED_TEXT_CHARS = 20_000; -const SECRET_PATTERNS: Array<[RegExp, string]> = [ - // AWS access keys (AKIA...) - [/(?:AWS|aws)[\s_-]*(?:access[\s_-]*)?(?:key[\s_-]*id|akid)[\s]*[:=][\s]*([A-Z0-9]{16,})/gi, "AWS_ACCESS_KEY"], - [/(?:AKIA|ASIA)[A-Z0-9]{16}/g, "AWS_ACCESS_KEY_ID"], - // AWS secret keys — long alphanumeric with config keyword - [/(?:secret[\s_-]*)?(?:access[\s_-]*)?key[\s]*[:=][\s]*['"]?([A-Za-z0-9\/+=]{20,60})['"]?/gi, "AWS_SECRET_KEY"], - // AWS session tokens - [/(?:session[\s_-]*)?(?:token|x-amz-security-token)[\s]*[:=][\s]*['"]?([A-Za-z0-9\/+=]{100,})['"]?/gi, "AWS_SESSION_TOKEN"], - // Alibaba Cloud AK - [/(?:LTAI)[A-Za-z0-9]{16,20}/g, "ALIBABA_ACCESS_KEY"], - // Tencent Cloud SecretId - [/(?:AKID)[A-Za-z0-9]{32,48}/g, "TENCENT_SECRET_ID"], - // Baidu Cloud AK - [/(?:ak[\s]*=|access_key[\s]*=)[\s]*['"]?([a-f0-9]{32})['"]?/gi, "BAIDU_ACCESS_KEY"], - // Generic Authorization: Bearer / Basic tokens - [/Authorization[\s]*:[\s]*(?:Bearer|Basic|AWS4-HMAC-SHA256)[\s]+([^\s]{20,})/gi, "AUTHORIZATION_HEADER"], - // SigV4 Authorization headers carry the HMAC after the credential scope, e.g. - // "Credential=.../s3/aws4_request, Signature=". Redact the signature value - // while leaving the credential scope (date/region/service) — useful evidence — visible. - [/\bSignature=([0-9a-fA-F]{16,})/g, "SIGV4_SIGNATURE"], - // Private keys (PEM) — incl. plain PKCS8 "PRIVATE KEY" used by GCP service-account keys - [/-----BEGIN (?:(?:RSA|EC|DSA|OPENSSH|ENCRYPTED) )?PRIVATE KEY-----/g, "PRIVATE_KEY"], - // rclone config passwords - [/(?:pass|password|token|secret)[\s]*=[\s]*['"]?([^\s'"]{8,})['"]?/gi, "RCLONE_CREDENTIAL"], - // Generic API keys (sk-... for OpenAI/DeepSeek style) - [/(?:api[\s_-]*)?(?:key|token)[\s]*[:=][\s]*['"]?(sk-[A-Za-z0-9]{20,})['"]?/gi, "API_KEY"], - // GitHub tokens (ghp_, gho_, github_pat_) - [/(?:ghp_|gho_|github_pat_)[A-Za-z0-9]{36,}/g, "GITHUB_TOKEN"], - // Presigned-URL signature material — extremely common in rclone/aws/s5cmd debug logs - [/[?&](?:X-Amz-Signature|X-Goog-Signature)=([A-Za-z0-9%]{16,})/gi, "PRESIGNED_SIGNATURE"], - [/[?&]X-Amz-(?:Credential|Security-Token)=([^&\s]{16,})/gi, "PRESIGNED_AWS_PARAM"], - [/[?&](?:OSSAccessKeyId|Signature)=([^&\s]{10,})/gi, "OSS_PRESIGNED"], - [/[?&]q-(?:signature|ak)=([^&\s]{8,})/gi, "COS_PRESIGNED"], - // GCP service-account key id (the PEM private_key itself is caught above) - [/"private_key_id"[\s]*:[\s]*"([a-f0-9]{16,})"/gi, "GCP_PRIVATE_KEY_ID"], - // Azure storage account key + SAS signature - [/AccountKey=([A-Za-z0-9+\/=]{40,})/gi, "AZURE_ACCOUNT_KEY"], - [/[?&]sig=([A-Za-z0-9%]{20,})/gi, "AZURE_SAS"], -]; - -function secretFingerprint(value: string): string { - return "sha256:" + crypto.createHash("sha256").update(value).digest("hex").slice(0, 12); -} - -type SecretFinding = { - line: number; - column: number; - type: string; - length: number; - fingerprint: string; -}; - -function matchSecretRange(match: RegExpMatchArray): { start: number; end: number; value: string } { - const matchStart = match.index ?? 0; - const full = match[0] || ""; - const captured = match.slice(1).find(v => typeof v === "string" && v.length > 0); - if (captured) { - const offset = full.lastIndexOf(captured); - if (offset >= 0) { - return { start: matchStart + offset, end: matchStart + offset + captured.length, value: captured }; - } - } - return { start: matchStart, end: matchStart + full.length, value: full }; -} - -function lineAndColumn(text: string, index: number): { line: number; column: number } { - const prefix = text.slice(0, index); - const line = prefix.split("\n").length; - const lastNewline = prefix.lastIndexOf("\n"); - return { line, column: index - lastNewline }; -} - -export function redactText(text: string): { findings: SecretFinding[]; redacted: string; truncated: boolean } { - const scanText = text.slice(0, MAX_SECRET_SCAN_CHARS); - const findings: SecretFinding[] = []; - const ranges: Array<[number, number]> = []; - - for (const [pattern, type] of SECRET_PATTERNS) { - // Reset lastIndex for global regex - pattern.lastIndex = 0; - const matches = Array.from(scanText.matchAll(pattern)); - for (const m of matches) { - const { start, end, value } = matchSecretRange(m); - if (ranges.some(([rangeStart, rangeEnd]) => start < rangeEnd && end > rangeStart)) { - continue; - } - const { line, column } = lineAndColumn(scanText, start); - const length = value.length; - const fingerprint = secretFingerprint(value); - ranges.push([start, end]); - findings.push({ line, column, type, length, fingerprint }); - } - } - - let redacted = scanText; - for (const [start, end] of [...ranges].sort((a, b) => b[0] - a[0])) { - redacted = redacted.slice(0, start) + "[REDACTED]" + redacted.slice(end); - } - - return { - findings, - redacted: redacted.slice(0, MAX_REDACTED_TEXT_CHARS), - truncated: text.length > scanText.length || redacted.length > MAX_REDACTED_TEXT_CHARS, - }; -} - - -// ── Domain Detection ──────────────────────────────────────────────────────── -// Signature-based domain classification from evidence text. -// Replaces the old Python storageops/utils/signatures.py - -const DOMAIN_SIGNATURES: Record> = { - "storageops-security-iam-policy": [ - [/403\s*(?:Forbidden|Access\s*Denied)/i, "access_denied"], - [/AccessDenied/i, "access_denied_api"], - [/InvalidAccessKeyId/i, "invalid_key"], - [/kms:\w|\bKMS\b.*\b(key|encrypt|decrypt|SSE|cipher|grant|CMK|GenerateDataKey)\b|\b(key|encrypt|decrypt|SSE|cipher|CMK)\b.*\bKMS\b/i, "kms_error"], - [/401\s*Unauthorized|\bUnauthorized\b.*\b(request|access|operation|principal|credential|token|user|role|perform|API|HTTP|header|signature)\b|\b(request|access|operation|principal|credential|token|HTTP|API|signature)\b.*\bUnauthorized\b/i, "unauthorized"], - [/AssumeRole|sts:/i, "role_error"], - [/权限|无权限|鉴权|拒绝访问|访问被拒/i, "access_denied_cjk"], - ], - "storageops-s3-protocol-compatibility": [ - [/SignatureDoesNotMatch|AuthorizationHeaderMalformed|InvalidArgument/i, "signature_or_protocol_error"], - [/RequestTimeTooSkewed|RequestExpired|NotImplemented|MissingContentLength|EntityTooLarge|EntityTooSmall|PreconditionFailed/i, "protocol_error_code"], - [/\bBadDigest(?:SHA256|MD5)?\b/i, "payload_digest"], - [/CanonicalRequest|StringToSign|AWS4-HMAC-SHA256|SigV4|SigV2/i, "signature_debug"], - [/CORS|preflight|Access-Control-Allow-Origin|Access-Control-Allow-Methods/i, "cors"], - [/MalformedXML|InvalidDigest|Content-MD5|x-(?:amz|bce)-content-sha256/i, "protocol_header"], - [/virtual.?hosted|path.?style|chunked|STREAMING-AWS4-HMAC-SHA256-PAYLOAD/i, "provider_compatibility"], - [/签名|校验和|校验值/i, "signature_or_protocol_cjk"], - ], - "storageops-performance-diagnosis": [ - [/429|TooManyRequests|RequestRateLimitExceeded/i, "rate_limit"], - [/SlowDown/i, "slow_down"], - [/throttl/i, "throttle"], - [/timeout|timed?\s*out/i, "timeout"], - [/bandwidth/i, "bandwidth"], - [/retry/i, "retry"], - [/限速|限流|超时|慢|带宽/i, "performance_cjk"], - ], - "storageops-network-endpoint-access": [ - [/DNS|Name\s*or\s*service\s*not\s*known|NXDOMAIN/i, "dns"], - [/Could\s*not\s*connect|Connection\s*refused|connect\s*ETIMEDOUT/i, "connectivity"], - [/TLS|SSL|\bcert(?:ificate)?\b/i, "tls"], - [/连接(?:失败|超时|被拒)|证书|解析失败|无法访问/i, "network_cjk"], - [/VPC|endpoint|ENDPOINT/i, "endpoint"], - [/host\s*unreachable|no\s*route/i, "route"], - [/RequestTimeout|connection\s*reset|reset\s*by\s*peer|broken\s*pipe|\bECONNRESET\b|\bEPIPE\b|unexpected\s*EOF/i, "transport"], - ], - "storageops-cli-sdk-diagnosis": [ - [/rclone/i, "rclone"], - [/s5cmd/i, "s5cmd"], - [/awscli|botocore|boto3/i, "aws_cli"], - [/\bbcecmd\b|\bgo-bcecli\b/i, "bcecmd"], - [/\bobsutil\b|\bobs:\/\//i, "obsutil"], - [/corrupted\s*on\s*transfer|multipart.*etag/i, "corruption"], - [/损坏|校验失败|传输失败/i, "corruption_cjk"], - ], - "storageops-replication-versioning": [ - [/replicat/i, "replication"], - [/CRR|SRR/i, "replication_type"], - [/\bversioning\b|version\s*id/i, "versioning"], - [/DeleteMarker/i, "delete_marker"], - [/sync\s*(?:lag|delay)/i, "sync_lag"], - ], - "storageops-lifecycle-cost": [ - [/lifecycle/i, "lifecycle"], - [/Standard_IA|Glacier|Deep_Archive/i, "storage_class"], - [/\bcost\b.*\b(storage|request|egress|transition|tier|bill|lifecycle|retrieval|IA|Glacier|Archive|GB|TB)\b|\b(storage|request|egress|transition|tier|bill|lifecycle|retrieval|IA|Glacier|Archive|GB|TB)\b.*\bcost\b|费用|计费|账单/i, "cost"], - [/transition|\bexpir(?:e|es|ed|ation|ing)?\b/i, "transition"], - [/objects.*small|small.*objects/i, "small_objects"], - ], - "storageops-mount-filesystem-workspace": [ - [/\bmount(?:ed|ing)?\b|\bFUSE\b|\bs3fs\b|\bgoofys\b/i, "mount"], - [/\bFUSE\b/i, "fuse"], - [/\bfilesystem\b/i, "filesystem"], - ], - "storageops-migration-sync": [ - [/migrat|搬迁|迁移/i, "migration"], - [/\b(?:rclone|s5cmd|obsutil|bcecmd|ossutil|coscli)\s+sync\b|\baws\s+s3\s+sync\b|\bmc\s+mirror\b|\bsync\b.*\b(?:s3|oss|cos|obs|bos|bucket|object|prefix|remote:|provider|checksum|verify|verification)\b|\b(?:s3|oss|cos|obs|bos|bucket|object|prefix|remote:|provider)\b.*\bsync\b|cp\s+-r/i, "sync"], - [/\btransfer(?:s|red|ring)?\b.*\b(?:s3|oss|cos|obs|bos|bucket|object|prefix|provider|cross-region|multipart)\b|\b(?:s3|oss|cos|obs|bos|bucket|object|prefix|provider|cross-region|multipart)\b.*\btransfer(?:s|red|ring)?\b/i, "transfer"], - ], - "storageops-data-consistency": [ - [/consistenc|一致性/i, "consistency"], - [/stale|陈旧/i, "stale"], - [/\bmismatch\b.*\b(etag|checksum|hash|md5|sha|content|object|data|digest|size|byte)\b|\b(etag|checksum|hash|md5|sha|content|object|data|digest)\b.*\bmismatch\b/i, "mismatch"], - [/checksum|ETag/i, "checksum"], - ], - "storageops-bigdata-pipeline": [ - [/\b(?:Spark|Hive|Flink|Hadoop|S3A|EMR)\b/i, "bigdata_engine"], - [/FileOutputCommitter|MagicCommitter|S3ACommitter|_temporary|speculative execution/i, "committer"], - [/partition|Parquet|Iceberg|Delta|Hudi/i, "table_or_partition"], - [/small files|many files|listing storm|listObjects/i, "small_file_query"], - ], - "storageops-event-notification": [ - [/notification|通知/i, "notification"], - [/event\s*notifications?|bucket\s*events?|\bSQS\b|\bLambda\b/i, "event"], - [/prefix filter|suffix filter|ObjectCreated|ObjectRemoved/i, "event_filter"], - ], - "storageops-access-log-analysis": [ - [/access\s*log|server\s*access\s*log/i, "access_log"], - [/log\s*(?:analysis|分析)|request\s*analysis|traffic\s*analysis/i, "log_analysis"], - [/403\s*spike|503\s*spike|error\s*rate|错误率/i, "error_spike"], - [/who\s+is\s+accessing|top\s*requester|requester/i, "requester"], - [/cost\s*attribution|费用归因|成本归因/i, "cost_attribution"], - ], -}; - -type DomainDetection = { - domain: string; - recommended_skill: string; - confidence: number; - subdomains: string[]; - signals: string[]; - next_action: string; -}; - -// ── Provider Detection ────────────────────────────────────────────────────── -// Object-storage misdiagnosis is dominated by applying AWS assumptions to a -// non-AWS provider. Identify the provider deterministically from endpoint hosts, -// vendor header prefixes, vendor CLIs, and URI schemes — so provider-specific -// quirks get applied even when the user never names the provider. Conservative: -// returns "unknown" unless a clear signal is present. Note: x-amz-* headers are -// shared by all S3-compatible providers, so they are NOT an AWS signal. - -type ProviderEntry = { - provider: string; - confidence: "high" | "medium" | "low"; - signals: string[]; - quirks_ref: string | null; -}; - -type ProviderDetection = ProviderEntry & { - // All providers detected in the evidence, strongest first. For migration/sync - // the source and destination differ and need different quirks; the top-level - // fields are the primary (strongest) provider for backward compatibility. - providers: ProviderEntry[]; -}; - -const PROVIDER_SIGNATURES: Array<[string, RegExp, string]> = [ - ["aws", /\.amazonaws\.com\b/i, "endpoint:amazonaws.com"], - ["bos", /\.bcebos\.com\b/i, "endpoint:bcebos.com"], - ["bos", /x-bce-|\bbcecmd\b|\bgo-bcecli\b|\bbos:\/\//i, "header/cli:bce"], - ["oss", /\.aliyuncs\.com\b/i, "endpoint:aliyuncs.com"], - ["oss", /x-oss-|\bossutil\b|\boss:\/\//i, "header/cli:oss"], - ["cos", /\.myqcloud\.com\b/i, "endpoint:myqcloud.com"], - ["cos", /x-cos-|\bcoscli\b|\bcoscmd\b|q-sign-algorithm|\bcos:\/\//i, "header/cli:cos"], - ["gcs", /storage\.googleapis\.com\b/i, "endpoint:googleapis.com"], - ["gcs", /x-goog-|\bgsutil\b|\bgs:\/\//i, "header/cli:goog"], - ["azure", /\.blob\.core\.windows\.net\b/i, "endpoint:blob.core.windows.net"], - ["azure", /x-ms-(?:blob|version|date|meta)|\baz\s+storage\b/i, "header/cli:azure"], - ["obs", /\.myhuaweicloud\.com\b|\bobs\.[a-z0-9-]+\.myhuaweicloud/i, "endpoint:myhuaweicloud.com"], - ["obs", /x-obs-|\bobsutil\b|\bobs:\/\//i, "header/cli:obs"], - ["minio", /\bMinIO\b|x-minio-/i, "marker:minio"], -]; - -const PROVIDER_QUIRKS_REF: Record = { - bos: "storageops-s3-protocol-compatibility/references/provider-quirks/bos.md", - oss: "storageops-s3-protocol-compatibility/references/provider-quirks/oss.md", - cos: "storageops-s3-protocol-compatibility/references/provider-quirks/cos.md", - minio: "storageops-s3-protocol-compatibility/references/provider-quirks/minio.md", -}; - -export function detectProvider(text: string): ProviderDetection { - const evidence = (text || "").slice(0, 100_000); - const hits: Record = {}; - for (const [provider, regex, label] of PROVIDER_SIGNATURES) { - regex.lastIndex = 0; - if (regex.test(evidence)) (hits[provider] ||= []).push(label); - } - const names = Object.keys(hits); - if (names.length === 0) { - return { provider: "unknown", confidence: "low", signals: [], quirks_ref: null, providers: [] }; - } - // Strongest provider = most signal hits; an endpoint host match is high confidence. - names.sort((a, b) => hits[b].length - hits[a].length); - const providers: ProviderEntry[] = names.map(name => ({ - provider: name, - confidence: hits[name].some(l => l.startsWith("endpoint:")) ? "high" : "medium", - signals: hits[name].map(l => `${name}:${l}`), - quirks_ref: PROVIDER_QUIRKS_REF[name] ?? null, - })); - const primary = providers[0]; - return { - provider: primary.provider, - confidence: primary.confidence, - signals: providers.flatMap(p => p.signals), - quirks_ref: primary.quirks_ref, - providers, - }; -} - -const DOMAIN_NEXT_ACTION: Record = { - "storageops-security-iam-policy": "Check identity, policy, key validity, bucket policy, and KMS constraints before changing permissions.", - "storageops-s3-protocol-compatibility": "Compare endpoint style, region, canonical request shape, signing version, and required headers.", - "storageops-performance-diagnosis": "Separate service throttling, client retry behavior, network latency, and object layout signals.", - "storageops-network-endpoint-access": "Verify DNS, endpoint, route, proxy, and TLS certificate evidence before testing application logic.", - "storageops-cli-sdk-diagnosis": "Confirm the exact CLI/SDK, config path, provider, endpoint, and version before applying reference docs.", - "storageops-replication-versioning": "Inspect versioning, delete markers, replication rules, and observed replication lag.", - "storageops-lifecycle-cost": "Check lifecycle rules, storage class transitions, request patterns, and dated pricing references.", - "storageops-mount-filesystem-workspace": "Treat mount tools as filesystem adapters; verify cache, FUSE, permissions, and consistency expectations.", - "storageops-migration-sync": "Verify read-only inventory, delta strategy, checksums, and idempotent sync planning.", - "storageops-data-consistency": "Collect timestamps, ETags/checksums, list/head differences, and cross-client observations.", - "storageops-bigdata-pipeline": "Inspect engine, committer, partition layout, speculative execution, and object-listing behavior.", - "storageops-event-notification": "Check event rules, prefix/suffix filters, target permissions, and delivery logs.", - "storageops-access-log-analysis": "Summarize request IDs, status spikes, top requesters, user agents, and time windows.", -}; - -export function detectDomain(text: string): DomainDetection[] { - const scores: Record; signals: string[] }> = {}; - const evidence = text.slice(0, 100_000); - - for (const [domain, patterns] of Object.entries(DOMAIN_SIGNATURES)) { - for (const [regex, subdomain] of patterns) { - regex.lastIndex = 0; - const match = regex.exec(evidence); - if (match) { - if (!scores[domain]) scores[domain] = { score: 0, subdomains: new Set(), signals: [] }; - scores[domain].score += 1; - scores[domain].subdomains.add(subdomain); - scores[domain].signals.push(match[0].slice(0, 80)); - } - } - } - - return Object.entries(scores) - .map(([domain, info]) => ({ - domain, - recommended_skill: domain, - confidence: Math.min(0.5 + info.score * 0.15, 0.95), - subdomains: Array.from(info.subdomains), - signals: info.signals.slice(0, 5), - next_action: DOMAIN_NEXT_ACTION[domain] || "Collect more evidence before choosing a specialized skill.", - })) - .sort((a, b) => b.confidence - a.confidence); -} - - -// ── Memory Search ─────────────────────────────────────────────────────────── -// Searches Pi session JSONL files for past diagnostic context. - -type MemoryResult = { - sessionId: string; - snippet: string; - updated: string; - source: "summary" | "jsonl"; - score: number; -}; - -export function searchTokens(query: string): string[] { - const ascii = query.toLowerCase().match(/[a-z0-9_\-:.]{3,}/g) || []; - // CJK queries carry no ASCII word tokens, so the old tokenizer returned [] and - // recall was empty for Chinese. Emit overlapping bigrams (and single chars for - // length-1 runs) so Chinese memory searches recall partial matches. - const cjkTokens: string[] = []; - for (const run of query.match(/[一-鿿]+/g) || []) { - if (run.length === 1) cjkTokens.push(run); - else for (let i = 0; i < run.length - 1; i++) cjkTokens.push(run.slice(i, i + 2)); - } - return Array.from(new Set([...ascii, ...cjkTokens])).slice(0, 12); -} - -function scoreText(text: string, tokens: string[]): number { - const lower = text.toLowerCase(); - return tokens.reduce((score, token) => score + (lower.includes(token) ? 1 : 0), 0); -} - -function safeMemorySnippet(text: string): string { - return redactText(text.replace(/\s+/g, " ").trim()).redacted.slice(0, 240); -} - -const MAX_SESSION_SCAN_DEPTH = 4; -const MAX_SESSION_FILES = 200; - -// Pi stores session transcripts under scope subdirectories -// (e.g. sessions//.jsonl), so a flat top-level scan misses them. -// Walk the sessions tree with bounded depth/count and index by .jsonl files; -// .meta.json is optional sibling enrichment, not required for recall. -export function collectSessionJsonl(root: string): string[] { - const found: string[] = []; - const walk = (dir: string, depth: number): void => { - if (depth > MAX_SESSION_SCAN_DEPTH || found.length >= MAX_SESSION_FILES) return; - let entries: fs.Dirent[]; - try { - entries = fs.readdirSync(dir, { withFileTypes: true }); - } catch { - return; - } - for (const entry of entries) { - if (found.length >= MAX_SESSION_FILES) return; - const full = path.join(dir, entry.name); - if (entry.isDirectory()) { - walk(full, depth + 1); - } else if (entry.isFile() && entry.name.endsWith(".jsonl")) { - found.push(full); - } - } - }; - walk(root, 0); - return found; -} - -function readSessionMeta(jsonlPath: string): { sessionId: string; summary: string; updated: string } { - const sessionId = path.basename(jsonlPath, ".jsonl"); - const metaPath = path.join(path.dirname(jsonlPath), `${sessionId}.meta.json`); - if (fs.existsSync(metaPath)) { - try { - const meta = JSON.parse(fs.readFileSync(metaPath, "utf8")); - return { - sessionId: meta.id || sessionId, - summary: meta.summary || meta.name || "", - updated: meta.updated || meta.created || "", - }; - } catch { - // Malformed meta; the jsonl content is still searchable. - } - } - return { sessionId, summary: "", updated: "" }; -} - -export function searchMemory(query: string, limit: number = 5): MemoryResult[] { - const agentDir = process.env.PI_CODING_AGENT_DIR || path.join(os.homedir(), ".pi", "agent"); - const primarySessionsDir = path.join(agentDir, "sessions"); - const fallbackSessionsDir = path.join(os.homedir(), ".pi", "agent", "sessions"); - const sessionsDir = fs.existsSync(primarySessionsDir) ? primarySessionsDir : fallbackSessionsDir; - if (!fs.existsSync(sessionsDir)) return []; - - const tokens = searchTokens(query); - if (tokens.length === 0) return []; - const cappedLimit = Math.min(Math.max(limit || 5, 1), 10); - - const jsonlFiles = collectSessionJsonl(sessionsDir) - .sort() - .reverse() - .slice(0, MAX_SESSION_FILES); - - const results: MemoryResult[] = []; - - for (const jsonlPath of jsonlFiles) { - try { - const { sessionId, summary, updated } = readSessionMeta(jsonlPath); - - const summaryScore = summary ? scoreText(summary, tokens) : 0; - if (summaryScore > 0) { - results.push({ - sessionId, - snippet: safeMemorySnippet(summary), - updated, - source: "summary", - score: summaryScore, - }); - } - - const content = fs.readFileSync(jsonlPath, "utf8").slice(0, 40_000); - const jsonlScore = scoreText(content, tokens); - if (jsonlScore > 0) { - const line = content.split(/\r?\n/).find(x => scoreText(x, tokens) > 0) || summary || `Session ${sessionId.slice(0, 8)}...`; - results.push({ - sessionId, - snippet: safeMemorySnippet(line), - updated, - source: "jsonl", - score: jsonlScore, - }); - } - } catch { - // Skip unreadable files - } - } - - // Keep only the best-scoring entry per session so one session can't occupy - // multiple result slots (a session matching in both summary and jsonl would - // otherwise crowd out other relevant sessions). - const bestBySession = new Map(); - for (const r of results) { - const prev = bestBySession.get(r.sessionId); - if (!prev || r.score > prev.score) bestBySession.set(r.sessionId, r); - } - - return Array.from(bestBySession.values()) - .sort((a, b) => b.score - a.score || String(b.updated).localeCompare(String(a.updated))) - .slice(0, cappedLimit); -} - - -// ── HTTP Trace Capture ────────────────────────────────────────────────────── -// Bounded wrapper around httpmon. This intentionally exposes only a narrow, -// read-only subset to Pi: no raw HAR, no record files, no replay, and no body -// capture in returned output. - -type TraceRequest = { - id: number; - method: string; - url: string; - host: string; - headers: Record; -}; - -type TraceResponse = { - req_id: number; - status: number; - headers: Record; - body?: string; -}; - -const MAX_TRACE_REQUESTS = 20; -const MAX_TRACE_SECONDS = 30; -const MAX_UNKNOWN_TRACE_REQUESTS = 5; -const MAX_UNKNOWN_TRACE_SECONDS = 15; -const MAX_TRACE_COMMAND_ARGS = 40; - -const READ_ONLY_AWS_S3API = new Set([ - "head-object", - "head-bucket", - "list-objects", - "list-objects-v2", - "list-buckets", - "list-object-versions", - "list-multipart-uploads", - "get-bucket-location", - "get-bucket-versioning", - "get-bucket-replication", - "get-bucket-encryption", - "get-bucket-policy-status", - "get-bucket-policy", - "get-bucket-logging", - "get-bucket-notification-configuration", - "get-bucket-website", - "get-bucket-request-payment", - "get-bucket-ownership-controls", - "get-bucket-object-lock-configuration", - "get-public-access-block", - "get-bucket-cors", - "get-bucket-lifecycle-configuration", - "get-bucket-tagging", - "get-bucket-acl", - "get-object-attributes", - "get-object-retention", - "get-object-legal-hold", -]); - -const READ_ONLY_CLIENT_OPS = new Set(["ls", "lsf", "lsd", "stat", "head"]); -const MUTATING_CLIENT_OPS = new Set([ - "rm", - "rb", - "mb", - "mv", - "cp", - "sync", - "copy", - "delete", - "purge", - "move", -]); - -const MUTATING_AWS_S3API = new Set([ - "put-object", - "delete-object", - "delete-objects", - "delete-bucket", - "create-bucket", - "copy-object", - "complete-multipart-upload", - "abort-multipart-upload", - "restore-object", - "put-bucket", - "put-object-acl", - "put-bucket-acl", - "put-bucket-policy", - "delete-bucket-policy", -]); - -const KNOWN_TRACE_CLIENTS = new Set(["aws", "curl", "rclone", "s5cmd", "mc", "bcecmd", "obsutil", "s3cmd"]); -const UNKNOWN_MUTATING_TOKENS = new Set([ - "post", - "put", - "delete", - "remove", - "upload", - "copy", - "sync", - "move", - "create", - "update", - "patch", - "rm", - "mv", - "cp", -]); -const WRITE_HTTP_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); - -const SIGNED_QUERY_KEYS = [ - "x-amz-signature", - "x-amz-security-token", - "x-amz-credential", - "x-oss-signature", - "x-oss-credential", - "ossaccesskeyid", - "signature", - "security-token", -]; - -function normalizeFilterHost(value: string): string { - const input = (value || "").trim(); - if (!input) return ""; - try { - if (/^https?:\/\//i.test(input)) { - return new URL(input).host.toLowerCase(); - } - } catch { - return ""; - } - const host = input.split("/")[0].toLowerCase(); - if (!/^[a-z0-9.-]+(?::\d+)?$/i.test(host)) return ""; - return host; -} - -function hasSignedQueryMaterial(command: string[]): boolean { - const text = command.join(" ").toLowerCase(); - return SIGNED_QUERY_KEYS.some(key => text.includes(key)); -} - -function firstKnownClientOp(args: string[]): string { - for (const arg of args.slice(1)) { - if (READ_ONLY_CLIENT_OPS.has(arg) || MUTATING_CLIENT_OPS.has(arg)) return arg; - } - return ""; -} - -function curlMethod(command: string[], lowered: string[]): string { - for (let i = 1; i < lowered.length; i += 1) { - const rawArg = command[i]; - const arg = lowered[i]; - if (rawArg === "-X" || arg === "--request") { - return (lowered[i + 1] || "get").toUpperCase(); - } - if (rawArg.startsWith("-X") && rawArg.length > 2) { - return rawArg.slice(2).toUpperCase(); - } - if (arg.startsWith("--request=")) { - return arg.slice("--request=".length).toUpperCase(); - } - } - const usesHeadFlag = command.includes("-I") || lowered.includes("--head"); - return usesHeadFlag ? "HEAD" : "GET"; -} - -function commandHosts(command: string[]): string[] { - const hosts: string[] = []; - for (let i = 0; i < command.length; i += 1) { - const arg = command[i]; - let candidate = arg; - if (arg === "--url" || arg === "-url") { - candidate = command[i + 1] || ""; - } else if (arg.startsWith("--url=")) { - candidate = arg.slice("--url=".length); - } - if (!/^https?:\/\//i.test(candidate)) continue; - try { - hosts.push(new URL(candidate).host.toLowerCase()); - } catch { - // Ignore malformed URL-like arguments; the wrapped tool will report them. - } - } - return Array.from(new Set(hosts)); -} - -function curlHasBodyUploadFlag(command: string[], lowered: string[]): boolean { - const exactLower = new Set(["--data", "--data-raw", "--data-binary", "--form", "--upload-file"]); - return command.some((raw, index) => { - const lower = lowered[index]; - return raw === "-d" - || raw.startsWith("-d") - || raw === "-F" - || raw.startsWith("-F") - || raw === "-T" - || raw.startsWith("-T") - || exactLower.has(lower) - || lower.startsWith("--data=") - || lower.startsWith("--data-raw=") - || lower.startsWith("--data-binary=") - || lower.startsWith("--form=") - || lower.startsWith("--upload-file="); - }); -} - -function clientPolicyForCommand(command: string[]): "known_adapter" | "unknown_observation" { - const exe = path.basename(command[0] || "").toLowerCase(); - return KNOWN_TRACE_CLIENTS.has(exe) ? "known_adapter" : "unknown_observation"; -} - -function explicitHttpWriteMethod(command: string[], lowered: string[]): string { - for (let i = 1; i < lowered.length; i += 1) { - const rawArg = command[i]; - const arg = lowered[i]; - if (rawArg === "-X" || arg === "--request" || arg === "--method") { - const method = (lowered[i + 1] || "").toUpperCase(); - if (WRITE_HTTP_METHODS.has(method)) return method; - } - if (rawArg.startsWith("-X") && rawArg.length > 2) { - const method = rawArg.slice(2).toUpperCase(); - if (WRITE_HTTP_METHODS.has(method)) return method; - } - for (const prefix of ["--request=", "--method="]) { - if (arg.startsWith(prefix)) { - const method = arg.slice(prefix.length).toUpperCase(); - if (WRITE_HTTP_METHODS.has(method)) return method; - } - } - } - return ""; -} - -function suspiciousUnknownToken(command: string[]): string { - const lowered = command.map(x => x.toLowerCase()); - return lowered.find((arg, index) => index > 0 && UNKNOWN_MUTATING_TOKENS.has(arg)) || ""; -} - -function validateUnknownTraceCommand(command: string[]): string[] { - const lowered = command.map(x => x.toLowerCase()); - const method = explicitHttpWriteMethod(command, lowered); - return method ? [`explicit HTTP write method is not allowed: ${method}`] : []; -} - -function operationUnclassified(command: string[]): boolean { - const exe = path.basename(command[0] || "").toLowerCase(); - const lowered = command.map(x => x.toLowerCase()); - if (!KNOWN_TRACE_CLIENTS.has(exe)) return true; - if (["rclone", "s5cmd", "mc", "bcecmd", "obsutil", "s3cmd"].includes(exe)) { - return !firstKnownClientOp(lowered); - } - return false; -} - -function traceHostMismatch(command: string[], filterHost: string): boolean { - const hosts = commandHosts(command); - return hosts.length > 0 && Boolean(filterHost) && !hosts.some(host => host === filterHost); -} - -function traceWarningsForCommand(command: string[], filterHost: string): string[] { - const warnings: string[] = []; - if (traceHostMismatch(command, filterHost)) { - warnings.push("command URL host differs from filter_host; trace may capture zero requests"); - } - const suspicious = suspiciousUnknownToken(command); - if (!KNOWN_TRACE_CLIENTS.has(path.basename(command[0] || "").toLowerCase()) && suspicious) { - warnings.push(`unknown client argument looks mutating but will be observed in bounded metadata mode: ${suspicious}`); - } - if (operationUnclassified(command)) { - warnings.push("operation is unclassified; using bounded metadata observation"); - } - return warnings; -} - -export function validateTraceCommand(command: string[], filterHost: string, captureBody: boolean): string[] { - const errors: string[] = []; - if (!Array.isArray(command) || command.length === 0) { - errors.push("command must be a non-empty argv array"); - return errors; - } - if (command.length > MAX_TRACE_COMMAND_ARGS) { - errors.push(`command has too many arguments; max is ${MAX_TRACE_COMMAND_ARGS}`); - } - if (!filterHost) { - errors.push("filter_host is required and must be a host name, not a broad substring"); - } - if (captureBody) { - errors.push("capture_body=true is not supported in the safe default tool"); - } - for (const arg of command) { - if (typeof arg !== "string" || arg.length === 0) { - errors.push("all command arguments must be non-empty strings"); - break; - } - } - if (hasSignedQueryMaterial(command)) { - errors.push("presigned URL material is not accepted by capture_http_trace; provide a redacted trace instead"); - } - - const exe = path.basename(command[0] || "").toLowerCase(); - const lowered = command.map(x => x.toLowerCase()); - const blockedShells = new Set(["sh", "bash", "zsh", "fish", "pwsh", "powershell", "sudo"]); - if (blockedShells.has(exe)) { - errors.push("shells and sudo are not allowed for HTTP trace capture"); - } - - if (exe === "aws") { - const s3api = lowered.indexOf("s3api"); - const s3 = lowered.indexOf("s3"); - if (s3api >= 0) { - const op = lowered[s3api + 1] || ""; - if (MUTATING_AWS_S3API.has(op)) { - errors.push(`mutating or high-risk operation is not allowed: ${op}`); - } - if (!READ_ONLY_AWS_S3API.has(op)) { - errors.push(`aws s3api operation is not in the read-only allowlist: ${op || "(missing)"}`); - } - } else if (s3 >= 0) { - const op = lowered[s3 + 1] || ""; - if (op !== "ls") { - errors.push(`aws s3 operation is not in the read-only allowlist: ${op || "(missing)"}`); - } - } else { - errors.push("aws capture requires an s3 or s3api read-only operation"); - } - } else if (exe === "curl") { - const method = curlMethod(command, lowered); - if (!["GET", "HEAD", "OPTIONS"].includes(method)) { - errors.push(`curl method is not read-only: ${method}`); - } - if (curlHasBodyUploadFlag(command, lowered)) { - errors.push("curl body upload flags are not allowed"); - } - } else if (["rclone", "s5cmd", "mc", "bcecmd", "obsutil", "s3cmd"].includes(exe)) { - const op = firstKnownClientOp(lowered); - if (MUTATING_CLIENT_OPS.has(op)) { - errors.push(`mutating or high-risk operation is not allowed: ${op}`); - } - if (op && !READ_ONLY_CLIENT_OPS.has(op)) { - errors.push(`${exe} operation is not in the read-only allowlist: ${op}`); - } - } else { - errors.push(...validateUnknownTraceCommand(command)); - } - - return [...new Set(errors)]; -} - -const WRITE_REJECTION_MARKERS = [ - "mutating", - "read-only allowlist", - "write method", - "method is not read-only", -]; - -// When a trace is rejected because the command writes (or is not provably -// read-only), point the agent at the evidence ladder instead of leaving a dead -// end. capture_http_trace executes commands, so tracing a write would perform a -// real request; the request shape is recoverable without re-sending the write. -export function traceRejectionGuidance(errors: string[]): string { - const isWriteRejection = (errors || []).some(err => - WRITE_REJECTION_MARKERS.some(marker => err.toLowerCase().includes(marker)), - ); - if (!isWriteRejection) return ""; - return ( - "This command writes or is not provably read-only, and capture_http_trace " + - "executes commands — tracing it would perform a real request. To diagnose a " + - "failing write, read the server error body and the client's own debug dump " + - "(aws --debug / rclone -vv --dump headers / boto3 set_stream_logger), then " + - "recompute offline. See storageops-s3-protocol-compatibility/references/" + - "checksum-etag.md (Write-side request evidence). Re-run capture_http_trace " + - "only on a read-only command." - ); -} - -function getHeader(headers: Record | undefined, name: string): string { - if (!headers) return ""; - const target = name.toLowerCase(); - for (const [key, value] of Object.entries(headers)) { - if (key.toLowerCase() === target) return String(value); - } - return ""; -} - -function safePathTemplate(rawURL: string): string { - try { - const u = new URL(rawURL); - const parts = u.pathname.split("/").map(part => { - if (!part) return part; - if (part.length > 48) return ":segment"; - if (/^(AKIA|ASIA|AKID|LTAI|sk-)/i.test(part)) return "[REDACTED]"; - if (/^[a-f0-9]{32,}$/i.test(part)) return ":hex"; - return part; - }); - const capped = parts.length > 7 ? parts.slice(0, 7).concat(["..."]) : parts; - return capped.join("/") || "/"; - } catch { - return "/"; - } -} - -function parseAuthShape(headers: Record | undefined): { - auth_header_present: boolean; - auth_scheme?: string; - signed_headers?: string[]; - credential_scope?: string; - credential_scope_region?: string; - credential_scope_service?: string; -} { - const auth = getHeader(headers, "Authorization"); - if (!auth) return { auth_header_present: false }; - - const scheme = auth.split(/\s+/)[0] || "unknown"; - const signed = /SignedHeaders=([^,\s]+)/i.exec(auth)?.[1] || ""; - const credential = /Credential=([^,\s]+)/i.exec(auth)?.[1] || ""; - let scope = ""; - let region = ""; - let service = ""; - if (credential) { - try { - const decoded = decodeURIComponent(credential); - const parts = decoded.split("/"); - if (parts.length >= 5) { - scope = parts.slice(1).join("/"); - region = parts[2] || ""; - service = parts[3] || ""; - } - } catch { - scope = ""; - } - } - return { - auth_header_present: true, - auth_scheme: scheme, - signed_headers: signed ? signed.split(/%3B|;/i).filter(Boolean) : [], - credential_scope: scope || undefined, - credential_scope_region: region || undefined, - credential_scope_service: service || undefined, - }; -} - -function queryKeySummary(rawURL: string): { has_presigned_query: boolean; query_keys: string[] } { - try { - const u = new URL(rawURL); - const keys = Array.from(new Set(Array.from(u.searchParams.keys()).map(k => k.toLowerCase()))).sort(); - const hasSigned = keys.some(k => SIGNED_QUERY_KEYS.includes(k)); - return { has_presigned_query: hasSigned, query_keys: keys.slice(0, 20) }; - } catch { - return { has_presigned_query: false, query_keys: [] }; - } -} - -function extractS3ErrorCode(body: string | undefined): string | undefined { - if (!body) return undefined; - const text = body.slice(0, 4096); - return /([^<]{1,80})<\/Code>/i.exec(text)?.[1] - || /"Code"\s*:\s*"([^"]{1,80})"/i.exec(text)?.[1] - || /"code"\s*:\s*"([^"]{1,80})"/i.exec(text)?.[1]; -} - -const SENSITIVE_RESPONSE_HEADERS = new Set([ - "set-cookie", - "www-authenticate", - "proxy-authenticate", - "authorization", -]); -const REDIRECT_HEADERS = new Set(["location", "content-location"]); -const MAX_RESPONSE_HEADERS = 32; -const MAX_HEADER_VALUE_CHARS = 256; - -// Surface response metadata (the diagnostic payload) with targeted sanitization: -// known credential-bearing headers (cookies/auth challenges) keep their name but -// have the value masked; redirect targets are redacted of presigned material -// (which can embed a replayable signature); all other headers pass through so -// metadata like ETag, retention-date, SSE, and checksums is not over-redacted. -export function sanitizeResponseHeaders( - headers: Record | undefined, -): Array<{ name: string; value: string }> { - if (!headers) return []; - const out: Array<{ name: string; value: string }> = []; - for (const [rawName, rawValue] of Object.entries(headers)) { - if (out.length >= MAX_RESPONSE_HEADERS) break; - const name = rawName.toLowerCase(); - let value = String(rawValue ?? ""); - if (SENSITIVE_RESPONSE_HEADERS.has(name)) { - value = "[REDACTED]"; - } else if (REDIRECT_HEADERS.has(name)) { - value = redactText(value).redacted; - } - out.push({ name, value: value.slice(0, MAX_HEADER_VALUE_CHARS) }); - } - return out; -} - -export function summarizeTraceRequest(req: TraceRequest, resp?: TraceResponse) { - const auth = parseAuthShape(req.headers); - const query = queryKeySummary(req.url); - let host = req.host || ""; - try { - host = new URL(req.url).host || host; - } catch { - // keep req.host - } - return { - id: req.id, - method: req.method, - method_violation: !["GET", "HEAD", "OPTIONS"].includes(String(req.method || "").toUpperCase()), - host, - path_template: safePathTemplate(req.url), - status: resp?.status, - s3_error_code: extractS3ErrorCode(resp?.body), - auth_header_present: auth.auth_header_present, - auth_scheme: auth.auth_scheme, - signed_headers: auth.signed_headers, - credential_scope: auth.credential_scope, - credential_scope_region: auth.credential_scope_region, - credential_scope_service: auth.credential_scope_service, - has_presigned_query: query.has_presigned_query, - query_keys: query.query_keys, - response_headers: sanitizeResponseHeaders(resp?.headers), - }; -} - -function findHttpmonBinary(): string { - const configured = process.env.STORAGEOPS_HTTPMON; - if (configured && fs.existsSync(configured)) return configured; - const managed = path.join(os.homedir(), ".storageops", "bin", process.platform === "win32" ? "httpmon.exe" : "httpmon"); - if (fs.existsSync(managed)) return managed; - return "httpmon"; -} - -async function captureHttpTrace(params: { - command: string[]; - filter_host: string; - max_requests?: number; - max_seconds?: number; - capture_body?: boolean; -}) { - const filterHost = normalizeFilterHost(params.filter_host || ""); - const clientPolicy = clientPolicyForCommand(params.command || []); - const warnings = traceWarningsForCommand(params.command || [], filterHost); - const hostMismatch = traceHostMismatch(params.command || [], filterHost); - const opUnclassified = operationUnclassified(params.command || []); - const strictObservation = clientPolicy === "unknown_observation" || opUnclassified; - const requestCap = strictObservation ? MAX_UNKNOWN_TRACE_REQUESTS : MAX_TRACE_REQUESTS; - const secondsCap = strictObservation ? MAX_UNKNOWN_TRACE_SECONDS : MAX_TRACE_SECONDS; - const maxRequests = Math.min(Math.max(Number(params.max_requests || requestCap), 1), requestCap); - const maxSeconds = Math.min(Math.max(Number(params.max_seconds || secondsCap), 1), secondsCap); - const captureBody = Boolean(params.capture_body); - const validationErrors = validateTraceCommand(params.command, filterHost, captureBody); - if (validationErrors.length > 0) { - return { - status: "rejected", - reason: "unsafe_or_unsupported_command", - client_policy: clientPolicy, - warnings, - host_mismatch: hostMismatch, - operation_unclassified: opUnclassified, - errors: validationErrors, - guidance: traceRejectionGuidance(validationErrors) || undefined, - limits: { max_requests: maxRequests, max_seconds: maxSeconds, capture_body: false }, - }; - } - - return new Promise(resolve => { - const httpmonBinary = findHttpmonBinary(); - const httpmon = childProcess.spawn( - httpmonBinary, - ["--format", "json", "--filter", filterHost, ...params.command], - { stdio: ["ignore", "pipe", "pipe"] }, - ); - - const requests = new Map(); - const responses = new Map(); - let buffer = ""; - let stderr = ""; - let killedForLimit = false; - let spawnError = ""; - let finished = false; - - const finish = (exitCode: number | null) => { - if (finished) return; - finished = true; - clearTimeout(timer); - const summaries = Array.from(requests.values()) - .slice(0, maxRequests) - .map(req => summarizeTraceRequest(req, responses.get(req.id))); - const redactedStderr = redactText(stderr.slice(0, 2000)).redacted; - resolve({ - status: spawnError ? "error" : "completed", - command_name: path.basename(params.command[0]), - client_policy: clientPolicy, - warnings, - host_mismatch: hostMismatch, - operation_unclassified: opUnclassified, - filter_host: filterHost, - exit_code: exitCode, - killed_for_limit: killedForLimit, - requests: summaries, - request_count: summaries.length, - method_violation: summaries.some((r: any) => r.method_violation), - stderr_summary: redactedStderr.slice(0, 500), - redaction: { - authorization_redacted: summaries.some((r: any) => r.auth_header_present), - presigned_query_redacted: summaries.some((r: any) => r.has_presigned_query), - response_headers_sanitized: true, - body_captured: false, - raw_trace_saved: false, - har_saved: false, - replay_performed: false, - }, - limits: { max_requests: maxRequests, max_seconds: maxSeconds, capture_body: false }, - error: spawnError || undefined, - }); - }; - - const handleLine = (line: string) => { - const trimmed = line.trim(); - if (!trimmed.startsWith("{")) return; - try { - const event = JSON.parse(trimmed); - if (typeof event.id === "number" && typeof event.method === "string" && typeof event.url === "string") { - requests.set(event.id, { - id: event.id, - method: event.method, - url: event.url, - host: event.host || "", - headers: event.headers || {}, - }); - if (requests.size >= maxRequests && httpmon.pid) { - killedForLimit = true; - httpmon.kill("SIGTERM"); - } - } else if (typeof event.req_id === "number" && typeof event.status === "number") { - responses.set(event.req_id, { - req_id: event.req_id, - status: event.status, - headers: event.headers || {}, - body: event.body || "", - }); - } - } catch { - // Ignore subprocess output that is not httpmon NDJSON. - } - }; - - const timer = setTimeout(() => { - killedForLimit = true; - if (httpmon.pid) httpmon.kill("SIGTERM"); - }, maxSeconds * 1000); - - httpmon.stdout?.on("data", chunk => { - buffer += chunk.toString("utf8"); - const lines = buffer.split(/\r?\n/); - buffer = lines.pop() || ""; - for (const line of lines) handleLine(line); - }); - httpmon.stderr?.on("data", chunk => { - stderr += chunk.toString("utf8"); - }); - httpmon.on("error", err => { - spawnError = err.message || String(err); - finish(null); - }); - httpmon.on("close", code => { - if (buffer.trim()) handleLine(buffer); - finish(code); - }); - }); -} - - -// ── Extension Entry Point ─────────────────────────────────────────────────── +export { redactText } from "./secrets.ts"; +export type { SecretFinding } from "./secrets.ts"; +export { detectDomain, detectProvider } from "./routing.ts"; +export type { DomainDetection, ProviderDetection } from "./routing.ts"; +export { searchTokens, searchMemory, collectSessionJsonl } from "./memory.ts"; +export type { MemoryResult } from "./memory.ts"; +export { + validateTraceCommand, + traceRejectionGuidance, + sanitizeResponseHeaders, + summarizeTraceRequest, +} from "./trace.ts"; + +import { redactText } from "./secrets.ts"; +import { detectDomain, detectProvider } from "./routing.ts"; +import { searchMemory } from "./memory.ts"; +import { captureHttpTrace } from "./trace.ts"; export default function (pi: ExtensionAPI) { // ── Tool: scan_secrets ── diff --git a/storageops_cli/extensions/trace.ts b/storageops_cli/extensions/trace.ts new file mode 100644 index 0000000..2e9e93f --- /dev/null +++ b/storageops_cli/extensions/trace.ts @@ -0,0 +1,666 @@ +import * as fs from "fs"; +import * as path from "path"; +import * as os from "os"; +import * as childProcess from "child_process"; +import { redactText } from "./secrets.ts"; + +type TraceRequest = { + id: number; + method: string; + url: string; + host: string; + headers: Record; +}; + +type TraceResponse = { + req_id: number; + status: number; + headers: Record; + body?: string; +}; + +const MAX_TRACE_REQUESTS = 20; +const MAX_TRACE_SECONDS = 30; +const MAX_UNKNOWN_TRACE_REQUESTS = 5; +const MAX_UNKNOWN_TRACE_SECONDS = 15; +const MAX_TRACE_COMMAND_ARGS = 40; + +const READ_ONLY_AWS_S3API = new Set([ + "head-object", + "head-bucket", + "list-objects", + "list-objects-v2", + "list-buckets", + "list-object-versions", + "list-multipart-uploads", + "get-bucket-location", + "get-bucket-versioning", + "get-bucket-replication", + "get-bucket-encryption", + "get-bucket-policy-status", + "get-bucket-policy", + "get-bucket-logging", + "get-bucket-notification-configuration", + "get-bucket-website", + "get-bucket-request-payment", + "get-bucket-ownership-controls", + "get-bucket-object-lock-configuration", + "get-public-access-block", + "get-bucket-cors", + "get-bucket-lifecycle-configuration", + "get-bucket-tagging", + "get-bucket-acl", + "get-object-attributes", + "get-object-retention", + "get-object-legal-hold", +]); + +const READ_ONLY_CLIENT_OPS = new Set(["ls", "lsf", "lsd", "stat", "head"]); +const MUTATING_CLIENT_OPS = new Set([ + "rm", + "rb", + "mb", + "mv", + "cp", + "sync", + "copy", + "delete", + "purge", + "move", +]); + +const MUTATING_AWS_S3API = new Set([ + "put-object", + "delete-object", + "delete-objects", + "delete-bucket", + "create-bucket", + "copy-object", + "complete-multipart-upload", + "abort-multipart-upload", + "restore-object", + "put-bucket", + "put-object-acl", + "put-bucket-acl", + "put-bucket-policy", + "delete-bucket-policy", +]); + +const KNOWN_TRACE_CLIENTS = new Set(["aws", "curl", "rclone", "s5cmd", "mc", "bcecmd", "obsutil", "s3cmd"]); +const UNKNOWN_MUTATING_TOKENS = new Set([ + "post", + "put", + "delete", + "remove", + "upload", + "copy", + "sync", + "move", + "create", + "update", + "patch", + "rm", + "mv", + "cp", +]); +const WRITE_HTTP_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); + +const SIGNED_QUERY_KEYS = [ + "x-amz-signature", + "x-amz-security-token", + "x-amz-credential", + "x-oss-signature", + "x-oss-credential", + "ossaccesskeyid", + "signature", + "security-token", +]; + +function normalizeFilterHost(value: string): string { + const input = (value || "").trim(); + if (!input) return ""; + try { + if (/^https?:\/\//i.test(input)) { + return new URL(input).host.toLowerCase(); + } + } catch { + return ""; + } + const host = input.split("/")[0].toLowerCase(); + if (!/^[a-z0-9.-]+(?::\d+)?$/i.test(host)) return ""; + return host; +} + +function hasSignedQueryMaterial(command: string[]): boolean { + const text = command.join(" ").toLowerCase(); + return SIGNED_QUERY_KEYS.some(key => text.includes(key)); +} + +function firstKnownClientOp(args: string[]): string { + for (const arg of args.slice(1)) { + if (READ_ONLY_CLIENT_OPS.has(arg) || MUTATING_CLIENT_OPS.has(arg)) return arg; + } + return ""; +} + +function curlMethod(command: string[], lowered: string[]): string { + for (let i = 1; i < lowered.length; i += 1) { + const rawArg = command[i]; + const arg = lowered[i]; + if (rawArg === "-X" || arg === "--request") { + return (lowered[i + 1] || "get").toUpperCase(); + } + if (rawArg.startsWith("-X") && rawArg.length > 2) { + return rawArg.slice(2).toUpperCase(); + } + if (arg.startsWith("--request=")) { + return arg.slice("--request=".length).toUpperCase(); + } + } + const usesHeadFlag = command.includes("-I") || lowered.includes("--head"); + return usesHeadFlag ? "HEAD" : "GET"; +} + +function commandHosts(command: string[]): string[] { + const hosts: string[] = []; + for (let i = 0; i < command.length; i += 1) { + const arg = command[i]; + let candidate = arg; + if (arg === "--url" || arg === "-url") { + candidate = command[i + 1] || ""; + } else if (arg.startsWith("--url=")) { + candidate = arg.slice("--url=".length); + } + if (!/^https?:\/\//i.test(candidate)) continue; + try { + hosts.push(new URL(candidate).host.toLowerCase()); + } catch { + // Ignore malformed URL-like arguments; the wrapped tool will report them. + } + } + return Array.from(new Set(hosts)); +} + +function curlHasBodyUploadFlag(command: string[], lowered: string[]): boolean { + const exactLower = new Set(["--data", "--data-raw", "--data-binary", "--form", "--upload-file"]); + return command.some((raw, index) => { + const lower = lowered[index]; + return raw === "-d" + || raw.startsWith("-d") + || raw === "-F" + || raw.startsWith("-F") + || raw === "-T" + || raw.startsWith("-T") + || exactLower.has(lower) + || lower.startsWith("--data=") + || lower.startsWith("--data-raw=") + || lower.startsWith("--data-binary=") + || lower.startsWith("--form=") + || lower.startsWith("--upload-file="); + }); +} + +function clientPolicyForCommand(command: string[]): "known_adapter" | "unknown_observation" { + const exe = path.basename(command[0] || "").toLowerCase(); + return KNOWN_TRACE_CLIENTS.has(exe) ? "known_adapter" : "unknown_observation"; +} + +function explicitHttpWriteMethod(command: string[], lowered: string[]): string { + for (let i = 1; i < lowered.length; i += 1) { + const rawArg = command[i]; + const arg = lowered[i]; + if (rawArg === "-X" || arg === "--request" || arg === "--method") { + const method = (lowered[i + 1] || "").toUpperCase(); + if (WRITE_HTTP_METHODS.has(method)) return method; + } + if (rawArg.startsWith("-X") && rawArg.length > 2) { + const method = rawArg.slice(2).toUpperCase(); + if (WRITE_HTTP_METHODS.has(method)) return method; + } + for (const prefix of ["--request=", "--method="]) { + if (arg.startsWith(prefix)) { + const method = arg.slice(prefix.length).toUpperCase(); + if (WRITE_HTTP_METHODS.has(method)) return method; + } + } + } + return ""; +} + +function suspiciousUnknownToken(command: string[]): string { + const lowered = command.map(x => x.toLowerCase()); + return lowered.find((arg, index) => index > 0 && UNKNOWN_MUTATING_TOKENS.has(arg)) || ""; +} + +function validateUnknownTraceCommand(command: string[]): string[] { + const lowered = command.map(x => x.toLowerCase()); + const method = explicitHttpWriteMethod(command, lowered); + return method ? [`explicit HTTP write method is not allowed: ${method}`] : []; +} + +function operationUnclassified(command: string[]): boolean { + const exe = path.basename(command[0] || "").toLowerCase(); + const lowered = command.map(x => x.toLowerCase()); + if (!KNOWN_TRACE_CLIENTS.has(exe)) return true; + if (["rclone", "s5cmd", "mc", "bcecmd", "obsutil", "s3cmd"].includes(exe)) { + return !firstKnownClientOp(lowered); + } + return false; +} + +function traceHostMismatch(command: string[], filterHost: string): boolean { + const hosts = commandHosts(command); + return hosts.length > 0 && Boolean(filterHost) && !hosts.some(host => host === filterHost); +} + +function traceWarningsForCommand(command: string[], filterHost: string): string[] { + const warnings: string[] = []; + if (traceHostMismatch(command, filterHost)) { + warnings.push("command URL host differs from filter_host; trace may capture zero requests"); + } + const suspicious = suspiciousUnknownToken(command); + if (!KNOWN_TRACE_CLIENTS.has(path.basename(command[0] || "").toLowerCase()) && suspicious) { + warnings.push(`unknown client argument looks mutating but will be observed in bounded metadata mode: ${suspicious}`); + } + if (operationUnclassified(command)) { + warnings.push("operation is unclassified; using bounded metadata observation"); + } + return warnings; +} + +export function validateTraceCommand(command: string[], filterHost: string, captureBody: boolean): string[] { + const errors: string[] = []; + if (!Array.isArray(command) || command.length === 0) { + errors.push("command must be a non-empty argv array"); + return errors; + } + if (command.length > MAX_TRACE_COMMAND_ARGS) { + errors.push(`command has too many arguments; max is ${MAX_TRACE_COMMAND_ARGS}`); + } + if (!filterHost) { + errors.push("filter_host is required and must be a host name, not a broad substring"); + } + if (captureBody) { + errors.push("capture_body=true is not supported in the safe default tool"); + } + for (const arg of command) { + if (typeof arg !== "string" || arg.length === 0) { + errors.push("all command arguments must be non-empty strings"); + break; + } + } + if (hasSignedQueryMaterial(command)) { + errors.push("presigned URL material is not accepted by capture_http_trace; provide a redacted trace instead"); + } + + const exe = path.basename(command[0] || "").toLowerCase(); + const lowered = command.map(x => x.toLowerCase()); + const blockedShells = new Set(["sh", "bash", "zsh", "fish", "pwsh", "powershell", "sudo"]); + if (blockedShells.has(exe)) { + errors.push("shells and sudo are not allowed for HTTP trace capture"); + } + + if (exe === "aws") { + const s3api = lowered.indexOf("s3api"); + const s3 = lowered.indexOf("s3"); + if (s3api >= 0) { + const op = lowered[s3api + 1] || ""; + if (MUTATING_AWS_S3API.has(op)) { + errors.push(`mutating or high-risk operation is not allowed: ${op}`); + } + if (!READ_ONLY_AWS_S3API.has(op)) { + errors.push(`aws s3api operation is not in the read-only allowlist: ${op || "(missing)"}`); + } + } else if (s3 >= 0) { + const op = lowered[s3 + 1] || ""; + if (op !== "ls") { + errors.push(`aws s3 operation is not in the read-only allowlist: ${op || "(missing)"}`); + } + } else { + errors.push("aws capture requires an s3 or s3api read-only operation"); + } + } else if (exe === "curl") { + const method = curlMethod(command, lowered); + if (!["GET", "HEAD", "OPTIONS"].includes(method)) { + errors.push(`curl method is not read-only: ${method}`); + } + if (curlHasBodyUploadFlag(command, lowered)) { + errors.push("curl body upload flags are not allowed"); + } + } else if (["rclone", "s5cmd", "mc", "bcecmd", "obsutil", "s3cmd"].includes(exe)) { + const op = firstKnownClientOp(lowered); + if (MUTATING_CLIENT_OPS.has(op)) { + errors.push(`mutating or high-risk operation is not allowed: ${op}`); + } + if (op && !READ_ONLY_CLIENT_OPS.has(op)) { + errors.push(`${exe} operation is not in the read-only allowlist: ${op}`); + } + } else { + errors.push(...validateUnknownTraceCommand(command)); + } + + return [...new Set(errors)]; +} + +const WRITE_REJECTION_MARKERS = [ + "mutating", + "read-only allowlist", + "write method", + "method is not read-only", +]; + +// When a trace is rejected because the command writes (or is not provably +// read-only), point the agent at the evidence ladder instead of leaving a dead +// end. capture_http_trace executes commands, so tracing a write would perform a +// real request; the request shape is recoverable without re-sending the write. +export function traceRejectionGuidance(errors: string[]): string { + const isWriteRejection = (errors || []).some(err => + WRITE_REJECTION_MARKERS.some(marker => err.toLowerCase().includes(marker)), + ); + if (!isWriteRejection) return ""; + return ( + "This command writes or is not provably read-only, and capture_http_trace " + + "executes commands — tracing it would perform a real request. To diagnose a " + + "failing write, read the server error body and the client's own debug dump " + + "(aws --debug / rclone -vv --dump headers / boto3 set_stream_logger), then " + + "recompute offline. See storageops-s3-protocol-compatibility/references/" + + "checksum-etag.md (Write-side request evidence). Re-run capture_http_trace " + + "only on a read-only command." + ); +} + +function getHeader(headers: Record | undefined, name: string): string { + if (!headers) return ""; + const target = name.toLowerCase(); + for (const [key, value] of Object.entries(headers)) { + if (key.toLowerCase() === target) return String(value); + } + return ""; +} + +function safePathTemplate(rawURL: string): string { + try { + const u = new URL(rawURL); + const parts = u.pathname.split("/").map(part => { + if (!part) return part; + if (part.length > 48) return ":segment"; + if (/^(AKIA|ASIA|AKID|LTAI|sk-)/i.test(part)) return "[REDACTED]"; + if (/^[a-f0-9]{32,}$/i.test(part)) return ":hex"; + return part; + }); + const capped = parts.length > 7 ? parts.slice(0, 7).concat(["..."]) : parts; + return capped.join("/") || "/"; + } catch { + return "/"; + } +} + +function parseAuthShape(headers: Record | undefined): { + auth_header_present: boolean; + auth_scheme?: string; + signed_headers?: string[]; + credential_scope?: string; + credential_scope_region?: string; + credential_scope_service?: string; +} { + const auth = getHeader(headers, "Authorization"); + if (!auth) return { auth_header_present: false }; + + const scheme = auth.split(/\s+/)[0] || "unknown"; + const signed = /SignedHeaders=([^,\s]+)/i.exec(auth)?.[1] || ""; + const credential = /Credential=([^,\s]+)/i.exec(auth)?.[1] || ""; + let scope = ""; + let region = ""; + let service = ""; + if (credential) { + try { + const decoded = decodeURIComponent(credential); + const parts = decoded.split("/"); + if (parts.length >= 5) { + scope = parts.slice(1).join("/"); + region = parts[2] || ""; + service = parts[3] || ""; + } + } catch { + scope = ""; + } + } + return { + auth_header_present: true, + auth_scheme: scheme, + signed_headers: signed ? signed.split(/%3B|;/i).filter(Boolean) : [], + credential_scope: scope || undefined, + credential_scope_region: region || undefined, + credential_scope_service: service || undefined, + }; +} + +function queryKeySummary(rawURL: string): { has_presigned_query: boolean; query_keys: string[] } { + try { + const u = new URL(rawURL); + const keys = Array.from(new Set(Array.from(u.searchParams.keys()).map(k => k.toLowerCase()))).sort(); + const hasSigned = keys.some(k => SIGNED_QUERY_KEYS.includes(k)); + return { has_presigned_query: hasSigned, query_keys: keys.slice(0, 20) }; + } catch { + return { has_presigned_query: false, query_keys: [] }; + } +} + +function extractS3ErrorCode(body: string | undefined): string | undefined { + if (!body) return undefined; + const text = body.slice(0, 4096); + return /([^<]{1,80})<\/Code>/i.exec(text)?.[1] + || /"Code"\s*:\s*"([^"]{1,80})"/i.exec(text)?.[1] + || /"code"\s*:\s*"([^"]{1,80})"/i.exec(text)?.[1]; +} + +const SENSITIVE_RESPONSE_HEADERS = new Set([ + "set-cookie", + "www-authenticate", + "proxy-authenticate", + "authorization", +]); +const REDIRECT_HEADERS = new Set(["location", "content-location"]); +const MAX_RESPONSE_HEADERS = 32; +const MAX_HEADER_VALUE_CHARS = 256; + +// Surface response metadata (the diagnostic payload) with targeted sanitization: +// known credential-bearing headers (cookies/auth challenges) keep their name but +// have the value masked; redirect targets are redacted of presigned material +// (which can embed a replayable signature); all other headers pass through so +// metadata like ETag, retention-date, SSE, and checksums is not over-redacted. +export function sanitizeResponseHeaders( + headers: Record | undefined, +): Array<{ name: string; value: string }> { + if (!headers) return []; + const out: Array<{ name: string; value: string }> = []; + for (const [rawName, rawValue] of Object.entries(headers)) { + if (out.length >= MAX_RESPONSE_HEADERS) break; + const name = rawName.toLowerCase(); + let value = String(rawValue ?? ""); + if (SENSITIVE_RESPONSE_HEADERS.has(name)) { + value = "[REDACTED]"; + } else if (REDIRECT_HEADERS.has(name)) { + value = redactText(value).redacted; + } + out.push({ name, value: value.slice(0, MAX_HEADER_VALUE_CHARS) }); + } + return out; +} + +export function summarizeTraceRequest(req: TraceRequest, resp?: TraceResponse) { + const auth = parseAuthShape(req.headers); + const query = queryKeySummary(req.url); + let host = req.host || ""; + try { + host = new URL(req.url).host || host; + } catch { + // keep req.host + } + return { + id: req.id, + method: req.method, + method_violation: !["GET", "HEAD", "OPTIONS"].includes(String(req.method || "").toUpperCase()), + host, + path_template: safePathTemplate(req.url), + status: resp?.status, + s3_error_code: extractS3ErrorCode(resp?.body), + auth_header_present: auth.auth_header_present, + auth_scheme: auth.auth_scheme, + signed_headers: auth.signed_headers, + credential_scope: auth.credential_scope, + credential_scope_region: auth.credential_scope_region, + credential_scope_service: auth.credential_scope_service, + has_presigned_query: query.has_presigned_query, + query_keys: query.query_keys, + response_headers: sanitizeResponseHeaders(resp?.headers), + }; +} + +function findHttpmonBinary(): string { + const configured = process.env.STORAGEOPS_HTTPMON; + if (configured && fs.existsSync(configured)) return configured; + const managed = path.join(os.homedir(), ".storageops", "bin", process.platform === "win32" ? "httpmon.exe" : "httpmon"); + if (fs.existsSync(managed)) return managed; + return "httpmon"; +} + +export async function captureHttpTrace(params: { + command: string[]; + filter_host: string; + max_requests?: number; + max_seconds?: number; + capture_body?: boolean; +}) { + const filterHost = normalizeFilterHost(params.filter_host || ""); + const clientPolicy = clientPolicyForCommand(params.command || []); + const warnings = traceWarningsForCommand(params.command || [], filterHost); + const hostMismatch = traceHostMismatch(params.command || [], filterHost); + const opUnclassified = operationUnclassified(params.command || []); + const strictObservation = clientPolicy === "unknown_observation" || opUnclassified; + const requestCap = strictObservation ? MAX_UNKNOWN_TRACE_REQUESTS : MAX_TRACE_REQUESTS; + const secondsCap = strictObservation ? MAX_UNKNOWN_TRACE_SECONDS : MAX_TRACE_SECONDS; + const maxRequests = Math.min(Math.max(Number(params.max_requests || requestCap), 1), requestCap); + const maxSeconds = Math.min(Math.max(Number(params.max_seconds || secondsCap), 1), secondsCap); + const captureBody = Boolean(params.capture_body); + const validationErrors = validateTraceCommand(params.command, filterHost, captureBody); + if (validationErrors.length > 0) { + return { + status: "rejected", + reason: "unsafe_or_unsupported_command", + client_policy: clientPolicy, + warnings, + host_mismatch: hostMismatch, + operation_unclassified: opUnclassified, + errors: validationErrors, + guidance: traceRejectionGuidance(validationErrors) || undefined, + limits: { max_requests: maxRequests, max_seconds: maxSeconds, capture_body: false }, + }; + } + + return new Promise(resolve => { + const httpmonBinary = findHttpmonBinary(); + const httpmon = childProcess.spawn( + httpmonBinary, + ["--format", "json", "--filter", filterHost, ...params.command], + { stdio: ["ignore", "pipe", "pipe"] }, + ); + + const requests = new Map(); + const responses = new Map(); + let buffer = ""; + let stderr = ""; + let killedForLimit = false; + let spawnError = ""; + let finished = false; + + const finish = (exitCode: number | null) => { + if (finished) return; + finished = true; + clearTimeout(timer); + const summaries = Array.from(requests.values()) + .slice(0, maxRequests) + .map(req => summarizeTraceRequest(req, responses.get(req.id))); + const redactedStderr = redactText(stderr.slice(0, 2000)).redacted; + resolve({ + status: spawnError ? "error" : "completed", + command_name: path.basename(params.command[0]), + client_policy: clientPolicy, + warnings, + host_mismatch: hostMismatch, + operation_unclassified: opUnclassified, + filter_host: filterHost, + exit_code: exitCode, + killed_for_limit: killedForLimit, + requests: summaries, + request_count: summaries.length, + method_violation: summaries.some((r: any) => r.method_violation), + stderr_summary: redactedStderr.slice(0, 500), + redaction: { + authorization_redacted: summaries.some((r: any) => r.auth_header_present), + presigned_query_redacted: summaries.some((r: any) => r.has_presigned_query), + response_headers_sanitized: true, + body_captured: false, + raw_trace_saved: false, + har_saved: false, + replay_performed: false, + }, + limits: { max_requests: maxRequests, max_seconds: maxSeconds, capture_body: false }, + error: spawnError || undefined, + }); + }; + + const handleLine = (line: string) => { + const trimmed = line.trim(); + if (!trimmed.startsWith("{")) return; + try { + const event = JSON.parse(trimmed); + if (typeof event.id === "number" && typeof event.method === "string" && typeof event.url === "string") { + requests.set(event.id, { + id: event.id, + method: event.method, + url: event.url, + host: event.host || "", + headers: event.headers || {}, + }); + if (requests.size >= maxRequests && httpmon.pid) { + killedForLimit = true; + httpmon.kill("SIGTERM"); + } + } else if (typeof event.req_id === "number" && typeof event.status === "number") { + responses.set(event.req_id, { + req_id: event.req_id, + status: event.status, + headers: event.headers || {}, + body: event.body || "", + }); + } + } catch { + // Ignore subprocess output that is not httpmon NDJSON. + } + }; + + const timer = setTimeout(() => { + killedForLimit = true; + if (httpmon.pid) httpmon.kill("SIGTERM"); + }, maxSeconds * 1000); + + httpmon.stdout?.on("data", chunk => { + buffer += chunk.toString("utf8"); + const lines = buffer.split(/\r?\n/); + buffer = lines.pop() || ""; + for (const line of lines) handleLine(line); + }); + httpmon.stderr?.on("data", chunk => { + stderr += chunk.toString("utf8"); + }); + httpmon.on("error", err => { + spawnError = err.message || String(err); + finish(null); + }); + httpmon.on("close", code => { + if (buffer.trim()) handleLine(buffer); + finish(code); + }); + }); +} diff --git a/tests/extension/routing-corpus.test.ts b/tests/extension/routing-corpus.test.ts index ce26b70..00b523e 100644 --- a/tests/extension/routing-corpus.test.ts +++ b/tests/extension/routing-corpus.test.ts @@ -32,6 +32,7 @@ const EXPECT_TOP1 = new Set([ "routing-migration-checksum", "routing-slow-mount-vs-throughput", "routing-spark-committer", + "routing-triage-vague-evidence", "signature-clock-skew", "throttling-hot-prefix", "versioned-delete-marker", diff --git a/tests/extension/routing-negative-corpus.test.ts b/tests/extension/routing-negative-corpus.test.ts index 49ec8ed..e70a2c2 100644 --- a/tests/extension/routing-negative-corpus.test.ts +++ b/tests/extension/routing-negative-corpus.test.ts @@ -75,6 +75,11 @@ const NEGATIVE_CASES: NegativeCase[] = [ text: "The KMS team will present the quarterly knowledge-management-system roadmap.", forbidden: [["storageops-security-iam-policy", "kms_error"]], }, + { + name: "cors workshop agenda is not a bucket cors failure", + text: "The quarterly CORS policy workshop for frontend engineers is on Tuesday.", + forbidden: [["storageops-s3-protocol-compatibility", "cors"]], + }, ]; test("routing negative corpus blocks known bare-substring false positives", () => { diff --git a/tests/test_coverage_check.py b/tests/test_coverage_check.py index 5089c5a..92be098 100644 --- a/tests/test_coverage_check.py +++ b/tests/test_coverage_check.py @@ -20,4 +20,4 @@ def test_repo_passes_coverage(): def test_min_cases_floor_is_set(): m = load() - assert m.MIN_CASES >= 2 + assert m.MIN_CASES >= 3 diff --git a/tests/test_extension_domain_signatures.py b/tests/test_extension_domain_signatures.py index 1b6b18e..21a6f34 100644 --- a/tests/test_extension_domain_signatures.py +++ b/tests/test_extension_domain_signatures.py @@ -1,9 +1,14 @@ from pathlib import Path -def test_domain_signatures_use_skill_names_and_cover_routing_domains(): +def _extension_sources() -> str: root = Path(__file__).resolve().parents[1] - extension = (root / "storageops_cli" / "extensions" / "storageops.ts").read_text() + ext_dir = root / "storageops_cli" / "extensions" + return "\n".join(p.read_text() for p in sorted(ext_dir.glob("*.ts"))) + + +def test_domain_signatures_use_skill_names_and_cover_routing_domains(): + extension = _extension_sources() for skill in [ "storageops-s3-protocol-compatibility", @@ -19,8 +24,7 @@ def test_domain_signatures_use_skill_names_and_cover_routing_domains(): def test_domain_confidence_has_single_signal_floor(): - root = Path(__file__).resolve().parents[1] - extension = (root / "storageops_cli" / "extensions" / "storageops.ts").read_text() + extension = _extension_sources() assert "0.5 + info.score * 0.15" in extension assert "recommended_skill: domain" in extension @@ -29,8 +33,7 @@ def test_domain_confidence_has_single_signal_floor(): def test_secret_scanner_deduplicates_overlapping_matches_by_range(): - root = Path(__file__).resolve().parents[1] - extension = (root / "storageops_cli" / "extensions" / "storageops.ts").read_text() + extension = _extension_sources() assert "const ranges: Array<[number, number]>" in extension assert "start < rangeEnd && end > rangeStart" in extension @@ -39,8 +42,7 @@ def test_secret_scanner_deduplicates_overlapping_matches_by_range(): def test_secret_scanner_findings_do_not_return_raw_secret_previews(): - root = Path(__file__).resolve().parents[1] - extension = (root / "storageops_cli" / "extensions" / "storageops.ts").read_text() + extension = _extension_sources() assert "fingerprint" in extension assert "secretFingerprint(value)" in extension @@ -50,8 +52,7 @@ def test_secret_scanner_findings_do_not_return_raw_secret_previews(): def test_memory_search_scores_and_redacts_snippets(): - root = Path(__file__).resolve().parents[1] - extension = (root / "storageops_cli" / "extensions" / "storageops.ts").read_text() + extension = _extension_sources() assert "searchTokens" in extension assert "scoreText" in extension @@ -62,8 +63,7 @@ def test_memory_search_scores_and_redacts_snippets(): def test_capture_http_trace_is_registered_as_bounded_safe_tool(): - root = Path(__file__).resolve().parents[1] - extension = (root / "storageops_cli" / "extensions" / "storageops.ts").read_text() + extension = _extension_sources() assert 'name: "capture_http_trace"' in extension assert "validateTraceCommand" in extension @@ -81,8 +81,7 @@ def test_capture_http_trace_is_registered_as_bounded_safe_tool(): def test_capture_http_trace_does_not_expose_raw_httpmon_artifacts(): - root = Path(__file__).resolve().parents[1] - extension = (root / "storageops_cli" / "extensions" / "storageops.ts").read_text() + extension = _extension_sources() assert '"--format", "json", "--filter", filterHost' in extension assert '"--record"' not in extension diff --git a/tests/test_extension_tools.py b/tests/test_extension_tools.py index 4c87d0c..578c1dd 100644 --- a/tests/test_extension_tools.py +++ b/tests/test_extension_tools.py @@ -1,15 +1,16 @@ from pathlib import Path -def _extension() -> str: +def _extension_sources() -> str: root = Path(__file__).resolve().parents[1] - return (root / "storageops_cli" / "extensions" / "storageops.ts").read_text() + ext_dir = root / "storageops_cli" / "extensions" + return "\n".join(p.read_text() for p in sorted(ext_dir.glob("*.ts"))) def test_search_memory_scans_scope_subdirectories(): # Recall must walk scope subdirs (e.g. sessions//.jsonl), not just # the top level — locks the v0.4.29 recursive-recall fix. - ext = _extension() + ext = _extension_sources() assert "collectSessionJsonl" in ext assert "MAX_SESSION_SCAN_DEPTH" in ext assert ".endsWith(\".jsonl\")" in ext @@ -17,14 +18,14 @@ def test_search_memory_scans_scope_subdirectories(): def test_search_tokens_handle_cjk_queries(): # CJK queries must produce tokens (bigrams) so Chinese memory search recalls. - ext = _extension() + ext = _extension_sources() assert "一-鿿" in ext assert "run.slice(i, i + 2)" in ext def test_http_trace_allowlist_covers_skill_read_only_ops(): # The CORS / lifecycle / multipart / tagging skills rely on these read-only ops. - ext = _extension() + ext = _extension_sources() for op in [ "head-bucket", "get-bucket-cors", @@ -40,7 +41,7 @@ def test_http_trace_allowlist_covers_skill_read_only_ops(): def test_scan_secrets_covers_presigned_and_multicloud_keys(): # Presigned URL material is extremely common in rclone/aws/s5cmd debug logs; # GCP/Azure are documented domains. The scanner must redact all of these. - ext = _extension() + ext = _extension_sources() for label in [ "PRESIGNED_SIGNATURE", "PRESIGNED_AWS_PARAM", @@ -56,13 +57,13 @@ def test_scan_secrets_covers_presigned_and_multicloud_keys(): def test_search_memory_dedupes_per_session(): - ext = _extension() + ext = _extension_sources() assert "bestBySession" in ext def test_detect_domain_has_cjk_parity_for_core_domains(): # Chinese inputs to these domains previously matched nothing. - ext = _extension() + ext = _extension_sources() for term in ["访问被拒", "限速", "连接", "损坏", "签名"]: assert term in ext