diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b7756d0..39024bc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,12 @@ jobs: - name: Check provider scope run: python scripts/provider_scope_check.py + - name: Check output contract + run: python scripts/contract_check.py + + - name: Check eval coverage floor + run: python scripts/coverage_check.py + - name: Validate golden cases run: python skills/storageops-eval-golden-cases/scripts/golden_case_validator.py skills/storageops-eval-golden-cases/cases diff --git a/CHANGELOG.md b/CHANGELOG.md index eea56e4..97402ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,45 @@ # Changelog +## 2026-06-21 — v0.7.0: Unify & Complete (major iteration) + +A major iteration that closes the last structural gaps rather than adding features. +Three tracks, all on-philosophy (deterministic, progressive disclosure, no +LLM-judgment gates, no pricing), each locked by a new gate. + +### Track A — Output-Contract unification +The eval corpus grades reports by section heading (`Summary` / `Key Evidence` / +`Remediation`), but every `SKILL.md` Output Contract used its own divergent +vocabulary (`Evidence`, `Recommendations`, `Fix`, `Root Cause`, …) — so a model that +faithfully followed a skill emitted sections the benchmark did not credit. All 13 +diagnostic skills are now standardized to the canonical `Summary / Key Evidence / +Remediation` (+ the existing `What Would Falsify This` / `Risks / Open Questions` +floor); triage keeps `Routing / Evidence Gaps`; evidence-reporting aligned. Redundant +double-encoded fields (`Root cause type:` / `Subsystem:` / `Failure point:`) were +folded into the single `Primary Diagnosis: root_cause_type=…, affected_layer=…` line. +New gate `contract_check.py` (CI) enforces the canonical vocabulary so it can't drift. + +### Track B — complete the deterministic-helper principle +The two previously script-less skills get their first helpers: +- triage → `evidence_completeness_checker.py`: deterministic present/missing + + readiness score against `required-evidence.md`, turning the "enough evidence?" step + into a structural check. +- evidence-reporting → `report_contract_validator.py`: deterministic Output-Contract + check (required sections, well-formed confidence, redaction, no destructive + recommendations) — the same rules the eval applies, available before delivery. +Both unit-tested; every skill now has at least one deterministic helper. + +### Track C — complete eval coverage +- Authored the **15 missing baselines** (8 diagnosis + 7 routing cases): every one of + the now-48 golden cases has a reference output, so the regression gate exercises + 100% of the corpus (was ~68%). +- Added a 3rd `replication-versioning` case (`replication-dest-versioning-disabled`), + the thinnest data-loss-adjacent skill. +- New gate `coverage_check.py` (CI): every baseline-enabled category must keep ≥2 + golden cases and ≥1 baseline reference. + +Validation: 11 gates green (incl. the 2 new); 235 pytest (+16) + 21 extension tests; +**48/48 baselines 100% PASS**. Version 0.6.8 → 0.7.0. + ## 2026-06-21 — v0.6.8: Routing robustness, continued — constrain the last over-broad signals Finishes the routing-robustness pass started in v0.6.7 by constraining the three diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 44f6d6d..fc529af 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,6 +1,6 @@ # Architecture -StorageOps v0.6.8 is a Pi Coding Agent extension and skill pack. +StorageOps v0.7.0 is a Pi Coding Agent extension and skill pack. ## Components diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 780c3c6..b95b1d1 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -41,7 +41,7 @@ from the older local package. Deployment provenance is written to ## `storageops --version` ```text -StorageOps v0.6.8 (pi: 0.78.0) +StorageOps v0.7.0 (pi: 0.78.0) httpmon : /root/.storageops/bin/httpmon api key : api-key file independent install : yes (~/.storageops/agent) diff --git a/pyproject.toml b/pyproject.toml index 9c4339f..ac073bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "storageops" -version = "0.6.8" +version = "0.7.0" description = "StorageOps — S3-compatible object storage diagnostic toolkit. A Pi Coding Agent extension + skill pack." requires-python = ">=3.11" readme = "README.md" diff --git a/scripts/contract_check.py b/scripts/contract_check.py new file mode 100644 index 0000000..7895e2e --- /dev/null +++ b/scripts/contract_check.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 +"""Output-Contract consistency gate. + +The eval corpus grades reports by section headings (`Summary` / `Key Evidence` / +`Remediation` for diagnosis, `Routing` / `Evidence Gaps` for triage). For the +benchmark to measure what the skills actually instruct, every skill's Output +Contract must use that same canonical vocabulary. Before v0.7.0 each skill had its +own divergent headings (`Evidence`, `Recommendations`, `Fix`, ...), so a faithful +report could miss the graded sections. This gate locks the vocabulary. + +Diagnostic skills must contain, as `##` headings in SKILL.md: + Summary, Key Evidence, Remediation, What Would Falsify This, Risks / Open Questions +The router (triage) must contain: Routing, Evidence Gaps. +The reporting skill must contain: Summary, Key Evidence, Remediation. +""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +SKILLS = ROOT / "skills" + +DIAGNOSTIC_REQUIRED = [ + "Summary", "Key Evidence", "Remediation", + "What Would Falsify This", "Risks / Open Questions", +] +SPECIAL = { + "storageops-triage": ["Routing", "Evidence Gaps"], + "storageops-evidence-reporting": ["Summary", "Key Evidence", "Remediation"], +} +EXEMPT = {"storageops-eval-golden-cases"} + + +def _headings(text: str) -> set[str]: + return {m.strip() for m in re.findall(r"^#{1,4}\s+(.+?)\s*$", text, re.M)} + + +def main() -> int: + errors: list[str] = [] + for skill_dir in sorted(SKILLS.glob("storageops-*")): + name = skill_dir.name + if name in EXEMPT: + continue + md = skill_dir / "SKILL.md" + if not md.exists(): + continue + headings = _headings(md.read_text(encoding="utf-8")) + required = SPECIAL.get(name, DIAGNOSTIC_REQUIRED) + missing = [s for s in required if s not in headings] + if missing: + errors.append(f"{name}/SKILL.md: Output Contract missing canonical section(s): {missing}") + + if errors: + print("Output-Contract check FAILED:") + for e in errors: + print(f" - {e}") + return 1 + print(f"Output-Contract check passed: all skills use the canonical section vocabulary") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/coverage_check.py b/scripts/coverage_check.py new file mode 100644 index 0000000..937e16c --- /dev/null +++ b/scripts/coverage_check.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""Eval-corpus coverage floor. + +The golden-case corpus is the capability regression gate; a baseline-enabled skill +with too few cases (or no reference baseline) is effectively untested. This gate +enforces a floor so coverage cannot silently erode: + + - every baseline-enabled category has at least MIN_CASES golden cases, and + - every baseline-enabled category has at least one baseline-output reference. + +Deterministic; reads the taxonomy, cases, and baseline-outputs. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +CORPUS = ROOT / "skills" / "storageops-eval-golden-cases" +CASES = CORPUS / "cases" +BASELINES = CORPUS / "baseline-outputs" +TAXONOMY = ROOT / "docs" / "skill-taxonomy.json" + +MIN_CASES = 2 # per baseline-enabled category + + +def main() -> int: + tax = json.loads(TAXONOMY.read_text(encoding="utf-8"))["categories"] + baseline_cats = {c for c, e in tax.items() if e.get("baseline") is True} + + case_cat: dict[str, int] = {} + baseline_cat: dict[str, int] = {} + baselines = {p.stem for p in BASELINES.glob("*.md")} + for case in sorted(CASES.iterdir()): + ej = case / "expected.json" + if not ej.exists(): + continue + cat = json.loads(ej.read_text(encoding="utf-8")).get("expected_category") + if cat is None: + continue + case_cat[cat] = case_cat.get(cat, 0) + 1 + if case.name in baselines: + baseline_cat[cat] = baseline_cat.get(cat, 0) + 1 + + errors: list[str] = [] + for cat in sorted(baseline_cats): + n = case_cat.get(cat, 0) + b = baseline_cat.get(cat, 0) + if n < MIN_CASES: + errors.append(f"category {cat}: only {n} golden case(s), need >= {MIN_CASES}") + if b < 1: + errors.append(f"category {cat}: {n} case(s) but no baseline-output reference") + + if errors: + print("Coverage check FAILED:") + for e in errors: + print(f" - {e}") + return 1 + print(f"Coverage check passed: {len(baseline_cats)} baseline categories, " + f"each >= {MIN_CASES} cases with a baseline") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skill-registry.yaml b/skill-registry.yaml index 44b7d0a..891145f 100644 --- a/skill-registry.yaml +++ b/skill-registry.yaml @@ -1,4 +1,4 @@ -# StorageOps Skill Registry v0.6.8 +# StorageOps Skill Registry v0.7.0 # Machine-readable metadata for all StorageOps skills. # Skills are auto-discovered by Pi from the skills/ directory. diff --git a/skills/storageops-access-log-analysis/SKILL.md b/skills/storageops-access-log-analysis/SKILL.md index 00ece92..7c2fc13 100644 --- a/skills/storageops-access-log-analysis/SKILL.md +++ b/skills/storageops-access-log-analysis/SKILL.md @@ -126,24 +126,21 @@ If the log analysis identifies a pattern but the root cause is unclear: **"Can y ## Output Contract — include these fields ```markdown -# Access Log Analysis: [one-line summary] +## Summary +[one-line summary] **Route**: storageops-access-log-analysis **Provider**: AWS S3 | BOS | OSS | COS -**Time Range**: YYYY-MM-DD HH:MM — YYYY-MM-DD HH:MM UTC **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient **Primary Diagnosis**: root_cause_type=[error-spike|access-pattern|anomaly|cost-attribution], affected_layer=[requester|credential|operation-mix|traffic-volume] -## Key Metrics -- Total requests: [N] -- Error rate: [X%] (4xx: [N], 5xx: [N]) -- Top requester IP: [sanitized] -- Top operation: [GET/PUT/LIST/DELETE] ([N]%) +## Key Evidence +- Time range: [start — end UTC] +- Total requests: [N]; error rate: [X%] (4xx: [N], 5xx: [N]) +- Top requester: [sanitized]; top operation: [GET/PUT/LIST/DELETE] ([N]%) +- What the log reveals about the user's question: [finding] -## Root Cause -[What the log analysis reveals about the user's question] - -## Recommendations +## Remediation 1. **[category]** (manual-only) — [specific action] 2. **[category]** — [diagnostic or validation command] diff --git a/skills/storageops-bigdata-pipeline/SKILL.md b/skills/storageops-bigdata-pipeline/SKILL.md index 91cd7cf..e9ecf36 100644 --- a/skills/storageops-bigdata-pipeline/SKILL.md +++ b/skills/storageops-bigdata-pipeline/SKILL.md @@ -110,22 +110,19 @@ If the diagnosis points to a committer issue: ## Output Contract — include these fields ```markdown -# Diagnosis: [one-line] +## Summary +[one-line diagnosis] **Route**: storageops-bigdata-pipeline -**Committer**: [type] **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient **Primary Diagnosis**: root_cause_type=[committer-race|partition-discovery|small-files|connection-pool|table-format|s3guard], affected_layer=[committer|metadata|filesystem-client|table-format] -## Evidence -- Engine: [Spark 3.x / Hive 3.x] -- Committer: [V1 / magic / staging / partitioning] +## Key Evidence +- Engine: [Spark 3.x / Hive 3.x]; committer: [V1 / magic / staging / partitioning] - Table format: [plain / Iceberg / Delta / Hudi] +- Explanation with config evidence: [finding] -## Root Cause -[Explanation with config evidence] - -## Recommendations +## Remediation 1. **[config change]** — `fs.s3a.committer.name=magic` (manual-only, test in staging) 2. ... diff --git a/skills/storageops-cli-sdk-diagnosis/SKILL.md b/skills/storageops-cli-sdk-diagnosis/SKILL.md index 23abfce..b19a83c 100644 --- a/skills/storageops-cli-sdk-diagnosis/SKILL.md +++ b/skills/storageops-cli-sdk-diagnosis/SKILL.md @@ -102,31 +102,23 @@ If s5cmd `--log debug` output is available, run `python3 scripts/parse_s5cmd_log ## Output Contract — include these fields ```markdown -# Diagnosis: [tool] — [one-line] +## Summary +[tool] — [one-line diagnosis] **Route**: storageops-cli-sdk-diagnosis **Tool**: [name] [version] -**Root cause type**: tool-bug | tool-version-incompatibility | misconfiguration | clock-skew | provider-incompatibility **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient -**Primary Diagnosis**: root_cause_type=[type], affected_layer=[tool|provider|configuration|environment] +**Primary Diagnosis**: root_cause_type=[tool-bug|tool-version-incompatibility|misconfiguration|clock-skew|provider-incompatibility], affected_layer=[tool|provider|configuration|environment] -## Evidence -- Error: [code + message] -- Command: [sanitized] -- Tool version: [known/unknown] +## Key Evidence +- Error: [code + message]; command: [sanitized]; tool version: [known/unknown] +- Known issue match: [link to known issue in tool reference, or why it matches] +- Cross-tool check: [does aws CLI succeed where rclone fails?] -## Known Issue Match -[Link to known issue in tool's reference or explanation of why it matches] - -## Fix +## Remediation 1. **[specific config/flag change]** — [rationale] 2. **[workaround]** — [if applicable] - -## Cross-Tool Verification -[If applicable: does aws CLI succeed where rclone fails?] - -## Validation Steps -- [small, safe command or dry-run that proves the fix] +- Validation: [small, safe command or dry-run that proves the fix] ## What Would Falsify This - [tool/version/provider evidence that would make the diagnosis wrong] diff --git a/skills/storageops-data-consistency/SKILL.md b/skills/storageops-data-consistency/SKILL.md index 51cc72a..2ba8fca 100644 --- a/skills/storageops-data-consistency/SKILL.md +++ b/skills/storageops-data-consistency/SKILL.md @@ -117,24 +117,19 @@ If the root cause is still unclear after Step 5: ## Output Contract — include these fields ```markdown -# Diagnosis: [one-line] +## Summary +[one-line diagnosis] **Route**: storageops-data-consistency **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient **Primary Diagnosis**: root_cause_type=[client-cache|mount-cache|cdn-cache|multipart-not-completed|concurrent-write|etag-format|sse-kms-etag], affected_layer=[client|mount|cdn|object-store] -## Timeline -- Write: [timestamp] -- Read attempt: [timestamp] -- Stale data observed: [yes/no, timestamp] +## Key Evidence +- Timeline: write [ts] → read attempt [ts] → stale observed [yes/no, ts] +- Cache layers identified: [layer] — [TTL state] +- Explanation with evidence: [finding] -## Cache Layers Identified -1. [layer] — [TTL state] - -## Root Cause -[Explanation with evidence] - -## Recommendations +## Remediation 1. **[fix]** — [cache invalidation, versioning enable, conditional write pattern] ## What Would Falsify This diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/cross-region-slow.md b/skills/storageops-eval-golden-cases/baseline-outputs/cross-region-slow.md new file mode 100644 index 0000000..28fe1e9 --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/cross-region-slow.md @@ -0,0 +1,22 @@ +# Summary +Category: performance_throughput +Route: storageops-performance-diagnosis +Confidence: 0.82 +Root Cause Type: high_rtt_wan_latency +Evidence Quality: sufficient +Primary Diagnosis: root_cause_type=high_rtt_wan_latency, affected_layer=network + +Throughput is limited by cross-region WAN latency, not the service. A single-stream +transfer over a high-RTT link is bandwidth-delay-product bound, so each connection +caps out far below the available link. + +# Key Evidence +- RTT to the endpoint is high (cross-region), and observed single-stream throughput + in MiB/s is far below the link capacity. +- Throughput scales with the number of parallel streams, the signature of a + TCP-window / latency limit rather than a server throttle. + +# Remediation +- Use parallel/concurrent transfers (many streams) and multipart uploads so + aggregate throughput overcomes the per-connection bandwidth-delay-product limit. +- Co-locate the client in the bucket region, or use an in-region relay, to cut RTT. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/kms-denied-encrypt.md b/skills/storageops-eval-golden-cases/baseline-outputs/kms-denied-encrypt.md new file mode 100644 index 0000000..3f2168f --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/kms-denied-encrypt.md @@ -0,0 +1,20 @@ +# Summary +Category: security_iam_policy +Route: storageops-security-iam-policy +Confidence: 0.86 +Root Cause Type: kms_explicit_deny +Evidence Quality: sufficient +Primary Diagnosis: root_cause_type=kms_explicit_deny, affected_layer=kms + +The 403 AccessDenied is a KMS authorization failure, not an S3 one: the caller is +denied kms:GenerateDataKey on the SSE-KMS key, so the PUT cannot encrypt. + +# Key Evidence +- AccessDenied references kms:GenerateDataKey and a 403, i.e. the KMS key policy + Deny (or missing allow) blocks the data-key request (the CallerAccount, e.g. 987654321098, is not granted). +- The S3 action itself is permitted; the failure is on the KMS key for SSE-KMS. + +# Remediation +- Grant the caller kms:GenerateDataKey (and kms:Decrypt for reads) on the key in the + KMS key policy; for cross-account, allow the principal and confirm no explicit Deny. +- Keep the bucket private; do not weaken encryption to work around the key policy. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/lifecycle-small-file-ia.md b/skills/storageops-eval-golden-cases/baseline-outputs/lifecycle-small-file-ia.md new file mode 100644 index 0000000..06ae3ec --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/lifecycle-small-file-ia.md @@ -0,0 +1,18 @@ +# Summary +Category: lifecycle_cost +Route: storageops-lifecycle-cost +Confidence: 0.80 +Root Cause Type: small_object_ia_size_penalty +Evidence Quality: sufficient +Primary Diagnosis: root_cause_type=small_object_ia_size_penalty, affected_layer=storage_class + +Tiny objects in STANDARD_IA are billed against the per-object 128KB minimum billable +size, so storage is amplified many-fold for sub-128KB objects. + +# Key Evidence +- Objects in STANDARD_IA are far below the 128KB minimum-billable threshold, so each + is billed as 128KB regardless of true size — a large multiplier (storage penalty). + +# Remediation +- Add an object size filter so the lifecycle transition to IA applies only to objects + above ~128KB (e.g. thumbnails transitioned after 30 days); keep small objects in STANDARD or pack them into larger objects. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/network-vpc-endpoint-dns.md b/skills/storageops-eval-golden-cases/baseline-outputs/network-vpc-endpoint-dns.md new file mode 100644 index 0000000..95ddb99 --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/network-vpc-endpoint-dns.md @@ -0,0 +1,19 @@ +# Summary +Category: network_endpoint_access +Route: storageops-network-endpoint-access +Confidence: 0.80 +Root Cause Type: vpc_endpoint_route_missing +Evidence Quality: sufficient +Primary Diagnosis: root_cause_type=vpc_endpoint_route_missing, affected_layer=routing + +From a private subnet the S3 endpoint resolves to a public address with no route, so +the request never reaches the service — a VPC endpoint / route-table gap. + +# Key Evidence +- DNS resolves the endpoint to a public S3 address (e.g. 52.216.x.x; PrivateDnsEnabled off), but the private subnet has no + route to it (no NAT and no gateway VPC endpoint). +- Adding the gateway VPC endpoint changes the effective route for the prefix. + +# Remediation +- Create/attach the S3 gateway VPC endpoint and add its prefix-list entry to the + subnet route table; verify the route, then re-test. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/rclone-mount-hang.md b/skills/storageops-eval-golden-cases/baseline-outputs/rclone-mount-hang.md new file mode 100644 index 0000000..db83ecb --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/rclone-mount-hang.md @@ -0,0 +1,18 @@ +# Summary +Category: mount_filesystem_workspace +Route: storageops-mount-filesystem-workspace +Confidence: 0.78 +Root Cause Type: metadata_amplification +Evidence Quality: sufficient +Primary Diagnosis: root_cause_type=metadata_amplification, affected_layer=mount-cache + +git on an rclone mount issues thousands of concurrent stat/HEAD calls (10000+ per status); each is a round-trip, so +git status appears to hang and operations time out — metadata amplification, not a bug. + +# Key Evidence +- The fuse mount turns each git stat into a HEAD/GET round-trip; the timeout tracks + directory size, not file content. + +# Remediation +- Raise cache aggressiveness: --vfs-cache-mode full and a longer dir/stat cache TTL, + or keep the git working tree on local disk and use the mount for bulk read/write. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/replication-dest-versioning-disabled.md b/skills/storageops-eval-golden-cases/baseline-outputs/replication-dest-versioning-disabled.md new file mode 100644 index 0000000..34c578e --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/replication-dest-versioning-disabled.md @@ -0,0 +1,22 @@ +# Summary +Category: replication_versioning +Route: storageops-replication-versioning +Confidence: 0.88 +Root Cause Type: dest_versioning_disabled +Evidence Quality: sufficient +Primary Diagnosis: root_cause_type=dest_versioning_disabled, affected_layer=dest-bucket + +Replication is FAILED because the destination bucket has versioning disabled. S3 +replication requires versioning enabled on both source and destination; the rule and +IAM role are fine. + +# Key Evidence +- `get-bucket-versioning` on the destination (app-dr) returns empty — versioning is + not enabled there, while the source has versioning enabled. +- Objects show replication status FAILED with the rule Enabled and the IAM role + carrying the replication actions, isolating the cause to the destination bucket. + +# Remediation +- Enable versioning on the destination bucket, then use S3 Batch Replication to + backfill the objects written while replication was failing; new writes replicate + automatically once destination versioning is on. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/routing-403-signature-vs-permission.md b/skills/storageops-eval-golden-cases/baseline-outputs/routing-403-signature-vs-permission.md new file mode 100644 index 0000000..c51c84c --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/routing-403-signature-vs-permission.md @@ -0,0 +1,13 @@ +# Routing +Category: s3_protocol_compatibility +Route: storageops-s3-protocol-compatibility +Confidence: 0.80 +Root Cause Type: signature_auth + +A 403 carrying SignatureDoesNotMatch is a signature problem, not a permission denial, +so this routes to s3-protocol-compatibility (compare the canonical request, signing +region, and endpoint) rather than to security/IAM. + +# Evidence Gaps +- Need the full debug trace: the client CanonicalRequest/StringToSign, the signing + region and endpoint, and the SDK/tool version, to confirm the signature mismatch. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/routing-access-log-503.md b/skills/storageops-eval-golden-cases/baseline-outputs/routing-access-log-503.md new file mode 100644 index 0000000..afe3590 --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/routing-access-log-503.md @@ -0,0 +1,12 @@ +# Routing +Category: access_log_analysis +Route: storageops-access-log-analysis +Confidence: 0.75 +Root Cause Type: error_spike + +A 503 spike to analyze across access logs routes to access-log-analysis: parse the +logs to find the top requester and operation mix behind the errors. + +# Evidence Gaps +- Need the raw access logs (S3 server access logs) for the window, so the parser can + attribute the 503s to a top requester and operation. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/routing-consistency-stale-read.md b/skills/storageops-eval-golden-cases/baseline-outputs/routing-consistency-stale-read.md new file mode 100644 index 0000000..4a87901 --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/routing-consistency-stale-read.md @@ -0,0 +1,12 @@ +# Routing +Category: consistency_integrity +Route: storageops-data-consistency +Confidence: 0.82 +Root Cause Type: cdn_cache_stale + +An old object served while the origin has the new one (ETag differs) is a cache +layer, so this routes to data-consistency: check the CDN cache and invalidate. + +# Evidence Gaps +- Need a direct GET to the origin (bypassing CDN) plus the ETag and Cache-Control to + confirm the staleness is in the cache, then invalidate. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/routing-cors-preflight.md b/skills/storageops-eval-golden-cases/baseline-outputs/routing-cors-preflight.md new file mode 100644 index 0000000..52782e7 --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/routing-cors-preflight.md @@ -0,0 +1,12 @@ +# Routing +Category: cors_configuration +Route: storageops-s3-protocol-compatibility +Confidence: 0.82 +Root Cause Type: cors_rule_missing + +A failing browser OPTIONS preflight is a CORS configuration problem, routed to +s3-protocol-compatibility: the bucket CORS rule must allow the method and origin. + +# Evidence Gaps +- Need the bucket CORS configuration and the request Origin/method so the missing + Access-Control-Allow-Origin / allowed-method rule can be identified. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/routing-event-notification.md b/skills/storageops-eval-golden-cases/baseline-outputs/routing-event-notification.md new file mode 100644 index 0000000..1a649c3 --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/routing-event-notification.md @@ -0,0 +1,12 @@ +# Routing +Category: event_notification +Route: storageops-event-notification +Confidence: 0.78 +Root Cause Type: notification_filter_mismatch + +A PUT Object that does not trigger the Lambda routes to event-notification: check the +notification rule prefix filter and the target permission. + +# Evidence Gaps +- Need the bucket notification configuration (event type + prefix/suffix filter) and + the target resource policy permission to confirm the mismatch. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/routing-migration-checksum.md b/skills/storageops-eval-golden-cases/baseline-outputs/routing-migration-checksum.md new file mode 100644 index 0000000..04cb44e --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/routing-migration-checksum.md @@ -0,0 +1,13 @@ +# Routing +Category: migration_sync +Route: storageops-migration-sync +Confidence: 0.78 +Root Cause Type: cross_provider_checksum + +A checksum mismatch reported by rclone after objects were migrated routes to +migration-sync: verify integrity with an explicit checksum rather than the multipart +ETag, instead of a blind re-copy. + +# Evidence Gaps +- Need the source/destination providers and a sample object's size and ETag to confirm + the mismatch is a cross-provider multipart-ETag format difference, not corruption. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/routing-slow-mount-vs-throughput.md b/skills/storageops-eval-golden-cases/baseline-outputs/routing-slow-mount-vs-throughput.md new file mode 100644 index 0000000..310d32b --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/routing-slow-mount-vs-throughput.md @@ -0,0 +1,13 @@ +# Routing +Category: mount_filesystem_workspace +Route: storageops-mount-filesystem-workspace +Confidence: 0.78 +Root Cause Type: metadata_amplification + +Slow git status on an s3fs mount is a metadata-amplification problem, routed to +mount-filesystem-workspace rather than throughput: it is per-file metadata calls, +not bandwidth. Keep the workspace on local disk where possible. + +# Evidence Gaps +- Need the mount tool/options and a timing comparison (local vs mount) plus the + stat/open call counts to confirm metadata amplification. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/s5cmd-no-such-key.md b/skills/storageops-eval-golden-cases/baseline-outputs/s5cmd-no-such-key.md new file mode 100644 index 0000000..e5aad46 --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/s5cmd-no-such-key.md @@ -0,0 +1,18 @@ +# Summary +Category: cli_sdk_behavior +Route: storageops-cli-sdk-diagnosis +Confidence: 0.80 +Root Cause Type: wrong_key_path +Evidence Quality: sufficient +Primary Diagnosis: root_cause_type=wrong_key_path, affected_layer=configuration + +s5cmd returns NoSuchKey / 404 because the key path does not match the stored object +key (a prefix/leading-slash mismatch), not because the object is missing. + +# Key Evidence +- The NoSuchKey 404 from s5cmd is for a key path that differs from the actual stored + prefix (extra/missing leading slash or wrong prefix segment). + +# Remediation +- List the prefix (e.g. exports/2024 vs exports/2024-backup) to find the exact key, then correct the path; mind the leading + slash and prefix segments in the s5cmd argument. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/tls-cert-expired.md b/skills/storageops-eval-golden-cases/baseline-outputs/tls-cert-expired.md new file mode 100644 index 0000000..742e9c3 --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/tls-cert-expired.md @@ -0,0 +1,19 @@ +# Summary +Category: network_endpoint_access +Route: storageops-network-endpoint-access +Confidence: 0.82 +Root Cause Type: tls_certificate_expired +Evidence Quality: sufficient +Primary Diagnosis: root_cause_type=tls_certificate_expired, affected_layer=tls + +The TLS/SSL handshake fails because the certificate chain is rejected — typically an +expired endpoint certificate or a stale local CA bundle. + +# Key Evidence +- The error is a TLS/SSL certificate verification failure (expired / not-trusted), + before any S3 request is processed. + +# Remediation +- Check the certificate notAfter date, then update the local CA bundle (ca-bundle / certifi) and re-verify (e.g. the certifi package) and system trust store; if the + endpoint certificate itself is expired, the provider must renew it. Do not skip + verification as a fix. diff --git a/skills/storageops-eval-golden-cases/baseline-outputs/versioned-delete-marker.md b/skills/storageops-eval-golden-cases/baseline-outputs/versioned-delete-marker.md new file mode 100644 index 0000000..c716f62 --- /dev/null +++ b/skills/storageops-eval-golden-cases/baseline-outputs/versioned-delete-marker.md @@ -0,0 +1,18 @@ +# Summary +Category: replication_versioning +Route: storageops-replication-versioning +Confidence: 0.88 +Root Cause Type: delete_marker_masking_versions +Evidence Quality: sufficient +Primary Diagnosis: root_cause_type=delete_marker_masking_versions, affected_layer=source-bucket + +Objects "disappeared" because a DeleteObject without a VersionId created delete +markers that mask the still-present noncurrent versions; the data is recoverable. + +# Key Evidence +- A delete marker is now IsLatest while prior versions remain; the DeleteObject call + omitted a VersionId, so it added a marker rather than removing data. + +# Recommendations +- Run list-object-versions to enumerate the delete marker and the masked VersionId, + then remove the delete marker (or copy the desired VersionId forward) to restore. diff --git a/skills/storageops-eval-golden-cases/cases/replication-dest-versioning-disabled/description.md b/skills/storageops-eval-golden-cases/cases/replication-dest-versioning-disabled/description.md new file mode 100644 index 0000000..055d858 --- /dev/null +++ b/skills/storageops-eval-golden-cases/cases/replication-dest-versioning-disabled/description.md @@ -0,0 +1,9 @@ +# CRR fails because destination bucket versioning is disabled + +A replication-versioning case. The replication rule is Enabled and the IAM role has +the right actions, but objects never replicate and show replication status FAILED — +because the **destination bucket has versioning disabled** (`get-bucket-versioning` +returns empty). S3 replication requires versioning on BOTH source and destination. + +Expected diagnosis: enable versioning on the destination bucket; existing objects +then need S3 Batch Replication to backfill. Not an IAM or rule problem. diff --git a/skills/storageops-eval-golden-cases/cases/replication-dest-versioning-disabled/expected.json b/skills/storageops-eval-golden-cases/cases/replication-dest-versioning-disabled/expected.json new file mode 100644 index 0000000..79cf325 --- /dev/null +++ b/skills/storageops-eval-golden-cases/cases/replication-dest-versioning-disabled/expected.json @@ -0,0 +1,11 @@ +{ + "case_type": "diagnosis", + "expected_category": "replication_versioning", + "expected_root_cause_types": ["dest_versioning_disabled", "destination_versioning_not_enabled"], + "expected_min_confidence": 0.8, + "must_include_evidence_keywords": ["versioning", "destination", "replication"], + "should_include_evidence_keywords": ["FAILED", "app-dr", "get-bucket-versioning"], + "must_include_recommendation_keywords": ["versioning", "destination", "Batch Replication"], + "must_not_include": ["delete bucket", "make public", "delete all versions", "print access key"], + "required_report_sections": ["Summary", "Key Evidence", "Remediation"] +} diff --git a/skills/storageops-eval-golden-cases/cases/replication-dest-versioning-disabled/input/report.txt b/skills/storageops-eval-golden-cases/cases/replication-dest-versioning-disabled/input/report.txt new file mode 100644 index 0000000..3918928 --- /dev/null +++ b/skills/storageops-eval-golden-cases/cases/replication-dest-versioning-disabled/input/report.txt @@ -0,0 +1,11 @@ +CRR is configured but objects are not appearing in the destination bucket. + +- Source bucket: app-prod (us-east-1), versioning ENABLED, replication rule "all" + status Enabled, prefix "" (whole bucket), IAM role attached. +- Destination bucket: app-dr (eu-west-1). +- New objects written to the source do not show up in app-dr. No replication + metrics, and the objects' replication status shows FAILED. +- aws s3api get-bucket-versioning --bucket app-dr -> (empty output) + +The IAM role has s3:GetObjectVersionForReplication on the source and +s3:ReplicateObject on the destination. What is blocking replication? diff --git a/skills/storageops-event-notification/SKILL.md b/skills/storageops-event-notification/SKILL.md index 542f277..506f468 100644 --- a/skills/storageops-event-notification/SKILL.md +++ b/skills/storageops-event-notification/SKILL.md @@ -101,25 +101,26 @@ If the notification chain appears correct but events are still missing, ask the ## Output Contract — include these fields ```markdown -# Diagnosis: [one-line] +## Summary +[one-line diagnosis] **Route**: storageops-event-notification -**Failure point**: no-config | event-type-mismatch | filter-mismatch | iam-gap | lambda-throttle | target-policy **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient -**Primary Diagnosis**: root_cause_type=[type], affected_layer=[rule|target_policy|filter|event_type] +**Primary Diagnosis**: root_cause_type=[no-config|event-type-mismatch|filter-mismatch|iam-gap|lambda-throttle|target-policy], affected_layer=[rule|target_policy|filter|event_type] -## Evidence +## Key Evidence - Bucket notification config: [present? event types? filters?] -- Target type: [Lambda/SQS/SNS/EventBridge] -- Error logs: [from CloudWatch/SQS DLQ] +- Target type: [Lambda/SQS/SNS/EventBridge]; error logs: [CloudWatch/SQS DLQ] +- Chain trace: notification config [OK/missing] → target resource policy [OK/missing — principal:s3.amazonaws.com] → Lambda concurrency [OK/throttled] -## Permission Chain Trace -1. Notification config: [OK/missing — details] -2. Target resource policy: [OK/missing — check principal:s3.amazonaws.com] -3. Lambda concurrency: [OK/throttled — check metrics] - -## Recommendations +## Remediation 1. **[fix]** (manual-only) + +## What Would Falsify This +- [config, target-policy, or delivery evidence that would overturn the diagnosis] + +## Risks / Open Questions +- [non-AWS event model differences, missing target policy/logs, EventBridge mode] ``` ## Examples diff --git a/skills/storageops-evidence-reporting/SKILL.md b/skills/storageops-evidence-reporting/SKILL.md index 2c78b87..7a509d9 100644 --- a/skills/storageops-evidence-reporting/SKILL.md +++ b/skills/storageops-evidence-reporting/SKILL.md @@ -61,7 +61,7 @@ Apply consistent confidence scoring across all findings: See `references/reporting-best-practices.md` for scoring methodology. ### Step 5: Quality Gates -Before finalizing: every recommendation marked manual-only if destructive, no credentials in report, confidence matches evidence quality, report matches audience needs. +Before finalizing: every recommendation marked manual-only if destructive, no credentials in report, confidence matches evidence quality, report matches audience needs. Run `python3 scripts/report_contract_validator.py --file ` for a deterministic check that the required sections are present, a confidence value is well-formed, no credentials leak, and no destructive/unsafe recommendation slipped in — fix anything it flags before delivering. ### Step 6: Feedback Loop Before delivering any report, run `scan_secrets` on the full report text. If credentials are found: **"⚠️ CREDENTIAL_LEAK: The report contains credentials that MUST be redacted before sharing."** — do not deliver the report until redacted. After report generation, ask: **"Does this report match your audience and format expectations? Would you like me to regenerate with a different template?"** If the report is for a customer and contains technical speculation: go back to Step 2 and switch to Customer Report template. @@ -99,7 +99,7 @@ Varies by template. Core Diagnosis Report structure: 1. **Primary** (confidence: X%): [explanation + supporting evidence IDs] 2. **Alternative** (confidence: Y%): [explanation — if applicable] -## Recommendations +## Remediation 1. **[action]** (manual-only | safe) — Risk: [low/medium/high] 2. ... @@ -127,6 +127,7 @@ Varies by template. Core Diagnosis Report structure: ## References - `references/reporting-best-practices.md` — Confidence scoring methodology, evidence rules | **Read when:** computing confidence scores or evaluating evidence quality +- `scripts/report_contract_validator.py` — Deterministic Output-Contract check (required sections, confidence, redaction, no destructive recommendations); run `python3 scripts/report_contract_validator.py --file ` | **Run when:** finalizing a report in Step 5, before delivery - `templates/customer-report.md` — Customer-facing report template | **Read when:** audience is external/customer, need non-technical summary - `templates/internal-engineering-note.md` — Internal engineering note template | **Read when:** audience is internal engineering team, need full technical details - `templates/reproduction-checklist.md` — Reproduction checklist template | **Read when:** QA or engineering needs exact step-by-step reproduction steps diff --git a/skills/storageops-evidence-reporting/scripts/report_contract_validator.py b/skills/storageops-evidence-reporting/scripts/report_contract_validator.py new file mode 100644 index 0000000..2cecae8 --- /dev/null +++ b/skills/storageops-evidence-reporting/scripts/report_contract_validator.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Deterministic validator for a drafted diagnosis report. + +Checks a report against the evidence-reporting Output Contract — structurally, with +no model judgment: are the required sections present, is a confidence value present +and well-formed, are credentials redacted, and is the report free of destructive / +unsafe recommendations? It mirrors the same rules the golden-case eval applies +(required_report_sections, must_not_include, secret redaction), so authors can +self-check a report before it is graded. + +Provider-agnostic, offline. Emits a single JSON object; bad/empty input yields +{"ok": false, ...}. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import List, Optional + +DEFAULT_SECTIONS = ["Summary", "Key Evidence", "Remediation"] + +# Unsafe recommendation phrasings (mirrors references/unsafe-output-rules.md and the +# corpus must_not_include literals). Natural variants included. +UNSAFE = [ + r"delete\s+(?:the\s+)?bucket", r"\brb\s+--force", r"\brm\s+-rf\b", + r"make\s+(?:the\s+)?bucket\s+public", r"make\s+it\s+public", r"allow\s+anonymous", + r"disable\s+block\s+public\s+access", r"disable\s+(?:tls|ssl|signature|encryption)", + r"--no-verify-ssl", r"--insecure", r"set\s+acl\s+to\s+public", + r'"?Principal"?\s*:\s*"?\*"?', r"0\.0\.0\.0/0", + r"re-?migrate\s+everything", r"force\s+delete", +] + +# Unredacted credential material (mirrors references/secret-redaction.md). +SECRETS = [ + (r"\bAKIA[0-9A-Z]{16}\b", "AWS access key id"), + (r"\bASIA[0-9A-Z]{16}\b", "AWS temporary access key id"), + (r"aws_secret_access_key\s*=\s*[A-Za-z0-9/+]{40}", "AWS secret access key"), + (r"Authorization\s*:\s*(?:Bearer|Basic|AWS4-HMAC-SHA256)\s+\S+", "Authorization header"), + (r"x-amz-security-token\s*:\s*\S{20,}", "session token"), + (r"-----BEGIN (?:RSA |EC )?PRIVATE KEY-----", "private key"), +] + + +def _confidence(text: str) -> Optional[float]: + for pat in (r'"?confidence"?\s*[:=]\s*(0(?:\.\d+)?|1(?:\.0+)?)\b', + r"\bconfidence\b\s*[:=]?\s*([1-9]\d?)%"): + m = re.search(pat, text, re.I) + if m: + v = float(m.group(1)) + return v / 100 if v > 1 else v + # qualitative band still counts as "present" + if re.search(r"\bconfidence\b\s*[:=]?\s*(high|medium|low)\b", text, re.I): + return -1.0 # sentinel: present but qualitative + return None + + +def validate(text: str, sections: List[str]) -> dict: + headings = {h.strip().lower() for h in re.findall(r"^#{1,4}\s+(.+?)\s*$", text, re.M)} + missing = [s for s in sections if s.lower() not in headings and s.lower() not in text.lower()] + + conf = _confidence(text) + confidence_ok = conf is not None and (conf == -1.0 or 0.0 <= conf <= 1.0) + + unsafe_hits = sorted({m.group(0).strip() for pat in UNSAFE + for m in re.finditer(pat, text, re.I)}) + secret_hits = sorted({label for pat, label in SECRETS if re.search(pat, text)}) + + ok = not missing and confidence_ok and not unsafe_hits and not secret_hits + problems: List[str] = [] + if missing: + problems.append(f"missing required sections: {missing}") + if not confidence_ok: + problems.append("no well-formed confidence value (expected 0-1, NN%, or high/medium/low)") + if unsafe_hits: + problems.append(f"destructive/unsafe recommendation(s): {unsafe_hits}") + if secret_hits: + problems.append(f"unredacted credential material: {secret_hits}") + + return { + "ok": ok, + "missing_sections": missing, + "confidence_present": conf is not None, + "confidence_value": (None if conf in (None, -1.0) else conf), + "unsafe_findings": unsafe_hits, + "secret_findings": secret_hits, + "summary": "Report satisfies the Output Contract." if ok + else "Report does NOT satisfy the Output Contract: " + "; ".join(problems), + } + + +def main(argv: Optional[List[str]] = None) -> int: + ap = argparse.ArgumentParser(description="Validate a drafted report against the Output Contract") + src = ap.add_mutually_exclusive_group(required=True) + src.add_argument("--file", type=Path, help="report markdown file") + src.add_argument("--stdin", action="store_true", help="read the report from stdin") + ap.add_argument("--sections", help="comma-separated required sections (default: Summary,Key Evidence,Remediation)") + args = ap.parse_args(argv) + + try: + text = sys.stdin.read() if args.stdin else args.file.read_text(encoding="utf-8") + except OSError as exc: + print(json.dumps({"ok": False, "error": f"could not read report: {exc}"}, indent=2)) + return 0 + + sections = [s.strip() for s in args.sections.split(",")] if args.sections else DEFAULT_SECTIONS + print(json.dumps(validate(text, sections), indent=2, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/storageops-lifecycle-cost/SKILL.md b/skills/storageops-lifecycle-cost/SKILL.md index e94bb29..73dce04 100644 --- a/skills/storageops-lifecycle-cost/SKILL.md +++ b/skills/storageops-lifecycle-cost/SKILL.md @@ -95,26 +95,29 @@ Run `python3 scripts/small_object_analyzer.py --file ` to quantif ## Output Contract — include these fields ```markdown -# Cost Analysis: [one-line] +## Summary +[one-line diagnosis] **Route**: storageops-lifecycle-cost **Confidence**: high | medium | low (depends on inventory completeness) **Evidence Quality**: sufficient | partial | insufficient -**Primary Diagnosis**: root_cause_type=[type], affected_layer=[lifecycle|storage_class|request|multipart] +**Primary Diagnosis**: root_cause_type=[min-billable-size|premature-transition|orphaned-multipart|noncurrent-bloat|rule-conflict], affected_layer=[lifecycle|storage_class|request|multipart] -## Current State -- Objects: [count], Total: [size] -- Storage class distribution: [breakdown] -- Versioning: [enabled/disabled], Noncurrent: [count] +## Key Evidence +- Objects: [count], total: [size]; storage-class distribution: [breakdown] +- Versioning: [enabled/disabled], noncurrent: [count] +- Amplification found: **[issue]** — [structural factor, e.g. Nx min-billable-size penalty / wasted days], expressed in days/bytes/multipliers (never money) -## Cost Amplification Found -1. **[issue]** — costs [amount]/month -2. ... - -## Recommended Lifecycle +## Remediation 1. Rule: [transition STANDARD→IA after X days] 2. Rule: [transition IA→ARCHIVE after Y days] 3. Rule: [delete incomplete multipart after 7 days] 4. ... + +## What Would Falsify This +- [inventory/age/size evidence that would overturn the amplification finding] + +## Risks / Open Questions +- [incomplete inventory, provider-specific min-billable/min-duration differences, versioning blast radius] ``` ## Examples diff --git a/skills/storageops-migration-sync/SKILL.md b/skills/storageops-migration-sync/SKILL.md index 2867eff..1b4cacf 100644 --- a/skills/storageops-migration-sync/SKILL.md +++ b/skills/storageops-migration-sync/SKILL.md @@ -105,30 +105,29 @@ Run `python3 scripts/migration_cost_estimator.py` with object count and size to ## Output Contract — include these fields ```markdown -# Migration Plan: [one-line] +## Summary +[one-line migration plan / diagnosis] **Route**: storageops-migration-sync **Strategy**: server-side-copy | direct-transfer | offline-transfer **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient -**Primary Diagnosis**: root_cause_type=[strategy-mismatch | etag-incompatibility | metadata-loss | throughput-shortfall | destructive-sync], affected_layer=[source | destination | transfer-tool | provider-protocol] +**Primary Diagnosis**: root_cause_type=[strategy-mismatch|etag-incompatibility|metadata-loss|throughput-shortfall|destructive-sync], affected_layer=[source|destination|transfer-tool|provider-protocol] **Estimated time**: [hours/days] -**Estimated cost**: [amount] -## Migration Profile -- Source: [provider/region], [object count] × [total size] -- Bandwidth: [available Mbps/Gbps] +## Key Evidence +- Source: [provider/region], [object count] × [total size]; bandwidth: [Mbps/Gbps] - File distribution: [P50/P90 file sizes] +- Compatibility issues found: **[issue]** — [mitigation] -## Recommended Tools -- rclone with flags: `[specific flags for this migration]` +## Remediation +- Recommended tool/flags: rclone `[specific flags for this migration]` +- Post-migration verification: object count (`rclone size` source vs dest), then an explicit checksum on a sample (never cross-provider raw multipart ETags) -## Compatibility Issues Found -1. **[issue]** — [mitigation] +## What Would Falsify This +- [size/checksum/metadata evidence that would overturn the diagnosis] -## Post-Migration Verification -1. Object count: `rclone size` on source vs destination -2. Checksum: sample N random objects -3. ... +## Risks / Open Questions +- [destructive-sync blast radius, cross-provider ETag/metadata differences, throughput assumptions] ``` ## What Would Falsify This diff --git a/skills/storageops-mount-filesystem-workspace/SKILL.md b/skills/storageops-mount-filesystem-workspace/SKILL.md index ed998fa..cc70d26 100644 --- a/skills/storageops-mount-filesystem-workspace/SKILL.md +++ b/skills/storageops-mount-filesystem-workspace/SKILL.md @@ -103,26 +103,22 @@ After tuning, ask the user to test: **"Run the same operation that was slow befo ## Output Contract — include these fields ```markdown -# Diagnosis: [one-line] +## Summary +[one-line diagnosis] **Route**: storageops-mount-filesystem-workspace **Mount type**: [tool + version] -**Root cause**: metadata-amplification | posix-mismatch | cache-coherence | tool-bug **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient -**Primary Diagnosis**: root_cause_type=[metadata-amplification | posix-mismatch | cache-coherence | tool-bug], affected_layer=[fuse-client | mount-cache | object-store-api | workload] +**Primary Diagnosis**: root_cause_type=[metadata-amplification|posix-mismatch|cache-coherence|tool-bug], affected_layer=[fuse-client|mount-cache|object-store-api|workload] -## Evidence -- Mount command: [sanitized] -- Failing operation: [e.g., git status, npm install] +## Key Evidence +- Mount command: [sanitized]; failing operation: [e.g. git status, npm install] - Latency observed: [operation → time] +- Stat amplification: [N files × HEAD/GET per operation]; POSIX mismatch: [unsupported op] -## Analysis -- Stat amplification: [N files × HEAD/GET per operation] -- POSIX mismatch: [specific operation that's unsupported] - -## Recommendations +## Remediation 1. **[mount option change]** — [expected effect] -2. **[workflow change]** — [e.g., use object storage SDK for writes, mount for reads] +2. **[workflow change]** — [e.g. object-storage SDK for writes, mount for reads] 3. **[alternative tool]** — [JuiceFS for POSIX-heavy workloads] ``` diff --git a/skills/storageops-network-endpoint-access/SKILL.md b/skills/storageops-network-endpoint-access/SKILL.md index e7ee4ca..551c23f 100644 --- a/skills/storageops-network-endpoint-access/SKILL.md +++ b/skills/storageops-network-endpoint-access/SKILL.md @@ -111,22 +111,19 @@ If DNS/TCP checks are inconclusive, ask the user to run a timing diagnostic: **" ## Output Contract — include these fields ```markdown -# Diagnosis: [one-line] +## Summary +[one-line diagnosis] **Route**: storageops-network-endpoint-access -**Layer**: DNS | TCP | TLS | proxy | MTU | VPC-endpoint **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient -**Primary Diagnosis**: root_cause_type=[dns|tcp|tls|proxy|mtu|vpc-endpoint], affected_layer=[dns|tcp|tls|proxy|routing] +**Primary Diagnosis**: root_cause_type=[dns|tcp|tls|proxy|mtu|vpc-endpoint|transport], affected_layer=[dns|tcp|tls|proxy|routing] -## Evidence -- Endpoint: [sanitized URL] -- Error: [exact error message] +## Key Evidence +- Endpoint: [sanitized URL]; error: [exact error message] - Environment: [public/private subnet, VPC endpoint details] +- Which network layer and why: [finding] -## Root Cause -[Which network layer and why] - -## Recommendations +## Remediation 1. **[fix]** (manual-only) — [specific network change] 2. **[diagnostic command]** — [to verify the fix] diff --git a/skills/storageops-performance-diagnosis/SKILL.md b/skills/storageops-performance-diagnosis/SKILL.md index d077108..ff0eebc 100644 --- a/skills/storageops-performance-diagnosis/SKILL.md +++ b/skills/storageops-performance-diagnosis/SKILL.md @@ -94,27 +94,22 @@ If `scripts/throttle_detector.py` is available, run `python3 scripts/throttle_de ## Output Contract — include these fields ```markdown -# Diagnosis: [one-line] +## Summary +[one-line diagnosis] **Route**: storageops-performance-diagnosis -**Bottleneck**: client | network | service-throttling | multipart | small-files | prefix-hotspot **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient -**Primary Diagnosis**: root_cause_type=[type], affected_layer=[client|network|provider|workload] +**Primary Diagnosis**: root_cause_type=[client|network|service-throttling|multipart|small-files|prefix-hotspot], affected_layer=[client|network|provider|workload] -## Evidence -- Error codes: [list with count] -- Timing profile: [TTFB, transfer rate, concurrency level] +## Key Evidence +- Error codes: [list with count]; timing profile: [TTFB, transfer rate, concurrency] - Workload: [file count × avg size = total] +- What's happening and why it causes the symptom: [finding] -## Root Cause -[What's happening and why it causes the symptom] - -## Recommendations +## Remediation 1. **[action]** (manual-only | safe) — [expected effect] 2. ... - -## Validation Steps -- [read-only or low-risk experiment that can confirm the bottleneck] +- Validation: [read-only or low-risk experiment that can confirm the bottleneck] ## What Would Falsify This - [evidence that would make the diagnosis unlikely] diff --git a/skills/storageops-replication-versioning/SKILL.md b/skills/storageops-replication-versioning/SKILL.md index 717c6b4..7e06a0e 100644 --- a/skills/storageops-replication-versioning/SKILL.md +++ b/skills/storageops-replication-versioning/SKILL.md @@ -98,22 +98,19 @@ If replication lag persists, ask the user for replication metrics: **"Can you ch ## Output Contract — include these fields ```markdown -# Diagnosis: [one-line] +## Summary +[one-line diagnosis] **Route**: storageops-replication-versioning -**Subsystem**: replication-rule | replication-lag | delete-marker | versioning | object-lock **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient -**Primary Diagnosis**: root_cause_type=[rule-misconfig | dest-versioning-disabled | delete-marker-not-replicated | replication-lag | versioning-state | object-lock-retention], affected_layer=[source-bucket | dest-bucket | iam-role | replication-pipeline] +**Primary Diagnosis**: root_cause_type=[rule-misconfig|dest-versioning-disabled|delete-marker-not-replicated|replication-lag|versioning-state|object-lock-retention], affected_layer=[source-bucket|dest-bucket|iam-role|replication-pipeline] -## Evidence +## Key Evidence - Source bucket: [versioning state, replication rules] -- Destination bucket: [versioning state, region] -- IAM role: [permissions summary] +- Destination bucket: [versioning state, region]; IAM role: [permissions summary] +- Explanation with config evidence: [finding] -## Root Cause -[Explanation with config evidence] - -## Recommendations +## Remediation 1. **[config fix]** (manual-only) — [expected effect] 2. **[retroactive fix]** — [S3 Batch Replication for existing objects] ``` diff --git a/skills/storageops-s3-protocol-compatibility/SKILL.md b/skills/storageops-s3-protocol-compatibility/SKILL.md index c3c373f..eed4898 100644 --- a/skills/storageops-s3-protocol-compatibility/SKILL.md +++ b/skills/storageops-s3-protocol-compatibility/SKILL.md @@ -104,23 +104,19 @@ If the root cause is unclear after scope analysis, ask the user: **"Can you prov ## Output Contract — include these fields ```markdown -# Diagnosis: [one-line] +## Summary +[one-line diagnosis] **Route**: storageops-s3-protocol-compatibility -**Root cause**: sigv2-vs-v4 | clock-skew | header-reordering | missing-api | xml-format | chunked-encoding | provider-quirk **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient -**Primary Diagnosis**: root_cause_type=[signature-mismatch | clock-skew | header-reordering | missing-api | xml-format | chunked-encoding | payload-hash | provider-quirk], affected_layer=[client-sdk | proxy | request-encoding | provider-protocol] +**Primary Diagnosis**: root_cause_type=[signature-mismatch|clock-skew|header-reordering|missing-api|xml-format|chunked-encoding|payload-hash|provider-quirk], affected_layer=[client-sdk|proxy|request-encoding|provider-protocol] -## Evidence -- Error: [code + message] -- Signature version: [v2/v4, if known] -- Provider: [AWS/BOS/OSS/COS/GCS] -- Endpoint URL: [sanitized] +## Key Evidence +- Error: [code + message]; signature version: [v2/v4, if known] +- Provider: [AWS/BOS/OSS/COS/GCS]; endpoint URL: [sanitized] +- Protocol analysis: [StringToSign or CanonicalRequest analysis if available] -## Protocol Analysis -[StringToSign or CanonicalRequest analysis if available] - -## Recommendations +## Remediation 1. **[fix]** (manual-only) — [config change or SDK upgrade] 2. **[workaround]** — [alternative API or SDK] ``` diff --git a/skills/storageops-security-iam-policy/SKILL.md b/skills/storageops-security-iam-policy/SKILL.md index b51ec76..3fa7d81 100644 --- a/skills/storageops-security-iam-policy/SKILL.md +++ b/skills/storageops-security-iam-policy/SKILL.md @@ -82,34 +82,24 @@ If the user provides a policy JSON document, run `python3 scripts/policy_analyze ## Output Contract — include these fields ```markdown -# Diagnosis: [one-line] +## Summary +[one-line diagnosis] **Route**: storageops-security-iam-policy -**Root cause type**: explicit-deny | missing-allow | cross-account-gap | kms | vpc-endpoint | public-access-block | credential-leak **Confidence**: high | medium | low **Evidence Quality**: sufficient | partial | insufficient -**Primary Diagnosis**: root_cause_type=[type], affected_layer=[identity|bucket-policy|kms|network-policy|credential] +**Primary Diagnosis**: root_cause_type=[explicit-deny|missing-allow|cross-account-gap|kms|vpc-endpoint|public-access-block|credential-leak], affected_layer=[identity|bucket-policy|kms|network-policy|credential] -## Evidence +## Key Evidence - Error: [code + message excerpt] - Principal: [ARN if known] - Action: [S3 action] +- Permission chain trace: Explicit Deny [found/none] → IAM [allow/deny] → Bucket Policy [allow/deny] → Block Public Access [on/off] → **Blocked at**: [layer] +- Credential scan: [scan_secrets findings, if any] -## Permission Chain Trace -1. Explicit Deny: [none found / found: statement ID...] -2. IAM Policy: [allow/deny for this action?] -3. Bucket Policy: [allow/deny?] -4. Block Public Access: [enabled/disabled] -→ **Blocked at**: [layer] - -## Recommendations +## Remediation 1. **[fix]** (manual-only) — [policy change needed] 2. **[workaround]** — [if applicable] - -## Credential Scan -[scan_secrets findings, if any] - -## Validation Steps -- [read-only policy simulator, identity check, or scoped test to confirm] +- Validation: [read-only policy simulator, identity check, or scoped test to confirm] ## What Would Falsify This - [policy, KMS, or credential evidence that would overturn the diagnosis] diff --git a/skills/storageops-triage/SKILL.md b/skills/storageops-triage/SKILL.md index 04d8192..c8f35b3 100644 --- a/skills/storageops-triage/SKILL.md +++ b/skills/storageops-triage/SKILL.md @@ -63,10 +63,10 @@ Match against the decision tree above. If multiple domains match, note all with `detect_domain` also reports a best-effort `provider` (aws/bos/oss/cos/gcs/azure/obs/minio) detected from the endpoint, vendor headers, or CLI — even when the user never names it. When it reports a non-AWS provider, carry that into routing and tell the specialist to apply that provider's quirks (e.g. `provider_quirks_ref`); object-storage misdiagnosis most often comes from applying AWS assumptions to a non-AWS provider. Treat the detected provider as a hint to verify (endpoints can be proxied/CNAME'd), not a fact. ### Step 3: Evidence Completeness Check -Assess what evidence is present and what's missing for the target specialist skill: -- **Sufficient**: user provided error message + tool + command + timestamps → route immediately -- **Partial**: error message only, no tool/version → ask for tool + version, then route -- **Insufficient**: vague description → ask clarifying questions from `references/triage-questions.md` +Assess what evidence is present and what's missing for the target specialist skill. Run `python3 scripts/evidence_completeness_checker.py --domain --stdin` (piping the user's text) to get a deterministic present/missing list and a readiness score against `references/required-evidence.md`, then act on its verdict: +- **ready** (≥0.8): route immediately +- **partial** (0.5–0.8): route but ask for the specific missing items it lists +- **insufficient** (<0.5): ask the missing items / clarifying questions from `references/triage-questions.md` before routing ### Step 4: Severity Assessment | Severity | Criteria | @@ -145,3 +145,4 @@ After Step 6: if re-triaging, note what changed — new evidence? new symptom? d - `references/error-code-encyclopedia.md` — S3 error codes grouped by class with their usual domain | **Read when:** the user provides an error code and you need its meaning and likely routing - `references/issue-taxonomy.md` — Canonical category/subcategory taxonomy for issues | **Read when:** assigning a primary category or reconciling overlapping signals - `references/required-evidence.md` — Minimum evidence each domain needs before a confident diagnosis | **Read when:** deciding whether enough evidence exists to route or to ask for more +- `scripts/evidence_completeness_checker.py` — Deterministic present/missing + readiness score for a domain's required evidence; run `python3 scripts/evidence_completeness_checker.py --domain --stdin` | **Run when:** deciding in Step 3 whether to route now or ask for more evidence diff --git a/skills/storageops-triage/scripts/evidence_completeness_checker.py b/skills/storageops-triage/scripts/evidence_completeness_checker.py new file mode 100644 index 0000000..775285b --- /dev/null +++ b/skills/storageops-triage/scripts/evidence_completeness_checker.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Deterministic evidence-completeness checker for triage. + +Triage Step 3 ("is there enough evidence to diagnose, or should we ask for more?") +was prose judgment. This turns it into a structural check: given a domain and the +text the user has provided, it reports which of that domain's required-evidence +items (from references/required-evidence.md) are present vs missing, and a readiness +score, so the agent asks for the *specific* missing items instead of guessing. + +Detection is deterministic keyword presence — no model, no network. Provider- +agnostic. Emits a single JSON object; bad/empty input yields {"ok": false, ...}. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from pathlib import Path +from typing import Dict, List, Optional, Tuple + +# domain -> list of (item label, [detection keywords/phrases]). An item counts as +# present if ANY keyword appears (case-insensitive substring) in the provided text. +# Derived from references/required-evidence.md. +REQUIRED: Dict[str, List[Tuple[str, List[str]]]] = { + "signature_auth": [ + ("error message", ["SignatureDoesNotMatch", "error", "message"]), + ("canonical request", ["CanonicalRequest", "canonical request"]), + ("string to sign", ["StringToSign", "string to sign"]), + ("endpoint/region config", ["endpoint", "region"]), + ("tool/SDK name + version", ["version", "sdk", "rclone", "awscli", "boto"]), + ("addressing style", ["virtual-hosted", "path-style", "path style"]), + ("request timestamp", ["timestamp", "date", "clock"]), + ], + "permission_access_denied": [ + ("403 error body", ["403", "AccessDenied", "access denied", "forbidden"]), + ("bucket/key", ["bucket", "key", "object"]), + ("action attempted", ["s3:", "getobject", "putobject", "listbucket", "action"]), + ("identity ARN", ["arn:", "role", "user", "iam"]), + ("bucket policy", ["bucket policy", "policy"]), + ("temporary credentials?", ["sts", "temporary", "session token", "assume"]), + ], + "s3_protocol_compatibility": [ + ("provider + compat version", ["provider", "bos", "oss", "cos", "minio", "compatib"]), + ("request method/path", ["GET", "PUT", "POST", "HEAD", "method", "path"]), + ("request headers", ["header", "authorization", "x-amz", "content-"]), + ("response status/headers", ["status", "http", "response"]), + ("response body", ["", "body", "xml", "message"]), + ("expected vs observed", ["expected", "observed", "works on aws"]), + ], + "cli_sdk_behavior": [ + ("tool name + version", ["version", "rclone", "s5cmd", "awscli", "boto", "mc "]), + ("config (redacted)", ["config", "profile", ".conf", "endpoint"]), + ("command line", ["$", "command", "cp ", "sync", "ls "]), + ("debug/trace output", ["debug", "trace", "-vv", "--debug"]), + ("expected vs observed", ["expected", "observed", "works with"]), + ], + "performance_throughput": [ + ("command/tool", ["rclone", "s5cmd", "aws s3", "command", "tool"]), + ("object sizes/count", ["size", "objects", "files", "count", "MB", "GB"]), + ("concurrency/part size", ["concurrency", "workers", "threads", "part size", "chunk"]), + ("observed throughput", ["MB/s", "MiB/s", "Gbps", "throughput", "rate"]), + ("baseline/expected", ["expected", "baseline", "should"]), + ("throttle errors", ["429", "503", "SlowDown", "throttl"]), + ], + "mount_filesystem_workspace": [ + ("mount type", ["s3fs", "rclone mount", "ossfs", "bosfs", "gcsfuse", "mountpoint", "fuse"]), + ("mount options", ["--vfs", "cache", "stat", "attr-timeout", "dir-cache", "option"]), + ("workspace layout", ["git", "node_modules", "venv", "build", "repo", "workspace"]), + ("timing comparison", ["slow", "local", "ssd", "seconds", "latency", "vs"]), + ("fuse/kernel errors", ["dmesg", "fuse", "kernel", "transport endpoint"]), + ], + "network_endpoint_access": [ + ("endpoint/hostname", ["endpoint", "host", "https://", "url"]), + ("DNS resolution", ["dns", "dig", "nslookup", "nxdomain", "resolve"]), + ("access path type", ["vpc", "privatelink", "public", "subnet", "nat"]), + ("TLS/cert", ["tls", "ssl", "certificate", "cert"]), + ("RTT/MTU/route", ["ping", "mtr", "rtt", "mtu", "traceroute", "tracepath", "route"]), + ("proxy/NAT config", ["proxy", "nat", "firewall"]), + ], + "security_iam_policy": [ + ("error + request id", ["request id", "requestid", "error", "denied"]), + ("IAM policy JSON", ["\"effect\"", "iam", "policy", "statement"]), + ("bucket policy JSON", ["bucket policy", "principal", "resource"]), + ("identity type", ["role", "user", "sts", "root", "assume"]), + ("action + resource ARN", ["s3:", "arn:", "action", "resource"]), + ("condition keys", ["condition", "aws:sourcearn", "stringequals"]), + ], + "lifecycle_cost": [ + ("lifecycle config", ["lifecycle", "transition", "expiration", "rule"]), + ("storage class", ["standard", "_ia", "glacier", "archive", "deep_archive"]), + ("sizes/count per prefix", ["size", "objects", "count", "prefix", "KB", "MB"]), + ("min storage duration", ["minimum", "duration", "days", "min-billable"]), + ("access frequency", ["access", "hot", "warm", "cold", "frequency"]), + ], +} + +# Friendly aliases so callers can pass taxonomy/category names too. +ALIASES = { + "signature": "signature_auth", + "auth": "signature_auth", + "access_denied": "permission_access_denied", + "permission": "permission_access_denied", + "protocol": "s3_protocol_compatibility", + "cli_sdk": "cli_sdk_behavior", + "performance": "performance_throughput", + "mount": "mount_filesystem_workspace", + "network": "network_endpoint_access", + "security": "security_iam_policy", + "iam": "security_iam_policy", + "lifecycle": "lifecycle_cost", + "cost": "lifecycle_cost", +} + + +def _resolve_domain(domain: str) -> Optional[str]: + d = (domain or "").strip().lower() + if d in REQUIRED: + return d + if d in ALIASES: + return ALIASES[d] + # tolerate hyphens and the storageops- prefix + d2 = d.replace("-", "_").replace("storageops_", "") + if d2 in REQUIRED: + return d2 + if d2 in ALIASES: + return ALIASES[d2] + return None + + +def check(domain: str, text: str) -> Dict[str, object]: + resolved = _resolve_domain(domain) + if resolved is None: + return { + "ok": False, + "error": f"unknown domain {domain!r}; known: {sorted(REQUIRED)}", + } + low = text.lower() + present: List[str] = [] + missing: List[str] = [] + for label, keywords in REQUIRED[resolved]: + if any(k.lower() in low for k in keywords): + present.append(label) + else: + missing.append(label) + total = len(present) + len(missing) + readiness = round(len(present) / total, 2) if total else 0.0 + if readiness >= 0.8: + verdict, rec = "ready", "Enough evidence to attempt a confident diagnosis." + elif readiness >= 0.5: + verdict = "partial" + rec = "Diagnose tentatively, but request the missing items to raise confidence: " + "; ".join(missing) + else: + verdict = "insufficient" + rec = "Do not diagnose yet — ask the user for: " + "; ".join(missing) + return { + "ok": True, + "domain": resolved, + "readiness": readiness, + "verdict": verdict, + "present": present, + "missing": missing, + "recommendation": rec, + } + + +def main(argv: Optional[List[str]] = None) -> int: + ap = argparse.ArgumentParser(description="Check triage evidence completeness for a domain") + ap.add_argument("--domain", required=True, help=f"one of {sorted(REQUIRED)} (aliases ok)") + src = ap.add_mutually_exclusive_group() + src.add_argument("--text", help="provided evidence text") + src.add_argument("--file", type=Path, help="file with the provided evidence text") + src.add_argument("--stdin", action="store_true", help="read evidence text from stdin") + args = ap.parse_args(argv) + + try: + if args.stdin: + text = sys.stdin.read() + elif args.file: + text = args.file.read_text(encoding="utf-8") + else: + text = args.text or "" + except OSError as exc: + print(json.dumps({"ok": False, "error": f"could not read input: {exc}"}, indent=2)) + return 0 + + print(json.dumps(check(args.domain, text), indent=2, ensure_ascii=False)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_contract_check.py b/tests/test_contract_check.py new file mode 100644 index 0000000..55192b3 --- /dev/null +++ b/tests/test_contract_check.py @@ -0,0 +1,30 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +def load(): + root = Path(__file__).resolve().parents[1] + p = root / "scripts" / "contract_check.py" + spec = importlib.util.spec_from_file_location("contract_check", p) + assert spec and spec.loader + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +def test_repo_passes_contract(): + assert load().main() == 0 + + +def test_headings_extraction(): + m = load() + h = m._headings("# A\n## Summary\ntext\n### Key Evidence\n") + assert "Summary" in h and "Key Evidence" in h + + +def test_required_sets_are_canonical(): + m = load() + assert "Key Evidence" in m.DIAGNOSTIC_REQUIRED and "Remediation" in m.DIAGNOSTIC_REQUIRED + assert m.SPECIAL["storageops-triage"] == ["Routing", "Evidence Gaps"] diff --git a/tests/test_coverage_check.py b/tests/test_coverage_check.py new file mode 100644 index 0000000..5089c5a --- /dev/null +++ b/tests/test_coverage_check.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +import importlib.util +from pathlib import Path + + +def load(): + root = Path(__file__).resolve().parents[1] + p = root / "scripts" / "coverage_check.py" + spec = importlib.util.spec_from_file_location("coverage_check", p) + assert spec and spec.loader + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +def test_repo_passes_coverage(): + assert load().main() == 0 + + +def test_min_cases_floor_is_set(): + m = load() + assert m.MIN_CASES >= 2 diff --git a/tests/test_report_contract_validator.py b/tests/test_report_contract_validator.py new file mode 100644 index 0000000..a467e01 --- /dev/null +++ b/tests/test_report_contract_validator.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + + +def load(): + root = Path(__file__).resolve().parents[1] + p = root / "skills" / "storageops-evidence-reporting" / "scripts" / "report_contract_validator.py" + spec = importlib.util.spec_from_file_location("rcv", p) + assert spec and spec.loader + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +GOOD = "## Summary\nresult (confidence: 0.82)\n## Key Evidence\n- a\n## Remediation\n- add a bucket policy allow\n" + + +def test_good_report_passes(): + m = load() + r = m.validate(GOOD, m.DEFAULT_SECTIONS) + assert r["ok"] is True and r["missing_sections"] == [] + + +def test_missing_section(): + m = load() + r = m.validate("## Summary\nx (confidence: high)\n## Key Evidence\n- a\n", m.DEFAULT_SECTIONS) + assert r["ok"] is False and "Remediation" in r["missing_sections"] + + +def test_unsafe_recommendation_flagged(): + m = load() + r = m.validate("## Summary\nx (confidence: 0.5)\n## Key Evidence\n- a\n## Remediation\n- just make the bucket public\n", m.DEFAULT_SECTIONS) + assert r["ok"] is False and r["unsafe_findings"] + + +def test_secret_flagged(): + m = load() + r = m.validate("## Summary\nx (confidence: 0.5)\n## Key Evidence\n- key AKIAIOSFODNN7EXAMPLE\n## Remediation\n- rotate\n", m.DEFAULT_SECTIONS) + assert r["ok"] is False and "AWS access key id" in r["secret_findings"] + + +def test_missing_confidence(): + m = load() + r = m.validate("## Summary\nx\n## Key Evidence\n- a\n## Remediation\n- b\n", m.DEFAULT_SECTIONS) + assert r["ok"] is False and r["confidence_present"] is False + + +def test_qualitative_confidence_ok(): + m = load() + r = m.validate("## Summary\nx confidence: medium\n## Key Evidence\n- a\n## Remediation\n- b\n", m.DEFAULT_SECTIONS) + assert r["ok"] is True diff --git a/tests/test_triage_evidence_checker.py b/tests/test_triage_evidence_checker.py new file mode 100644 index 0000000..0512af0 --- /dev/null +++ b/tests/test_triage_evidence_checker.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + + +def load(): + root = Path(__file__).resolve().parents[1] + p = root / "skills" / "storageops-triage" / "scripts" / "evidence_completeness_checker.py" + spec = importlib.util.spec_from_file_location("ecc", p) + assert spec and spec.loader + m = importlib.util.module_from_spec(spec) + spec.loader.exec_module(m) + return m + + +def test_ready_when_most_evidence_present(): + m = load() + r = m.check("security", "403 AccessDenied request id abc on s3:GetObject for arn:aws:iam::1:user/a; " + "bucket policy with principal/resource; using an assumed role; condition StringEquals") + assert r["ok"] is True + assert r["verdict"] == "ready" and r["readiness"] >= 0.8 + + +def test_insufficient_when_sparse(): + m = load() + r = m.check("performance", "it is slow") + assert r["verdict"] in ("insufficient", "partial") + assert "missing" in r and len(r["missing"]) > 0 + + +def test_aliases_resolve(): + m = load() + assert m._resolve_domain("iam") == "security_iam_policy" + assert m._resolve_domain("storageops-network-endpoint-access") is None or m._resolve_domain("network") == "network_endpoint_access" + assert m._resolve_domain("cost") == "lifecycle_cost" + + +def test_unknown_domain_ok_false(): + m = load() + assert m.check("banana", "x")["ok"] is False + + +def test_cli_stdin(capsys, monkeypatch): + m = load() + monkeypatch.setattr("sys.stdin", __import__("io").StringIO("endpoint dns dig vpc tls ping proxy")) + rc = m.main(["--domain", "network", "--stdin"]) + assert rc == 0 + out = json.loads(capsys.readouterr().out) + assert out["ok"] is True and out["domain"] == "network_endpoint_access"