From cd31bffab1247ef7e03865461f0952a774f9d2b3 Mon Sep 17 00:00:00 2001 From: John Morrissey <544926+tachyon-beep@users.noreply.github.com> Date: Tue, 19 May 2026 02:13:42 +1000 Subject: [PATCH 1/4] feat(wp5): add pre-ingest secret scanner --- .github/workflows/ci.yml | 3 + Cargo.lock | 36 + Cargo.toml | 3 + crates/clarion-cli/Cargo.toml | 1 + crates/clarion-cli/src/analyze.rs | 89 +- crates/clarion-cli/src/cli.rs | 9 + crates/clarion-cli/src/main.rs | 32 +- crates/clarion-cli/src/secret_scan.rs | 224 ++++ .../clarion-cli/src/secret_scan/findings.rs | 123 ++ crates/clarion-cli/tests/secret_scan.rs | 440 +++++++ crates/clarion-core/src/plugin/host.rs | 25 +- crates/clarion-mcp/src/lib.rs | 23 + crates/clarion-mcp/tests/storage_tools.rs | 56 + crates/clarion-scanner/Cargo.toml | 20 + crates/clarion-scanner/src/baseline.rs | 225 ++++ crates/clarion-scanner/src/entropy.rs | 60 + crates/clarion-scanner/src/lib.rs | 92 ++ crates/clarion-scanner/src/patterns.rs | 300 +++++ crates/clarion-scanner/tests/scanner.rs | 232 ++++ docs/clarion/v0.1/detailed-design.md | 15 + .../v0.1-publish/ws-a-secret-scanner.md | 1023 +++++++++++++++++ docs/operator/README.md | 2 + docs/operator/secret-scanning.md | 74 ++ tests/e2e/wp5_secret_scan.sh | 110 ++ 24 files changed, 3188 insertions(+), 29 deletions(-) create mode 100644 crates/clarion-cli/src/secret_scan.rs create mode 100644 crates/clarion-cli/src/secret_scan/findings.rs create mode 100644 crates/clarion-cli/tests/secret_scan.rs create mode 100644 crates/clarion-scanner/Cargo.toml create mode 100644 crates/clarion-scanner/src/baseline.rs create mode 100644 crates/clarion-scanner/src/entropy.rs create mode 100644 crates/clarion-scanner/src/lib.rs create mode 100644 crates/clarion-scanner/src/patterns.rs create mode 100644 crates/clarion-scanner/tests/scanner.rs create mode 100644 docs/implementation/v0.1-publish/ws-a-secret-scanner.md create mode 100644 docs/operator/secret-scanning.md create mode 100755 tests/e2e/wp5_secret_scan.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4cd32b3..54ba837a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -128,3 +128,6 @@ jobs: - name: run walking skeleton run: bash tests/e2e/sprint_1_walking_skeleton.sh + + - name: run WP5 secret scanner smoke + run: CARGO_BUILD=0 bash tests/e2e/wp5_secret_scan.sh diff --git a/Cargo.lock b/Cargo.lock index 2a433f0e..4eb52b52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -255,6 +255,7 @@ dependencies = [ "clarion-core", "clarion-mcp", "clarion-plugin-fixture", + "clarion-scanner", "clarion-storage", "dotenvy", "ignore", @@ -313,6 +314,18 @@ dependencies = [ "serde_json", ] +[[package]] +name = "clarion-scanner" +version = "0.1.0-dev" +dependencies = [ + "regex", + "serde", + "serde_norway", + "sha1", + "tempfile", + "thiserror 1.0.69", +] + [[package]] name = "clarion-storage" version = "0.1.0-dev" @@ -1312,6 +1325,18 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -1599,6 +1624,17 @@ dependencies = [ "serde", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures 0.2.17", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" diff --git a/Cargo.toml b/Cargo.toml index 4e6faf8e..94afb393 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,6 +6,7 @@ members = [ "crates/clarion-cli", "crates/clarion-mcp", "crates/clarion-plugin-fixture", + "crates/clarion-scanner", ] [workspace.package] @@ -38,9 +39,11 @@ dotenvy = "0.15" ignore = "0.4" rusqlite = { version = "0.31", features = ["bundled"] } reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls-native-roots"] } +regex = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" serde_norway = "0.9.42" +sha1 = "0.10" sha2 = "0.10" thiserror = "1" time = { version = "0.3", features = ["formatting", "macros", "parsing"] } diff --git a/crates/clarion-cli/Cargo.toml b/crates/clarion-cli/Cargo.toml index 2bd97d1c..4d6ebe70 100644 --- a/crates/clarion-cli/Cargo.toml +++ b/crates/clarion-cli/Cargo.toml @@ -19,6 +19,7 @@ blake3.workspace = true clap.workspace = true clarion-core = { path = "../clarion-core", version = "0.1.0-dev" } clarion-mcp = { path = "../clarion-mcp", version = "0.1.0-dev" } +clarion-scanner = { path = "../clarion-scanner", version = "0.1.0-dev" } clarion-storage = { path = "../clarion-storage", version = "0.1.0-dev" } dotenvy.workspace = true ignore.workspace = true diff --git a/crates/clarion-cli/src/analyze.rs b/crates/clarion-cli/src/analyze.rs index 21d44329..5ad5fa5b 100644 --- a/crates/clarion-cli/src/analyze.rs +++ b/crates/clarion-cli/src/analyze.rs @@ -41,6 +41,7 @@ const WEAK_MODULARITY_RULE_ID: &str = "CLA-FACT-CLUSTERING-WEAK-MODULARITY"; #[derive(Debug, Clone, Default)] pub(crate) struct AnalyzeOptions { pub(crate) config_path: Option, + pub(crate) secret_scan: crate::secret_scan::SecretScanOptions, } /// Run the analyze command against `project_path`. @@ -93,17 +94,6 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti let run_id = Uuid::new_v4().to_string(); let started_at = iso8601_now(); - writer - .send_wait(|ack| WriterCmd::BeginRun { - run_id: run_id.clone(), - config_json: analyze_config_json.clone(), - started_at: started_at.clone(), - ack, - }) - .await - .map_err(|e| anyhow::anyhow!("{e}")) - .context("BeginRun")?; - // ── Discover plugins ────────────────────────────────────────────────────── let discovery_results = discover(); let mut plugins: Vec = Vec::new(); @@ -138,6 +128,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti discovery_errors.join("; ") ); tracing::error!(run_id = %run_id, reason = %reason, "failing run: discovery errors"); + begin_run(&writer, &run_id, &analyze_config_json, &started_at).await?; let completed_at = iso8601_now(); writer .send_wait(|ack| WriterCmd::FailRun { @@ -164,6 +155,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti } tracing::warn!(run_id = %run_id, "no plugins discovered"); + begin_run(&writer, &run_id, &analyze_config_json, &started_at).await?; let completed_at = iso8601_now(); writer .send_wait(|ack| WriterCmd::CommitRun { @@ -215,6 +207,10 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti let source_files = collect_source_files(&project_root, &wanted_extensions); tracing::info!(file_count = source_files.len(), "source tree walk complete"); + let secret_scan_outcome = + crate::secret_scan::pre_ingest(&project_root, &source_files, &options.secret_scan)?; + begin_run(&writer, &run_id, &analyze_config_json, &started_at).await?; + // ── Per-plugin processing ───────────────────────────────────────────────── // // A per-plugin crash (spawn / handshake / analyze_file Err) does NOT tank @@ -242,6 +238,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti let mut run_outcome: RunOutcome = RunOutcome::Completed; let mut breaker = CrashLoopBreaker::default(); let mut crash_reasons: Vec = Vec::new(); + let mut secret_finding_anchors: BTreeMap = BTreeMap::new(); 'plugins: for plugin in plugins { let plugin_id = plugin.manifest.plugin.plugin_id.clone(); @@ -282,6 +279,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti let pid_clone = plugin_id.clone(); let exec_clone = plugin.executable.clone(); let files_clone = plugin_files.clone(); + let briefing_blocks_clone = secret_scan_outcome.briefing_blocks.clone(); // A JoinError here means the blocking task panicked (OOM, stack // overflow, internal unwrap, abort — anything that unwinds past the @@ -299,6 +297,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti &pid_clone, &exec_clone, &files_clone, + &briefing_blocks_clone, ) }) .await, @@ -361,6 +360,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti // edge FK references resolve at insert time (B.3 §5). let entity_count = entities.len() as u64; let edge_count = edges.len() as u64; + remember_secret_finding_anchors(&entities, &mut secret_finding_anchors); let mut insert_err: Option = None; for (id_str, record) in entities { let res = writer @@ -435,6 +435,22 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti } } + if !matches!(run_outcome, RunOutcome::HardFailed { .. }) + && let Err(e) = crate::secret_scan::emit_findings( + &writer, + &run_id, + &started_at, + &secret_scan_outcome, + &secret_finding_anchors, + ) + .await + { + tracing::error!(run_id = %run_id, error = %e, "secret finding persistence failed"); + run_outcome = RunOutcome::HardFailed { + reason: format!("secret finding persistence failed: {e:#}"), + }; + } + // ── Commit or fail the run ──────────────────────────────────────────────── // // Writer-actor failures set `run_outcome = HardFailed` above (and break). @@ -497,7 +513,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti match run_outcome { RunOutcome::Completed => { - let stats_json = serde_json::json!({ + let mut stats_json = serde_json::json!({ "entities_inserted": total_entity_count, "edges_inserted": total_edge_count, "dropped_edges_total": dropped_edges_total, @@ -513,8 +529,9 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti "pyright_index_parse_latency_p95_ms": pyright_index_parse_latency_p95_ms, "extractor_parse_latency_p95_ms": extractor_parse_latency_p95_ms, "clustering": phase3_output.clustering_stats.clone(), - }) - .to_string(); + }); + secret_scan_outcome.augment_stats(&mut stats_json); + let stats_json = stats_json.to_string(); writer .send_wait(|ack| WriterCmd::CommitRun { run_id: run_id.clone(), @@ -532,7 +549,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti // failed, atomically (writer folds the UPDATE into the open tx). // The stats JSON carries both fields so operators can see what // was persisted alongside the failure reason. - let stats_json = serde_json::json!({ + let mut stats_json = serde_json::json!({ "entities_inserted": total_entity_count, "edges_inserted": total_edge_count, "dropped_edges_total": dropped_edges_total, @@ -549,8 +566,9 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti "extractor_parse_latency_p95_ms": extractor_parse_latency_p95_ms, "clustering": phase3_output.clustering_stats.clone(), "failure_reason": reason, - }) - .to_string(); + }); + secret_scan_outcome.augment_stats(&mut stats_json); + let stats_json = stats_json.to_string(); writer .send_wait(|ack| WriterCmd::CommitRun { run_id: run_id.clone(), @@ -597,6 +615,24 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti Ok(()) } +async fn begin_run( + writer: &Writer, + run_id: &str, + analyze_config_json: &str, + started_at: &str, +) -> Result<()> { + writer + .send_wait(|ack| WriterCmd::BeginRun { + run_id: run_id.to_owned(), + config_json: analyze_config_json.to_owned(), + started_at: started_at.to_owned(), + ack, + }) + .await + .map_err(|e| anyhow::anyhow!("{e}")) + .context("BeginRun") +} + // ── Phase 3 subsystem materialisation ───────────────────────────────────────── #[derive(Debug, Clone)] @@ -1170,6 +1206,7 @@ fn run_plugin_blocking( plugin_id: &str, executable: &Path, files: &[PathBuf], + briefing_blocks: &BTreeMap, ) -> Result { use clarion_core::PluginHost; @@ -1185,6 +1222,7 @@ fn run_plugin_blocking( PluginRunError::new(format!("plugin {plugin_id} spawn/handshake error: {other}")) } })?; + host.set_briefing_blocks(briefing_blocks.clone()); let work_result: Result = (|| { let mut collected_entities: Vec<(String, EntityRecord)> = Vec::new(); @@ -1496,6 +1534,23 @@ fn absolute_from_import_submodule_target( .then_some(candidate) } +fn remember_secret_finding_anchors( + entities: &[(String, EntityRecord)], + anchors: &mut BTreeMap, +) { + for (id, record) in entities { + let Some(path) = record.source_file_path.as_deref() else { + continue; + }; + let key = Path::new(path) + .canonicalize() + .unwrap_or_else(|_| PathBuf::from(path)); + if record.kind == "module" || !anchors.contains_key(&key) { + anchors.insert(key, id.clone()); + } + } +} + /// Map an `AcceptedEntity` to an `EntityRecord` for the writer-actor. fn map_entity_to_record( entity: &AcceptedEntity, diff --git a/crates/clarion-cli/src/cli.rs b/crates/clarion-cli/src/cli.rs index 859631e8..34070d49 100644 --- a/crates/clarion-cli/src/cli.rs +++ b/crates/clarion-cli/src/cli.rs @@ -32,6 +32,15 @@ pub enum Command { /// Path to clarion.yaml (default: project-root/clarion.yaml if present). #[arg(long)] config: Option, + + /// Allow analysis of files containing unredacted secrets. Requires a + /// confirmation step when detections are present. + #[arg(long)] + allow_unredacted_secrets: bool, + + /// Non-TTY confirmation token for --allow-unredacted-secrets. + #[arg(long, value_name = "TOKEN", requires = "allow_unredacted_secrets")] + confirm_allow_unredacted_secrets: Option, }, /// Run the MCP stdio server. diff --git a/crates/clarion-cli/src/main.rs b/crates/clarion-cli/src/main.rs index 048d96c5..9b3c1a5a 100644 --- a/crates/clarion-cli/src/main.rs +++ b/crates/clarion-cli/src/main.rs @@ -3,6 +3,7 @@ mod cli; mod clustering; mod config; mod install; +mod secret_scan; mod serve; mod stats; @@ -20,20 +21,31 @@ fn main() -> Result<()> { let cli = cli::Cli::parse(); match cli.command { cli::Command::Install { force, path } => install::run(&path, force), - cli::Command::Analyze { path, config } => { + cli::Command::Analyze { + path, + config, + allow_unredacted_secrets, + confirm_allow_unredacted_secrets, + } => { let rt = tokio::runtime::Builder::new_multi_thread() .enable_all() .build()?; - if let Some(config_path) = config { - rt.block_on(analyze::run_with_options( - path, - analyze::AnalyzeOptions { - config_path: Some(config_path), - }, - )) - } else { - rt.block_on(analyze::run(path)) + if config.is_none() + && !allow_unredacted_secrets + && confirm_allow_unredacted_secrets.is_none() + { + return rt.block_on(analyze::run(path)); } + rt.block_on(analyze::run_with_options( + path, + analyze::AnalyzeOptions { + config_path: config, + secret_scan: secret_scan::SecretScanOptions { + allow_unredacted_secrets, + confirm_allow_unredacted_secrets, + }, + }, + )) } cli::Command::Serve { path, config } => serve::run(&path, config.as_deref()), } diff --git a/crates/clarion-cli/src/secret_scan.rs b/crates/clarion-cli/src/secret_scan.rs new file mode 100644 index 00000000..0e0e08cb --- /dev/null +++ b/crates/clarion-cli/src/secret_scan.rs @@ -0,0 +1,224 @@ +//! Pre-ingest secret scanning for `clarion analyze`. +//! +//! Exit codes used by this module: +//! - 0: analysis may continue, with or without an explicit secret override. +//! - 1: ordinary hard failure reported through the caller's `anyhow::Result`. +//! - 78 (`EX_CONFIG`): `--allow-unredacted-secrets` was misconfigured or not +//! confirmed; the caller must abort before `BeginRun`. + +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + io::{self, IsTerminal, Write}, + path::{Path, PathBuf}, +}; + +mod findings; + +use anyhow::{Context, Result}; +use clarion_scanner::{Baseline, BaselineError, Detection, Scanner, SuppressionResult}; +pub(crate) use findings::emit_findings; +use findings::{PendingFinding, secret_detected_finding}; +use serde_json::json; + +const SECRET_OVERRIDE_ALLOWED: &str = "CLA-SEC-UNREDACTED-SECRETS-ALLOWED"; +const BASELINE_NO_JUSTIFICATION: &str = "CLA-INFRA-SECRET-BASELINE-NO-JUSTIFICATION"; +const BASELINE_MATCH: &str = "CLA-INFRA-SECRET-BASELINE-MATCH"; +const OVERRIDE_UNCONFIRMED: &str = "CLA-INFRA-SECRET-OVERRIDE-UNCONFIRMED"; +const CONFIRM_TOKEN: &str = "yes-i-understand"; + +#[derive(Debug, Clone, Default)] +pub(crate) struct SecretScanOptions { + pub(crate) allow_unredacted_secrets: bool, + pub(crate) confirm_allow_unredacted_secrets: Option, +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct SecretScanOutcome { + pub(crate) briefing_blocks: BTreeMap, + findings: Vec, + override_files: Vec, +} + +impl SecretScanOutcome { + pub(crate) fn augment_stats(&self, stats: &mut serde_json::Value) { + if self.override_files.is_empty() { + return; + } + stats["secret_override_used"] = json!(true); + stats["secret_override_files_affected"] = json!( + self.override_files + .iter() + .map(|path| path.display().to_string()) + .collect::>() + ); + } +} + +pub(crate) fn pre_ingest( + project_root: &Path, + source_files: &[PathBuf], + options: &SecretScanOptions, +) -> Result { + let scanner = Scanner::new(); + let (baseline, mut findings) = load_baseline_for_scan(project_root)?; + let mut per_file = Vec::new(); + let mut all_allowed = Vec::new(); + let mut baseline_matches = Vec::new(); + + for file in source_files { + let buf = fs::read(file).with_context(|| format!("read {}", file.display()))?; + let detections = scanner.scan_bytes(&buf); + let SuppressionResult { + allowed, + fired_entries, + .. + } = baseline.suppress(detections, file); + if !allowed.is_empty() { + all_allowed.extend( + allowed + .iter() + .cloned() + .map(|detection| (canonical_or_original(file), detection)), + ); + } + baseline_matches.extend(fired_entries.into_iter().map(|entry| PendingFinding { + file_path: entry.file_path.clone(), + rule_id: BASELINE_MATCH, + kind: "fact", + severity: "INFO", + confidence: Some(1.0), + confidence_basis: Some("baseline"), + message: format!( + "Secret baseline entry matched {}:{}", + entry.file_path.display(), + entry.entry.line_number + ), + evidence: json!({ + "file_path": entry.file_path, + "line_number": entry.entry.line_number, + "rule": entry.entry.rule_type, + }), + })); + per_file.push((canonical_or_original(file), allowed)); + } + + let override_confirmed = confirm_override_if_needed(&all_allowed, options); + let mut briefing_blocks = BTreeMap::new(); + let mut override_files = BTreeSet::new(); + findings.extend(baseline_matches); + + for (file, allowed) in per_file { + if allowed.is_empty() { + continue; + } + if override_confirmed { + override_files.insert(file); + } else { + briefing_blocks.insert(file.clone(), "secret_present".to_owned()); + findings.extend( + allowed + .iter() + .map(|detection| secret_detected_finding(&file, detection)), + ); + } + } + + for file in &override_files { + findings.push(PendingFinding { + file_path: file.clone(), + rule_id: SECRET_OVERRIDE_ALLOWED, + kind: "defect", + severity: "ERROR", + confidence: Some(1.0), + confidence_basis: Some("operator_override"), + message: format!( + "Operator allowed unredacted secrets in {}", + display_relative(project_root, file) + ), + evidence: json!({ + "file_path": display_relative(project_root, file), + "override_used": true, + }), + }); + } + + Ok(SecretScanOutcome { + briefing_blocks, + findings, + override_files: override_files.into_iter().collect(), + }) +} + +fn load_baseline_for_scan(project_root: &Path) -> Result<(Baseline, Vec)> { + let path = project_root.join(".clarion/secrets-baseline.yaml"); + match clarion_scanner::load_baseline(&path) { + Ok(baseline) => Ok((baseline, Vec::new())), + Err(BaselineError::MissingJustification { file, line }) => Ok(( + Baseline::empty(), + vec![PendingFinding { + file_path: project_root.join(file.clone()), + rule_id: BASELINE_NO_JUSTIFICATION, + kind: "defect", + severity: "ERROR", + confidence: Some(1.0), + confidence_basis: Some("baseline_schema"), + message: format!( + "Secret baseline entry missing justification at {}:{}", + file.display(), + line + ), + evidence: json!({"file_path": file, "line_number": line}), + }], + )), + Err(err) => Err(err).context("load secret baseline"), + } +} + +fn confirm_override_if_needed( + allowed: &[(PathBuf, Detection)], + options: &SecretScanOptions, +) -> bool { + if allowed.is_empty() || !options.allow_unredacted_secrets { + return false; + } + if options.confirm_allow_unredacted_secrets.as_deref() == Some(CONFIRM_TOKEN) { + return true; + } + if options.confirm_allow_unredacted_secrets.is_some() { + abort_unconfirmed_override(); + } + if io::stdin().is_terminal() { + for (file, detection) in allowed { + eprintln!( + "{}:{} {}", + file.display(), + detection.line_number, + detection.rule_id + ); + } + eprint!("Type '{CONFIRM_TOKEN}' to proceed: "); + let _ = io::stderr().flush(); + let mut input = String::new(); + if io::stdin().read_line(&mut input).is_ok() && input.trim() == CONFIRM_TOKEN { + return true; + } + } + abort_unconfirmed_override(); +} + +fn abort_unconfirmed_override() -> ! { + eprintln!("{OVERRIDE_UNCONFIRMED}: --allow-unredacted-secrets requires confirmation"); + std::process::exit(78); +} + +fn canonical_or_original(path: &Path) -> PathBuf { + path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) +} + +fn display_relative(root: &Path, path: &Path) -> String { + path.strip_prefix(root) + .unwrap_or(path) + .display() + .to_string() +} diff --git a/crates/clarion-cli/src/secret_scan/findings.rs b/crates/clarion-cli/src/secret_scan/findings.rs new file mode 100644 index 00000000..733fabf9 --- /dev/null +++ b/crates/clarion-cli/src/secret_scan/findings.rs @@ -0,0 +1,123 @@ +use std::{ + collections::BTreeMap, + path::{Path, PathBuf}, +}; + +use anyhow::{Context, Result}; +use clarion_scanner::Detection; +use clarion_storage::{ + Writer, + commands::{FindingRecord, WriterCmd}, +}; +use serde_json::json; +use uuid::Uuid; + +use super::SecretScanOutcome; + +const SECRET_DETECTED: &str = "CLA-SEC-SECRET-DETECTED"; + +#[derive(Debug, Clone)] +pub(super) struct PendingFinding { + pub(super) file_path: PathBuf, + pub(super) rule_id: &'static str, + pub(super) kind: &'static str, + pub(super) severity: &'static str, + pub(super) confidence: Option, + pub(super) confidence_basis: Option<&'static str>, + pub(super) message: String, + pub(super) evidence: serde_json::Value, +} + +pub(crate) async fn emit_findings( + writer: &Writer, + run_id: &str, + started_at: &str, + outcome: &SecretScanOutcome, + entity_anchors: &BTreeMap, +) -> Result<()> { + for pending in &outcome.findings { + let entity_id = + finding_entity_id(&pending.file_path, entity_anchors).with_context(|| { + format!("anchor secret finding for {}", pending.file_path.display()) + })?; + let finding_id = Uuid::new_v4().to_string(); + writer + .send_wait(|ack| WriterCmd::InsertFinding { + finding: Box::new(FindingRecord { + id: finding_id.clone(), + tool: "clarion".to_owned(), + tool_version: env!("CARGO_PKG_VERSION").to_owned(), + run_id: run_id.to_owned(), + rule_id: pending.rule_id.to_owned(), + kind: pending.kind.to_owned(), + severity: pending.severity.to_owned(), + confidence: pending.confidence, + confidence_basis: pending.confidence_basis.map(str::to_owned), + entity_id, + related_entities_json: "[]".to_owned(), + message: pending.message.clone(), + evidence_json: pending.evidence.to_string(), + properties_json: "{}".to_owned(), + supports_json: "[]".to_owned(), + supported_by_json: "[]".to_owned(), + created_at: started_at.to_owned(), + updated_at: started_at.to_owned(), + }), + ack, + }) + .await + .map_err(|e| anyhow::anyhow!("{e}")) + .with_context(|| format!("InsertFinding {finding_id}"))?; + } + Ok(()) +} + +pub(super) fn secret_detected_finding(file: &Path, detection: &Detection) -> PendingFinding { + PendingFinding { + file_path: file.to_path_buf(), + rule_id: SECRET_DETECTED, + kind: "defect", + severity: "ERROR", + confidence: if detection.rule_id.starts_with("HighEntropy") { + Some(0.6) + } else { + Some(1.0) + }, + confidence_basis: if detection.rule_id.starts_with("HighEntropy") { + Some("entropy") + } else { + Some("pattern") + }, + message: format!( + "{} detected in {}:{}", + detection.rule_id, + file.display(), + detection.line_number + ), + evidence: json!({ + "file_path": file, + "line_number": detection.line_number, + "rule": detection.rule_id, + "hashed_secret_hex": hex20(detection.hashed_secret), + }), + } +} + +fn finding_entity_id(file_path: &Path, anchors: &BTreeMap) -> Option { + anchors.get(file_path).cloned().or_else(|| { + anchors + .iter() + .find(|(candidate, _)| candidate.ends_with(file_path)) + .map(|(_, entity_id)| entity_id.clone()) + }) +} + +fn hex20(bytes: [u8; 20]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(40); + for byte in bytes { + out.push(char::from(HEX[usize::from(byte >> 4)])); + out.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + out +} diff --git a/crates/clarion-cli/tests/secret_scan.rs b/crates/clarion-cli/tests/secret_scan.rs new file mode 100644 index 00000000..0399b758 --- /dev/null +++ b/crates/clarion-cli/tests/secret_scan.rs @@ -0,0 +1,440 @@ +#![cfg(unix)] + +use assert_cmd::Command; +use rusqlite::Connection; + +fn clarion_bin() -> Command { + Command::cargo_bin("clarion").expect("clarion binary") +} + +const PLUGIN_SCRIPT: &str = r#"#!/usr/bin/python3 +import json +import pathlib +import re +import sys + + +def read_frame(): + headers = {} + while True: + line = sys.stdin.buffer.readline() + if line in (b"", b"\r\n"): + break + name, value = line.decode("ascii").strip().split(":", 1) + headers[name.lower()] = value.strip() + length = int(headers["content-length"]) + return json.loads(sys.stdin.buffer.read(length)) + + +def write_frame(message): + body = json.dumps(message, separators=(",", ":")).encode("utf-8") + sys.stdout.buffer.write(b"Content-Length: " + str(len(body)).encode("ascii") + b"\r\n\r\n") + sys.stdout.buffer.write(body) + sys.stdout.buffer.flush() + + +while True: + msg = read_frame() + method = msg.get("method") + if method == "initialized": + continue + if method == "exit": + raise SystemExit(0) + ident = msg["id"] + if method == "initialize": + write_frame({ + "jsonrpc": "2.0", + "id": ident, + "result": { + "name": "clarion-plugin-secretfixture", + "version": "0.1.0", + "ontology_version": "0.1.0", + "capabilities": {}, + }, + }) + elif method == "analyze_file": + path = msg["params"]["file_path"] + name = "file_" + re.sub(r"[^A-Za-z0-9_]", "_", pathlib.Path(path).name) + write_frame({ + "jsonrpc": "2.0", + "id": ident, + "result": { + "entities": [ + { + "id": "secretfixture:module:" + name, + "kind": "module", + "qualified_name": name, + "source": {"file_path": path}, + } + ], + "edges": [], + }, + }) + elif method == "shutdown": + write_frame({"jsonrpc": "2.0", "id": ident, "result": {}}) + else: + raise SystemExit(1) +"#; + +const PLUGIN_MANIFEST: &str = r#" +[plugin] +name = "clarion-plugin-secretfixture" +plugin_id = "secretfixture" +version = "0.1.0" +protocol_version = "1.0" +executable = "clarion-plugin-secretfixture" +language = "secretfixture" +extensions = ["sec"] + +[capabilities.runtime] +expected_max_rss_mb = 256 +expected_entities_per_file = 100 +wardline_aware = false +reads_outside_project_root = false + +[ontology] +entity_kinds = ["module"] +edge_kinds = [] +rule_id_prefix = "CLA-SECRET-FIXTURE-" +ontology_version = "0.1.0" +"#; + +fn write_secret_fixture_plugin(plugin_dir: &std::path::Path) { + use std::os::unix::fs::PermissionsExt; + + let plugin_script = plugin_dir.join("clarion-plugin-secretfixture"); + std::fs::write(&plugin_script, PLUGIN_SCRIPT).expect("write plugin script"); + let mut perms = std::fs::metadata(&plugin_script) + .expect("stat plugin") + .permissions(); + perms.set_mode(0o755); + std::fs::set_permissions(&plugin_script, perms).expect("chmod plugin"); + + std::fs::write(plugin_dir.join("plugin.toml"), PLUGIN_MANIFEST).expect("write plugin manifest"); +} + +fn install_project(project: &std::path::Path) { + clarion_bin() + .args(["install", "--path"]) + .arg(project) + .assert() + .success(); +} + +fn plugin_path(plugin_dir: &std::path::Path) -> std::ffi::OsString { + std::env::join_paths(std::iter::once(plugin_dir.to_path_buf())).unwrap() +} + +fn conn(project: &std::path::Path) -> Connection { + Connection::open(project.join(".clarion/clarion.db")).expect("open clarion db") +} + +#[test] +fn clean_project_has_no_secret_findings() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write(project.path().join("clean.sec"), b"nothing to see\n").unwrap(); + + clarion_bin() + .arg("analyze") + .arg(project.path()) + .env("PATH", plugin_path(plugin.path())) + .assert() + .success(); + + let count: i64 = conn(project.path()) + .query_row( + "SELECT COUNT(*) FROM findings WHERE rule_id LIKE 'CLA-SEC-%'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, 0); +} + +#[test] +fn secret_file_persists_finding_and_briefing_block() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write( + project.path().join("leaky.sec"), + b"aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'\n", + ) + .unwrap(); + + clarion_bin() + .arg("analyze") + .arg(project.path()) + .env("PATH", plugin_path(plugin.path())) + .assert() + .success(); + + let db = conn(project.path()); + let blocked: String = db + .query_row( + "SELECT json_extract(properties, '$.briefing_blocked') FROM entities", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(blocked, "secret_present"); + let count: i64 = db + .query_row( + "SELECT COUNT(*) FROM findings WHERE rule_id = 'CLA-SEC-SECRET-DETECTED'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, 1); +} + +#[test] +fn baseline_suppresses_secret_and_emits_audit_match() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write( + project.path().join("leaky.sec"), + b"aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'\n", + ) + .unwrap(); + std::fs::write( + project.path().join(".clarion/secrets-baseline.yaml"), + r#" +version: "1.0" +results: + "leaky.sec": + - type: "AWS Access Key" + hashed_secret: "25910f981e85ca04baf359199dd0bd4a3ae738b6" + line_number: 1 + is_secret: false + justification: "AWS documentation example key." +"#, + ) + .unwrap(); + + clarion_bin() + .arg("analyze") + .arg(project.path()) + .env("PATH", plugin_path(plugin.path())) + .assert() + .success(); + + let db = conn(project.path()); + let secret_count: i64 = db + .query_row( + "SELECT COUNT(*) FROM findings WHERE rule_id = 'CLA-SEC-SECRET-DETECTED'", + [], + |row| row.get(0), + ) + .unwrap(); + let match_count: i64 = db + .query_row( + "SELECT COUNT(*) FROM findings WHERE rule_id = 'CLA-INFRA-SECRET-BASELINE-MATCH'", + [], + |row| row.get(0), + ) + .unwrap(); + let blocked_count: i64 = db + .query_row( + "SELECT COUNT(*) FROM entities WHERE json_extract(properties, '$.briefing_blocked') IS NOT NULL", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(secret_count, 0); + assert_eq!(match_count, 1); + assert_eq!(blocked_count, 0); +} + +#[test] +fn missing_baseline_justification_degrades_to_finding() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write( + project.path().join("leaky.sec"), + b"aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'\n", + ) + .unwrap(); + std::fs::write( + project.path().join(".clarion/secrets-baseline.yaml"), + r#" +version: "1.0" +results: + "leaky.sec": + - type: "AWS Access Key" + hashed_secret: "25910f981e85ca04baf359199dd0bd4a3ae738b6" + line_number: 1 + is_secret: false +"#, + ) + .unwrap(); + + clarion_bin() + .arg("analyze") + .arg(project.path()) + .env("PATH", plugin_path(plugin.path())) + .assert() + .success(); + + let count: i64 = conn(project.path()) + .query_row( + "SELECT COUNT(*) FROM findings WHERE rule_id = 'CLA-INFRA-SECRET-BASELINE-NO-JUSTIFICATION'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, 1); +} + +#[test] +fn non_tty_override_confirmed_allows_briefing_and_records_stats() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write( + project.path().join("leaky.sec"), + b"aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'\n", + ) + .unwrap(); + + clarion_bin() + .args([ + "analyze", + "--allow-unredacted-secrets", + "--confirm-allow-unredacted-secrets=yes-i-understand", + ]) + .arg(project.path()) + .env("PATH", plugin_path(plugin.path())) + .assert() + .success(); + + let db = conn(project.path()); + let blocked_count: i64 = db + .query_row( + "SELECT COUNT(*) FROM entities WHERE json_extract(properties, '$.briefing_blocked') IS NOT NULL", + [], + |row| row.get(0), + ) + .unwrap(); + let override_count: i64 = db + .query_row( + "SELECT COUNT(*) FROM findings WHERE rule_id = 'CLA-SEC-UNREDACTED-SECRETS-ALLOWED'", + [], + |row| row.get(0), + ) + .unwrap(); + let override_used: i64 = db + .query_row( + "SELECT json_extract(stats, '$.secret_override_used') FROM runs", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(blocked_count, 0); + assert_eq!(override_count, 1); + assert_eq!(override_used, 1); +} + +#[test] +fn non_tty_override_without_confirmation_exits_78_before_run_start() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write( + project.path().join("leaky.sec"), + b"aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'\n", + ) + .unwrap(); + + let assert = clarion_bin() + .args(["analyze", "--allow-unredacted-secrets"]) + .arg(project.path()) + .env("PATH", plugin_path(plugin.path())) + .assert() + .code(78); + let stderr = String::from_utf8_lossy(&assert.get_output().stderr); + assert!(stderr.contains("CLA-INFRA-SECRET-OVERRIDE-UNCONFIRMED")); + let run_count: i64 = conn(project.path()) + .query_row("SELECT COUNT(*) FROM runs", [], |row| row.get(0)) + .unwrap(); + assert_eq!(run_count, 0); +} + +#[test] +#[ignore = "TTY override confirmation needs an interactive terminal; WS-D owns the manual smoke."] +fn tty_override_confirmation_manual_smoke() {} + +#[test] +fn override_flag_is_noop_without_detections() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write(project.path().join("clean.sec"), b"nothing to see\n").unwrap(); + + clarion_bin() + .args(["analyze", "--allow-unredacted-secrets"]) + .arg(project.path()) + .env("PATH", plugin_path(plugin.path())) + .assert() + .success(); + + let db = conn(project.path()); + let override_count: i64 = db + .query_row( + "SELECT COUNT(*) FROM findings WHERE rule_id = 'CLA-SEC-UNREDACTED-SECRETS-ALLOWED'", + [], + |row| row.get(0), + ) + .unwrap(); + let stats_value: Option = db + .query_row( + "SELECT json_extract(stats, '$.secret_override_used') FROM runs", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(override_count, 0); + assert_eq!(stats_value, None); +} + +#[test] +fn only_secret_bearing_file_is_blocked_in_multi_file_project() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write(project.path().join("clean_a.sec"), b"nothing to see\n").unwrap(); + std::fs::write( + project.path().join("leaky.sec"), + b"aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'\n", + ) + .unwrap(); + std::fs::write(project.path().join("clean_b.sec"), b"still clean\n").unwrap(); + + clarion_bin() + .arg("analyze") + .arg(project.path()) + .env("PATH", plugin_path(plugin.path())) + .assert() + .success(); + + let blocked_count: i64 = conn(project.path()) + .query_row( + "SELECT COUNT(*) FROM entities WHERE json_extract(properties, '$.briefing_blocked') = 'secret_present'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(blocked_count, 1); +} diff --git a/crates/clarion-core/src/plugin/host.rs b/crates/clarion-core/src/plugin/host.rs index d084ebee..2afed830 100644 --- a/crates/clarion-core/src/plugin/host.rs +++ b/crates/clarion-core/src/plugin/host.rs @@ -27,7 +27,7 @@ //! only calls `setrlimit(2)`, which is async-signal-safe per POSIX.1-2017 //! §2.4.3. The `unsafe` block is the minimum required by the `pre_exec` API. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::io::{BufRead, Write}; use std::path::{Path, PathBuf}; @@ -414,6 +414,8 @@ where /// discarded on overflow so the plugin cannot back-pressure the host /// via stderr writes. stderr_tail: Option>>>, + /// Canonical source paths whose entities must not be sent for LLM briefing. + briefing_blocks: BTreeMap, } /// Size of the stderr ring buffer kept for diagnostics. 64 KiB holds @@ -661,6 +663,7 @@ impl PluginHost { terminated: false, ontology_version: None, stderr_tail: None, + briefing_blocks: BTreeMap::new(), } } @@ -688,6 +691,13 @@ impl PluginHost { self.ontology_version.as_deref() } + /// Install file-level briefing blocks produced by the core pre-ingest + /// scanner. Keys are canonical source paths and values are open-vocabulary + /// policy reasons such as `secret_present`. + pub fn set_briefing_blocks(&mut self, blocks: BTreeMap) { + self.briefing_blocks = blocks; + } + /// Perform the `initialize` → `initialized` handshake. /// /// Steps: @@ -824,7 +834,7 @@ impl PluginHost { let mut accepted = Vec::new(); for raw_val in afr.entities { - let raw: RawEntity = match serde_json::from_value(raw_val) { + let mut raw: RawEntity = match serde_json::from_value(raw_val) { Ok(e) => e, Err(e) => { // Drop the entity, but record the serde error so operators @@ -923,6 +933,8 @@ impl PluginHost { return Err(HostError::EntityCapExceeded(e)); } + self.apply_briefing_block(&mut raw, &jailed); + accepted.push(AcceptedEntity { id: expected_id, kind: raw.kind.clone(), @@ -942,6 +954,15 @@ impl PluginHost { }) } + fn apply_briefing_block(&self, raw: &mut RawEntity, jailed: &str) { + if let Some(reason) = self.briefing_blocks.get(Path::new(jailed)) { + raw.extra.insert( + "briefing_blocked".to_owned(), + serde_json::Value::String(reason.clone()), + ); + } + } + /// B.3: per-edge validation pipeline. Mirrors the entity loop's /// drop-on-violation/emit-finding posture but without the kill paths — /// edges do not participate in the path-escape breaker or entity cap diff --git a/crates/clarion-mcp/src/lib.rs b/crates/clarion-mcp/src/lib.rs index d44f06e3..78cfc2bd 100644 --- a/crates/clarion-mcp/src/lib.rs +++ b/crates/clarion-mcp/src/lib.rs @@ -1146,6 +1146,9 @@ impl ServerState { if entity.kind == "subsystem" { return Ok(SummaryRead::ScopeDeferred(Box::new(entity))); } + if let Some(reason) = briefing_block_reason(&entity) { + return Ok(SummaryRead::BriefingBlocked(Box::new(entity), reason)); + } let Some(content_hash) = entity.content_hash.clone() else { return Ok(SummaryRead::MissingContentHash(entity.id)); }; @@ -1473,6 +1476,7 @@ enum SummaryRead { EntityNotFound(String), MissingContentHash(String), ScopeDeferred(Box), + BriefingBlocked(Box, String), } struct SummaryReady { @@ -2274,6 +2278,7 @@ fn summary_read_error(read: SummaryRead) -> Value { false, ), SummaryRead::ScopeDeferred(entity) => summary_scope_deferred(&entity), + SummaryRead::BriefingBlocked(entity, reason) => summary_briefing_blocked(&entity, &reason), SummaryRead::Ready(_) => unreachable!("ready summary read is not an error"), } } @@ -2347,6 +2352,24 @@ fn summary_scope_deferred(entity: &EntityRow) -> Value { })) } +fn summary_briefing_blocked(entity: &EntityRow, reason: &str) -> Value { + success_envelope(json!({ + "available": false, + "entity_id": entity.id, + "entity": entity_json(entity), + "summary": null, + "briefing_blocked": reason, + "remediation": "File flagged by pre-ingest secret scan. Fix the secret or whitelist via .clarion/secrets-baseline.yaml. See ADR-013." + })) +} + +fn briefing_block_reason(entity: &EntityRow) -> Option { + entity_properties_json(entity) + .get("briefing_blocked") + .and_then(Value::as_str) + .map(str::to_owned) +} + fn tool_json_rpc_response(id: &Value, envelope: &Value) -> Value { let is_error = !envelope .get("ok") diff --git a/crates/clarion-mcp/tests/storage_tools.rs b/crates/clarion-mcp/tests/storage_tools.rs index 02519070..43c3dcc5 100644 --- a/crates/clarion-mcp/tests/storage_tools.rs +++ b/crates/clarion-mcp/tests/storage_tools.rs @@ -818,6 +818,62 @@ async fn summary_on_subsystem_returns_policy_envelope_without_llm_call() { handle.await.unwrap().unwrap(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn summary_on_secret_blocked_entity_returns_policy_envelope_without_llm_or_cache() { + let (project, db_path) = open_project(); + let conn = Connection::open(&db_path).expect("open sqlite"); + conn.execute( + "UPDATE entities SET properties = ?1 WHERE id = 'python:function:demo.entry'", + params![json!({"briefing_blocked": "secret_present"}).to_string()], + ) + .expect("mark entity blocked"); + drop(conn); + + let (writer, handle) = Writer::spawn(db_path.clone(), 50, 256).unwrap(); + let provider = Arc::new(AnySummaryProvider::new_output( + r#"{"purpose":"should not run"}"#, + 120, + 0.012, + )); + let state = state_for_summary( + project.path(), + &db_path, + &writer, + provider.clone(), + llm_config(), + ); + + let envelope = call_tool( + &state, + "summary", + json!({"id": "python:function:demo.entry"}), + ) + .await; + + assert_eq!(envelope["ok"], true); + assert_eq!(envelope["result"]["summary"], Value::Null); + assert_eq!(envelope["result"]["briefing_blocked"], "secret_present"); + assert_eq!(envelope["stats_delta"], json!({})); + assert!(provider.invocations().is_empty()); + + let entity_at = call_tool(&state, "entity_at", json!({"file": "demo.py", "line": 1})).await; + assert_eq!(entity_at["ok"], true); + + drop(state); + drop(writer); + handle.await.unwrap().unwrap(); + + let conn = Connection::open(&db_path).expect("open sqlite"); + let cache_rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM summary_cache WHERE entity_id = 'python:function:demo.entry'", + [], + |row| row.get(0), + ) + .expect("query summary cache"); + assert_eq!(cache_rows, 0); +} + #[tokio::test] async fn issues_for_returns_unavailable_when_filigree_disabled() { let (project, db_path) = open_project(); diff --git a/crates/clarion-scanner/Cargo.toml b/crates/clarion-scanner/Cargo.toml new file mode 100644 index 00000000..d64555d6 --- /dev/null +++ b/crates/clarion-scanner/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "clarion-scanner" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[lints] +workspace = true + +[dependencies] +regex.workspace = true +serde.workspace = true +serde_norway.workspace = true +sha1.workspace = true +thiserror.workspace = true + +[dev-dependencies] +tempfile.workspace = true diff --git a/crates/clarion-scanner/src/baseline.rs b/crates/clarion-scanner/src/baseline.rs new file mode 100644 index 00000000..ef1ea383 --- /dev/null +++ b/crates/clarion-scanner/src/baseline.rs @@ -0,0 +1,225 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + fs, + path::{Path, PathBuf}, +}; + +use serde::{Deserialize, Serialize}; + +use crate::{Detection, hex_decode_20}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Baseline { + version: String, + entries: BTreeMap>, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BaselineEntry { + pub rule_type: String, + pub hashed_secret: [u8; 20], + pub line_number: u32, + pub is_secret: bool, + pub justification: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BaselineMatch { + pub file_path: PathBuf, + pub entry: BaselineEntry, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SuppressionResult { + pub allowed: Vec, + pub suppressed: Vec, + pub fired_entries: Vec, +} + +#[derive(Debug, thiserror::Error)] +pub enum BaselineError { + #[error("baseline version mismatch: expected 1.0, got {0}")] + UnsupportedVersion(String), + #[error("baseline entry missing required field 'justification' at {file}:{line}")] + MissingJustification { file: PathBuf, line: u32 }, + #[error("baseline entry has invalid hashed_secret at {file}:{line}: {details}")] + InvalidHash { + file: PathBuf, + line: u32, + details: String, + }, + #[error("baseline parse error: {0}")] + Parse(#[from] serde_norway::Error), + #[error("baseline I/O error: {0}")] + Io(#[from] std::io::Error), +} + +pub fn load_baseline(path: &Path) -> Result { + match fs::read_to_string(path) { + Ok(raw) => Baseline::from_yaml_str(&raw), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(Baseline::empty()), + Err(err) => Err(BaselineError::Io(err)), + } +} + +impl Baseline { + #[must_use] + pub fn empty() -> Self { + Self { + version: "1.0".to_owned(), + entries: BTreeMap::new(), + } + } + + pub fn from_yaml_str(raw: &str) -> Result { + let parsed: RawBaseline = serde_norway::from_str(raw)?; + Self::from_raw(parsed) + } + + pub fn to_yaml_string(&self) -> Result { + let raw = RawBaseline::from(self); + serde_norway::to_string(&raw).map_err(BaselineError::Parse) + } + + #[must_use] + pub fn entries(&self) -> &BTreeMap> { + &self.entries + } + + #[must_use] + pub fn suppress(&self, detections: Vec, file: &Path) -> SuppressionResult { + let entries = self.entries_for(file); + let mut allowed = Vec::new(); + let mut suppressed = Vec::new(); + let mut fired_entries = Vec::new(); + let mut fired_keys = BTreeSet::new(); + + 'detections: for detection in detections { + for (baseline_path, entry) in &entries { + if entry.is_secret { + continue; + } + if entry.hashed_secret == detection.hashed_secret + && entry.line_number == detection.line_number + { + let key = ( + (*baseline_path).clone(), + entry.hashed_secret, + entry.line_number, + ); + if fired_keys.insert(key) { + fired_entries.push(BaselineMatch { + file_path: (*baseline_path).clone(), + entry: (*entry).clone(), + }); + } + suppressed.push(detection); + continue 'detections; + } + } + allowed.push(detection); + } + + SuppressionResult { + allowed, + suppressed, + fired_entries, + } + } + + fn from_raw(raw: RawBaseline) -> Result { + if raw.version != "1.0" { + return Err(BaselineError::UnsupportedVersion(raw.version)); + } + let mut entries = BTreeMap::new(); + for (file, raw_entries) in raw.results { + let mut converted = Vec::new(); + for entry in raw_entries { + let justification = entry.justification.unwrap_or_default(); + if justification.trim().is_empty() { + return Err(BaselineError::MissingJustification { + file, + line: entry.line_number, + }); + } + let hashed_secret = hex_decode_20(&entry.hashed_secret).map_err(|details| { + BaselineError::InvalidHash { + file: file.clone(), + line: entry.line_number, + details, + } + })?; + converted.push(BaselineEntry { + rule_type: entry.rule_type, + hashed_secret, + line_number: entry.line_number, + is_secret: entry.is_secret, + justification, + }); + } + entries.insert(file, converted); + } + Ok(Self { + version: raw.version, + entries, + }) + } + + fn entries_for(&self, file: &Path) -> Vec<(&PathBuf, &BaselineEntry)> { + self.entries + .iter() + .filter(|(candidate, _)| baseline_path_matches(file, candidate)) + .flat_map(|(path, entries)| entries.iter().map(move |entry| (path, entry))) + .collect() + } +} + +#[derive(Debug, Serialize, Deserialize)] +struct RawBaseline { + version: String, + #[serde(default)] + results: BTreeMap>, +} + +#[derive(Debug, Serialize, Deserialize)] +struct RawBaselineEntry { + #[serde(rename = "type")] + rule_type: String, + hashed_secret: String, + line_number: u32, + #[serde(default)] + is_secret: bool, + justification: Option, +} + +impl From<&Baseline> for RawBaseline { + fn from(value: &Baseline) -> Self { + let results = value + .entries + .iter() + .map(|(path, entries)| { + ( + path.clone(), + entries + .iter() + .map(|entry| RawBaselineEntry { + rule_type: entry.rule_type.clone(), + hashed_secret: crate::hex_encode(&entry.hashed_secret), + line_number: entry.line_number, + is_secret: entry.is_secret, + justification: Some(entry.justification.clone()), + }) + .collect(), + ) + }) + .collect(); + Self { + version: value.version.clone(), + results, + } + } +} + +fn baseline_path_matches(file: &Path, baseline_path: &Path) -> bool { + file == baseline_path || file.ends_with(baseline_path) +} diff --git a/crates/clarion-scanner/src/entropy.rs b/crates/clarion-scanner/src/entropy.rs new file mode 100644 index 00000000..0c52d766 --- /dev/null +++ b/crates/clarion-scanner/src/entropy.rs @@ -0,0 +1,60 @@ +use std::collections::BTreeMap; + +/// Entropy threshold and minimum candidate length for one alphabet. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct EntropyTuning { + pub min_len: usize, + pub min_entropy: f64, +} + +impl EntropyTuning { + pub const BASE64: Self = Self { + min_len: 20, + min_entropy: 4.5, + }; + pub const HEX: Self = Self { + min_len: 40, + min_entropy: 3.0, + }; + + pub(crate) fn accepts(self, candidate: &[u8]) -> bool { + candidate.len() >= self.min_len && shannon_entropy(candidate) >= self.min_entropy + } +} + +pub(crate) fn shannon_entropy(bytes: &[u8]) -> f64 { + if bytes.is_empty() { + return 0.0; + } + let mut counts = BTreeMap::::new(); + for byte in bytes { + *counts.entry(*byte).or_default() += 1; + } + let len = usize_to_f64(bytes.len()); + counts + .values() + .map(|count| { + let p = usize_to_f64(*count) / len; + -p * p.log2() + }) + .sum() +} + +fn usize_to_f64(value: usize) -> f64 { + f64::from(u32::try_from(value).unwrap_or(u32::MAX)) +} + +#[cfg(test)] +mod tests { + use super::shannon_entropy; + + #[test] + fn entropy_is_low_for_repetition() { + assert!(shannon_entropy(b"aaaaaaaaaaaaaaaaaaaaaaaa") < 0.1); + } + + #[test] + fn entropy_is_higher_for_varied_alphabet() { + assert!(shannon_entropy(b"0123456789abcdefABCDEF+/") > 4.0); + } +} diff --git a/crates/clarion-scanner/src/lib.rs b/crates/clarion-scanner/src/lib.rs new file mode 100644 index 00000000..abcb6924 --- /dev/null +++ b/crates/clarion-scanner/src/lib.rs @@ -0,0 +1,92 @@ +//! Core-owned pre-ingest secret scanner. +//! +//! The scanner intentionally stores only positions, rule identifiers, and a +//! detect-secrets-compatible SHA-1 digest of the matched bytes. Literal secret +//! values do not leave the scanning call. + +mod baseline; +mod entropy; +mod patterns; + +pub use baseline::{ + Baseline, BaselineEntry, BaselineError, BaselineMatch, SuppressionResult, load_baseline, +}; +pub use entropy::EntropyTuning; +pub use patterns::{PatternMeta, Scanner}; + +/// One secret-like match in one file buffer. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Detection { + pub rule_id: &'static str, + pub category: SecretCategory, + pub byte_offset: usize, + pub line_number: u32, + pub matched_len: usize, + pub hashed_secret: [u8; 20], +} + +/// High-level category used for evidence and future operator grouping. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SecretCategory { + CloudCredential, + VcsCredential, + AiProviderCredential, + PaymentsCredential, + MessagingCredential, + PrivateKey, + JwtToken, + HighEntropy, + ContextualCredential, +} + +fn sha1_digest(bytes: &[u8]) -> [u8; 20] { + use sha1::{Digest, Sha1}; + + let mut hasher = Sha1::new(); + hasher.update(bytes); + hasher.finalize().into() +} + +fn line_number_for_offset(buf: &[u8], offset: usize) -> u32 { + let line = buf + .get(..offset.min(buf.len())) + .unwrap_or(buf) + .iter() + .fold(0usize, |count, byte| count + usize::from(*byte == b'\n')) + + 1; + u32::try_from(line).unwrap_or(u32::MAX) +} + +fn hex_encode(bytes: &[u8; 20]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::with_capacity(bytes.len() * 2); + for byte in bytes { + out.push(char::from(HEX[usize::from(byte >> 4)])); + out.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + out +} + +fn hex_decode_20(input: &str) -> Result<[u8; 20], String> { + let raw = input.trim(); + if raw.len() != 40 { + return Err(format!("expected 40 hex characters, got {}", raw.len())); + } + let mut out = [0u8; 20]; + for (idx, chunk) in raw.as_bytes().chunks_exact(2).enumerate() { + let hi = hex_value(chunk[0]).ok_or_else(|| format!("invalid hex at byte {}", idx * 2))?; + let lo = + hex_value(chunk[1]).ok_or_else(|| format!("invalid hex at byte {}", idx * 2 + 1))?; + out[idx] = (hi << 4) | lo; + } + Ok(out) +} + +fn hex_value(byte: u8) -> Option { + match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + b'A'..=b'F' => Some(byte - b'A' + 10), + _ => None, + } +} diff --git a/crates/clarion-scanner/src/patterns.rs b/crates/clarion-scanner/src/patterns.rs new file mode 100644 index 00000000..9e802cd0 --- /dev/null +++ b/crates/clarion-scanner/src/patterns.rs @@ -0,0 +1,300 @@ +use regex::{Regex, RegexSet}; + +use crate::{ + Detection, SecretCategory, entropy::EntropyTuning, line_number_for_offset, sha1_digest, +}; + +/// Metadata for one named secret detector. +#[derive(Debug, Clone)] +pub struct PatternMeta { + pub rule_id: &'static str, + pub detect_secrets_type: &'static str, + pub category: SecretCategory, + pub pattern: &'static str, + capture_group: Option, +} + +#[derive(Debug)] +struct CompiledPattern { + meta: PatternMeta, + regex: Regex, +} + +/// Rust-native port of the ADR-013 v0.1 secret rule floor. +#[derive(Debug)] +pub struct Scanner { + patterns: RegexSet, + pattern_meta: Vec, + compiled_patterns: Vec, + entropy_b64: EntropyTuning, + entropy_hex: EntropyTuning, + entropy_b64_re: Regex, + entropy_hex_re: Regex, +} + +impl Default for Scanner { + fn default() -> Self { + Self::new() + } +} + +impl Scanner { + /// Build the default ADR-013 scanner. + /// + /// # Panics + /// + /// Panics only if one of the compiled-in regular expressions is invalid. + #[must_use] + pub fn new() -> Self { + let pattern_meta = default_pattern_meta(); + let patterns = RegexSet::new(pattern_meta.iter().map(|meta| meta.pattern)) + .expect("default secret patterns compile"); + let compiled_patterns = pattern_meta + .iter() + .cloned() + .map(|meta| CompiledPattern { + regex: Regex::new(meta.pattern).expect("default secret pattern compiles"), + meta, + }) + .collect(); + Self { + patterns, + pattern_meta, + compiled_patterns, + entropy_b64: EntropyTuning::BASE64, + entropy_hex: EntropyTuning::HEX, + entropy_b64_re: Regex::new(r"\b[A-Za-z0-9+/]{20,}={0,2}\b") + .expect("base64 candidate regex compiles"), + entropy_hex_re: Regex::new(r"\b[a-fA-F0-9]{40,}\b") + .expect("hex candidate regex compiles"), + } + } + + #[must_use] + pub fn pattern_meta(&self) -> &[PatternMeta] { + &self.pattern_meta + } + + #[must_use] + pub fn scan_bytes(&self, buf: &[u8]) -> Vec { + let source = String::from_utf8_lossy(buf); + let bytes = source.as_bytes(); + let _set_matches = self.patterns.matches(&source); + let mut detections = Vec::new(); + + for compiled in &self.compiled_patterns { + for captures in compiled.regex.captures_iter(&source) { + let Some(whole_match) = captures.get(0) else { + continue; + }; + if compiled.meta.category == SecretCategory::ContextualCredential + && line_is_comment(bytes, whole_match.start()) + { + continue; + } + let Some(secret_match) = compiled + .meta + .capture_group + .and_then(|group| captures.get(group)) + .or(Some(whole_match)) + else { + continue; + }; + detections.push(detection_from_match( + &compiled.meta, + bytes, + secret_match.start(), + secret_match.end(), + )); + } + } + + let named_ranges = detections + .iter() + .map(|detection| { + ( + detection.byte_offset, + detection.byte_offset + detection.matched_len, + ) + }) + .collect::>(); + self.scan_entropy(bytes, &named_ranges, &mut detections); + + detections.sort_by_key(|d| (d.byte_offset, d.rule_id)); + detections + } + + fn scan_entropy( + &self, + bytes: &[u8], + named_ranges: &[(usize, usize)], + detections: &mut Vec, + ) { + let source = String::from_utf8_lossy(bytes); + for candidate in self.entropy_b64_re.find_iter(&source) { + let candidate_bytes = &source.as_bytes()[candidate.start()..candidate.end()]; + if looks_like_sha256_base64(&source, candidate.start(), candidate.end()) { + continue; + } + if !range_overlaps(candidate.start(), candidate.end(), named_ranges) + && self.entropy_b64.accepts(candidate_bytes) + { + detections.push(entropy_detection( + "HighEntropyBase64", + bytes, + candidate.start(), + candidate.end(), + )); + } + } + for candidate in self.entropy_hex_re.find_iter(&source) { + let candidate_bytes = &source.as_bytes()[candidate.start()..candidate.end()]; + if !range_overlaps(candidate.start(), candidate.end(), named_ranges) + && self.entropy_hex.accepts(candidate_bytes) + { + detections.push(entropy_detection( + "HighEntropyHex", + bytes, + candidate.start(), + candidate.end(), + )); + } + } + } +} + +fn detection_from_match(meta: &PatternMeta, bytes: &[u8], start: usize, end: usize) -> Detection { + let matched = &bytes[start..end]; + Detection { + rule_id: meta.rule_id, + category: meta.category, + byte_offset: start, + line_number: line_number_for_offset(bytes, start), + matched_len: end.saturating_sub(start), + hashed_secret: sha1_digest(matched), + } +} + +fn entropy_detection(rule_id: &'static str, bytes: &[u8], start: usize, end: usize) -> Detection { + Detection { + rule_id, + category: SecretCategory::HighEntropy, + byte_offset: start, + line_number: line_number_for_offset(bytes, start), + matched_len: end.saturating_sub(start), + hashed_secret: sha1_digest(&bytes[start..end]), + } +} + +fn default_pattern_meta() -> Vec { + vec![ + PatternMeta { + rule_id: "AwsAccessKeyId", + detect_secrets_type: "AWS Access Key", + category: SecretCategory::CloudCredential, + pattern: r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b", + capture_group: None, + }, + PatternMeta { + rule_id: "AwsSecretAccessKey", + detect_secrets_type: "AWS Secret Access Key", + category: SecretCategory::CloudCredential, + pattern: r#"(?i)\baws[^:=\n]{0,32}(?:secret|access)[^:=\n]{0,32}(?:=|:|:=)\s*["']?([A-Za-z0-9/+=]{40})["']?"#, + capture_group: Some(1), + }, + PatternMeta { + rule_id: "GitHubPat", + detect_secrets_type: "GitHub Token", + category: SecretCategory::VcsCredential, + pattern: r"\bghp_[A-Za-z0-9]{36}\b", + capture_group: None, + }, + PatternMeta { + rule_id: "GitHubFineGrainedPat", + detect_secrets_type: "GitHub Fine-Grained Token", + category: SecretCategory::VcsCredential, + pattern: r"\bgithub_pat_[A-Za-z0-9_]{82,}\b", + capture_group: None, + }, + PatternMeta { + rule_id: "GitHubOAuth", + detect_secrets_type: "GitHub OAuth Token", + category: SecretCategory::VcsCredential, + pattern: r"\bgh[ousr]_[A-Za-z0-9]{36}\b", + capture_group: None, + }, + PatternMeta { + rule_id: "AnthropicApiKey", + detect_secrets_type: "Anthropic API Key", + category: SecretCategory::AiProviderCredential, + pattern: r"\bsk-ant-[A-Za-z0-9_-]{90,}\b", + capture_group: None, + }, + PatternMeta { + rule_id: "OpenAiApiKey", + detect_secrets_type: "OpenAI API Key", + category: SecretCategory::AiProviderCredential, + pattern: r"\bsk-[A-Za-z0-9]{48}\b", + capture_group: None, + }, + PatternMeta { + rule_id: "StripeApiKey", + detect_secrets_type: "Stripe API Key", + category: SecretCategory::PaymentsCredential, + pattern: r"\b(?:sk|pk|rk)_(?:live|test)_[A-Za-z0-9]{16,}\b", + capture_group: None, + }, + PatternMeta { + rule_id: "SlackToken", + detect_secrets_type: "Slack Token", + category: SecretCategory::MessagingCredential, + pattern: r"\bxox[baprs]-[A-Za-z0-9-]{10,}\b", + capture_group: None, + }, + PatternMeta { + rule_id: "JwtToken", + detect_secrets_type: "JWT Token", + category: SecretCategory::JwtToken, + pattern: r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b", + capture_group: None, + }, + PatternMeta { + rule_id: "PrivateKeyHeader", + detect_secrets_type: "Private Key", + category: SecretCategory::PrivateKey, + pattern: r"-----BEGIN (?:RSA|EC|DSA|OPENSSH|PGP|ENCRYPTED) PRIVATE KEY-----", + capture_group: None, + }, + PatternMeta { + rule_id: "ContextualCredential", + detect_secrets_type: "Keyword Detector", + category: SecretCategory::ContextualCredential, + pattern: r#"(?i)(?:^|[^A-Za-z0-9_-])(?:password|passwd|secret[_-]?token|secret|token|api[_-]?key)\s*(?:=|:=|:)\s*["']([^"'\s]{8,})["']"#, + capture_group: Some(1), + }, + ] +} + +fn range_overlaps(start: usize, end: usize, ranges: &[(usize, usize)]) -> bool { + ranges + .iter() + .any(|(range_start, range_end)| start < *range_end && end > *range_start) +} + +fn line_is_comment(bytes: &[u8], offset: usize) -> bool { + let line_start = bytes + .get(..offset.min(bytes.len())) + .and_then(|prefix| prefix.iter().rposition(|byte| *byte == b'\n')) + .map_or(0, |pos| pos + 1); + bytes[line_start..offset.min(bytes.len())] + .iter() + .copied() + .find(|byte| !byte.is_ascii_whitespace()) + == Some(b'#') +} + +fn looks_like_sha256_base64(source: &str, start: usize, end: usize) -> bool { + let candidate = &source[start..end]; + (candidate.len() == 44 && candidate.ends_with('=')) + || (candidate.len() == 43 && source.as_bytes().get(end) == Some(&b'=')) +} diff --git a/crates/clarion-scanner/tests/scanner.rs b/crates/clarion-scanner/tests/scanner.rs new file mode 100644 index 00000000..c031728a --- /dev/null +++ b/crates/clarion-scanner/tests/scanner.rs @@ -0,0 +1,232 @@ +use clarion_scanner::{Baseline, BaselineError, Scanner, load_baseline}; + +fn rules_for(input: &str) -> Vec<&'static str> { + Scanner::new() + .scan_bytes(input.as_bytes()) + .into_iter() + .map(|detection| detection.rule_id) + .collect() +} + +fn assert_detects(input: &str, rule_id: &str) { + let rules = rules_for(input); + assert!(rules.contains(&rule_id), "{rule_id} not found in {rules:?}"); +} + +fn assert_not_detects(input: &str, rule_id: &str) { + let rules = rules_for(input); + assert!( + !rules.contains(&rule_id), + "{rule_id} unexpectedly found in {rules:?}" + ); +} + +#[test] +fn named_patterns_detect_expected_credentials() { + assert_detects( + "aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'", + "AwsAccessKeyId", + ); + assert_detects( + "aws_secret_access_key = 'wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY'", + "AwsSecretAccessKey", + ); + assert_detects( + "token = 'ghp_abcdefghijklmnopqrstuvwxyzABCDEFGHIJ'", + "GitHubPat", + ); + assert_detects( + &format!("token = 'github_pat_{}'", "a".repeat(82)), + "GitHubFineGrainedPat", + ); + assert_detects( + "token = 'gho_abcdefghijklmnopqrstuvwxyzABCDEFGHIJ'", + "GitHubOAuth", + ); + assert_detects( + &format!("key = 'sk-ant-{}'", "A".repeat(90)), + "AnthropicApiKey", + ); + assert_detects(&format!("key = 'sk-{}'", "A".repeat(48)), "OpenAiApiKey"); + assert_detects("stripe = 'sk_live_abcdefghijklmnop'", "StripeApiKey"); + assert_detects("slack = 'xoxb-123456789012-abcdefghi'", "SlackToken"); + assert_detects( + "jwt = 'eyJabcdef12345.eyJabcdef67890.eyJabcdef99999'", + "JwtToken", + ); + assert_detects("-----BEGIN RSA PRIVATE KEY-----", "PrivateKeyHeader"); + assert_detects("-----BEGIN OPENSSH PRIVATE KEY-----", "PrivateKeyHeader"); +} + +#[test] +fn named_patterns_ignore_near_misses() { + assert_not_detects("AKIAIOSFODNN7EXAMPL", "AwsAccessKeyId"); + assert_not_detects("ghp_short", "GitHubPat"); + assert_not_detects("sk-not-long-enough", "OpenAiApiKey"); + assert_not_detects("-----BEGIN PUBLIC KEY-----", "PrivateKeyHeader"); +} + +#[test] +fn entropy_detection_has_expected_bounds() { + assert_detects( + "secret = AbCdEfGhIjKlMnOpQrStUvWxYz123456+/", + "HighEntropyBase64", + ); + assert_detects( + "digest = 0123456789abcdefABCDEF0123456789abcdefABCDEF0123456789abcdef", + "HighEntropyHex", + ); + assert_not_detects( + "uuid = 123e4567-e89b-12d3-a456-426614174000", + "HighEntropyHex", + ); + assert_not_detects( + "checksum = 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", + "HighEntropyBase64", + ); +} + +#[test] +fn contextual_credentials_detect_assignments_but_not_comments_or_hashes() { + assert_detects( + "password = \"correct-horse-battery-staple\"", + "ContextualCredential", + ); + assert_detects("api_key: \"abcDEF1234567890\"", "ContextualCredential"); + assert_detects( + "SECRET_TOKEN := \"abcDEF1234567890\"", + "ContextualCredential", + ); + assert_not_detects( + "password_hash = \"abcDEF1234567890\"", + "ContextualCredential", + ); + assert_not_detects("# password = \"abcDEF1234567890\"", "ContextualCredential"); +} + +#[test] +fn detection_records_line_and_sha1_hash_without_literal() { + let detections = Scanner::new().scan_bytes(b"clean\nkey = 'AKIAIOSFODNN7EXAMPLE'\n"); + let detection = detections + .iter() + .find(|detection| detection.rule_id == "AwsAccessKeyId") + .expect("AWS key detection"); + assert_eq!(detection.line_number, 2); + assert_eq!(detection.matched_len, "AKIAIOSFODNN7EXAMPLE".len()); + assert_ne!(detection.hashed_secret, [0u8; 20]); +} + +#[test] +fn baseline_suppresses_matching_detection_and_reports_fired_entry() { + let scanner = Scanner::new(); + let detections = scanner.scan_bytes(b"key = 'AKIAIOSFODNN7EXAMPLE'\n"); + let detection = detections + .iter() + .find(|detection| detection.rule_id == "AwsAccessKeyId") + .expect("AWS detection") + .clone(); + let baseline = Baseline::from_yaml_str(&format!( + r#" +version: "1.0" +results: + "src/demo.py": + - type: "AWS Access Key" + hashed_secret: "{}" + line_number: 1 + is_secret: false + justification: "Documented public AWS example key." +"#, + hex20(detection.hashed_secret) + )) + .expect("baseline parses"); + + let result = baseline.suppress(detections, std::path::Path::new("/repo/src/demo.py")); + assert!(result.allowed.is_empty()); + assert_eq!(result.suppressed.len(), 1); + assert_eq!(result.fired_entries.len(), 1); +} + +#[test] +fn baseline_is_secret_true_does_not_suppress() { + let scanner = Scanner::new(); + let detections = scanner.scan_bytes(b"key = 'AKIAIOSFODNN7EXAMPLE'\n"); + let detection = detections + .iter() + .find(|detection| detection.rule_id == "AwsAccessKeyId") + .expect("AWS detection") + .clone(); + let baseline = Baseline::from_yaml_str(&format!( + r#" +version: "1.0" +results: + "demo.py": + - type: "AWS Access Key" + hashed_secret: "{}" + line_number: 1 + is_secret: true + justification: "Still a real secret." +"#, + hex20(detection.hashed_secret) + )) + .expect("baseline parses"); + + let result = baseline.suppress(detections, std::path::Path::new("demo.py")); + assert_eq!(result.allowed.len(), 1); + assert!(result.suppressed.is_empty()); +} + +#[test] +fn baseline_missing_justification_errors() { + let err = Baseline::from_yaml_str( + r#" +version: "1.0" +results: + "src/demo.py": + - type: "AWS Access Key" + hashed_secret: "0123456789abcdef0123456789abcdef01234567" + line_number: 42 + is_secret: false +"#, + ) + .expect_err("missing justification should fail"); + assert!(matches!( + err, + BaselineError::MissingJustification { line: 42, .. } + )); +} + +#[test] +fn absent_baseline_file_is_empty() { + let dir = tempfile::tempdir().expect("tempdir"); + let baseline = load_baseline(&dir.path().join(".clarion/secrets-baseline.yaml")) + .expect("missing baseline is accepted"); + assert!(baseline.entries().is_empty()); +} + +#[test] +fn baseline_round_trips_through_yaml() { + let raw = r#" +version: "1.0" +results: + "src/demo.py": + - type: "AWS Access Key" + hashed_secret: "0123456789abcdef0123456789abcdef01234567" + line_number: 42 + is_secret: false + justification: "Example key." +"#; + let parsed = Baseline::from_yaml_str(raw).expect("parse baseline"); + let rendered = parsed.to_yaml_string().expect("serialize baseline"); + let reparsed = Baseline::from_yaml_str(&rendered).expect("reparse baseline"); + assert_eq!(parsed, reparsed); +} + +fn hex20(bytes: [u8; 20]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut out = String::new(); + for byte in bytes { + out.push(char::from(HEX[usize::from(byte >> 4)])); + out.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + out +} diff --git a/docs/clarion/v0.1/detailed-design.md b/docs/clarion/v0.1/detailed-design.md index 4fb249eb..81580f4e 100644 --- a/docs/clarion/v0.1/detailed-design.md +++ b/docs/clarion/v0.1/detailed-design.md @@ -1050,6 +1050,18 @@ These rules combine signals Clarion uniquely holds — clusters from Phase 3, Wa **`CLA-FACT-TIER-SUBSYSTEM-MIXING` threshold**: default `min_outlier_count: 2` and `min_outlier_fraction: 0.1` (i.e., at least 2 outliers AND at least 10% of subsystem members). Configurable via `clarion.yaml:analysis.clustering.tier_mixing_thresholds`. Tuned to avoid flagging subsystems where a single outlier is legitimate boundary-infrastructure (e.g., an `EXTERNAL_RAW` parser adjacent to an otherwise-`INTEGRAL` validation subsystem). +### Pre-ingest secret-scanning findings (WP5) + +These core-emitted rules are the ADR-013 audit surface. ADR-013 remains canonical for scanner behaviour; this catalogue records the emitted finding IDs. + +| Rule | Severity | Category | Description | Remediation | ADR | +|---|---|---|---|---|---| +| `CLA-SEC-SECRET-DETECTED` | ERROR | security | Pre-ingest secret scanner detected a credential pattern in a file slated for LLM dispatch. | Remove the secret, rotate the credential, or whitelist via `.clarion/secrets-baseline.yaml` with a justification. | [ADR-013](../adr/ADR-013-pre-ingest-secret-scanner.md) | +| `CLA-SEC-UNREDACTED-SECRETS-ALLOWED` | ERROR | security | Operator invoked `--allow-unredacted-secrets`; file content reached the LLM provider with secrets intact. | Audit override usage via `filigree list --rule-id=CLA-SEC-UNREDACTED-SECRETS-ALLOWED --since 30d`. | [ADR-013](../adr/ADR-013-pre-ingest-secret-scanner.md) | +| `CLA-INFRA-SECRET-BASELINE-NO-JUSTIFICATION` | ERROR | infra | Baseline entry missing required `justification` field; entry not honoured. | Add a `justification` string explaining why the match is safe. | [ADR-013](../adr/ADR-013-pre-ingest-secret-scanner.md) | +| `CLA-INFRA-SECRET-BASELINE-MATCH` | INFO | infra | Baseline entry suppressed a scanner detection as an audit event. | None; informational and retained for `NFR-SEC-04` audit. | [ADR-013](../adr/ADR-013-pre-ingest-secret-scanner.md) | +| `CLA-INFRA-SECRET-OVERRIDE-UNCONFIRMED` | ERROR | infra | `--allow-unredacted-secrets` supplied without confirmation; run aborted before start. | Supply `--confirm-allow-unredacted-secrets=yes-i-understand` in non-TTY contexts or run interactively. | [ADR-013](../adr/ADR-013-pre-ingest-secret-scanner.md) | + ### Entity-set diff (deletion detection) At Phase 7, Clarion compares the current run's entity IDs against the prior run's set (read from the prior `run_id`'s stats; if no prior run exists, this phase is a no-op). For each entity ID present before and absent now: @@ -1380,6 +1392,9 @@ Every security-relevant event emits a finding: - `CLA-SEC-SECRET-DETECTED` — unredacted secret blocked LLM dispatch - `CLA-SEC-UNREDACTED-SECRETS-ALLOWED` — operator overrode block with `--allow-unredacted-secrets` +- `CLA-INFRA-SECRET-BASELINE-NO-JUSTIFICATION` — baseline entry lacks the required review rationale +- `CLA-INFRA-SECRET-BASELINE-MATCH` — baseline entry suppressed a scanner hit +- `CLA-INFRA-SECRET-OVERRIDE-UNCONFIRMED` — override flag was not confirmed, so the run aborted before start - `CLA-INFRA-TOKEN-STORAGE-DEGRADED` — OS keychain unavailable, fell back to file-mode `0600` - `CLA-INFRA-BRIEFING-INVALID` — LLM returned schema-invalid content twice, possible injection - `CLA-SEC-VOCABULARY-CANDIDATE-NOVEL` — novel `patterns`/`antipatterns` tag proposed by LLM (light signal; mostly harmless) diff --git a/docs/implementation/v0.1-publish/ws-a-secret-scanner.md b/docs/implementation/v0.1-publish/ws-a-secret-scanner.md new file mode 100644 index 00000000..5f82376a --- /dev/null +++ b/docs/implementation/v0.1-publish/ws-a-secret-scanner.md @@ -0,0 +1,1023 @@ +# Workstream A — WP5 Pre-Ingest Secret Scanner (detailed package) + +**Status**: DRAFT — not yet seeded into Filigree; awaiting kickoff. +**Predecessor**: [`thread-1-pre-publish-blockers.md` §2](./thread-1-pre-publish-blockers.md#2-workstream-a--wp5-pre-ingest-secret-scanner) +**Successor**: Workstream D — external-operator smoke test (publish gate). +**Spec source**: [ADR-013 — Pre-Ingest Secret Scanner with LLM-Dispatch Block](../../clarion/adr/ADR-013-pre-ingest-secret-scanner.md) (Accepted; do not re-litigate). +**Anchoring ADRs**: [ADR-007 (cache key)](../../clarion/adr/ADR-007-summary-cache-key.md), [ADR-013](../../clarion/adr/ADR-013-pre-ingest-secret-scanner.md), [ADR-017 (severity + dedup)](../../clarion/adr/ADR-017-severity-and-dedup.md), [ADR-021 (plugin authority hybrid)](../../clarion/adr/ADR-021-plugin-authority-hybrid.md), [ADR-022 (core/plugin ontology)](../../clarion/adr/ADR-022-core-plugin-ontology.md), [ADR-023 (tooling baseline)](../../clarion/adr/ADR-023-tooling-baseline.md). +**Requirements floor**: `NFR-SEC-01` (pre-ingest scan + block + baseline), `NFR-SEC-04` (security events as findings), `NFR-OPS-01` / `NFR-OPS-04` (single-binary distribution). +**Effort estimate**: 6–9 working days at agentic velocity. Tasks 1 and 2 parallelisable; 3 → 4 → 5 sequential; 6 + 7 parallel with any. + +--- + +## 1. Purpose + +WP5 closes the design-review CRITICAL flag on secret exfiltration: every byte +reaching the LLM provider must first pass a Clarion-owned scanner. ADR-013 +locked the behaviour; this document turns the ADR into seven implementation +tasks an agent (or two agents working in parallel) can execute, and pins the +file paths, public APIs, schema changes, rule-ID catalogue entries, and tests +each task is responsible for. + +The workstream is the only one of Thread-1's four with significant +engineering weight; B and C are docs + packaging. WP5 ships green before the +publish-gate smoke test in Workstream D can attempt step 8 (the planted-`.env` +case). + +## 2. Scope + +In scope (this work package): + +- New workspace crate `crates/clarion-scanner/` implementing the ADR-013 + rule set as Rust-native regex + Shannon entropy detection. +- Baseline parser for `.clarion/secrets-baseline.yaml` (`detect-secrets` + v1.x format). +- CLI wiring: pre-ingest pass in `clarion-cli::analyze::run` between + source-tree walk and per-plugin dispatch. +- `--allow-unredacted-secrets` override surface with TTY + non-TTY gates. +- MCP-side awareness of `briefing_blocked: secret_present` in + `crates/clarion-mcp/`'s `summary` tool dispatch. +- Five new `CLA-SEC-*` / `CLA-INFRA-SECRET-*` rule-IDs in the + `detailed-design.md` §5 catalogue. +- Operator documentation under `docs/operator/secret-scanning.md`. + +Out of scope (deferred or owned elsewhere): + +- Filigree emission of `CLA-SEC-*` findings — WP9-B, v0.2. +- Custom-rule integration from Wardline — v0.2. +- Post-ingest scanning on LLM responses — v0.2+ additive defence. +- Redaction-instead-of-block (ADR-013 Alternative 2) — v0.2+ with opt-in. +- The override-finding monitoring loop (filigree-side dashboard work) — not + part of v0.1 publish. +- Windows support — `setrlimit` is Unix-only (`clarion-core::plugin::limits`); + WP5 follows the same posture. + +## 3. Locked surfaces and pre-existing context + +WP5 reads from and writes to surfaces already present at `main`: + +- Workspace at `Cargo.toml` (resolver 3, Rust 2024, MSRV 1.88). Current + members: `clarion-core`, `clarion-storage`, `clarion-cli`, `clarion-mcp`, + `clarion-plugin-fixture`. The scanner becomes the **sixth**. +- `crates/clarion-cli/src/analyze.rs` already contains: + - `pub async fn run(project_path: PathBuf) -> Result<()>` at line 52. + - `fn collect_source_files(root, wanted_extensions) -> Vec` at + line 1740 — the walk WP5's pre-ingest hook intercepts immediately after. + - `RunOutcome::SoftFailed { reason }` at line 1038 — the partial-success + path the override-unconfirmed exit must NOT take (override-unconfirmed + runs never start a `runs` row at all). +- `crates/clarion-core/src/plugin/host.rs`: + - `pub struct RawEntity { … extra … }` at line 105 — the `extra` field is + the carrier for `briefing_blocked`. Per the host doc-comment at line 74 + and 222, `extra` flows into `properties_json` downstream. + - `pub struct HostFinding` at `plugin/host_findings.rs:84` with subcode + constructors (`HostFinding::malformed_entity`, etc.) — the pattern WP5's + finding constructors follow. +- `crates/clarion-mcp/src/lib.rs`: + - `fn entity_properties_json(entity: &EntityRow) -> Value` at line 2382 — + central reader for `properties_json`; WP5's MCP awareness layer hooks + above this, before LLM dispatch. + - `fn summary_scope_deferred(entity: &EntityRow) -> Value` at line 2341 — + precedent for the "absence is policy" envelope shape WP5 introduces + (`briefing_blocked: secret_present`). +- `serde_norway` is already a workspace dep (replaced `serde_yaml` per + commit `9ffc5c8`). WP5 reuses it for baseline YAML. + +Surfaces this package does NOT touch: + +- Plugin protocol (no `analyze_file` RPC change). +- Writer-actor command set in `crates/clarion-storage/` — the scanner + produces `HostFinding`s and entity `extra` map entries; no new + `WriterCmd` variants. +- SQLite schema — no new tables or columns. `entities.properties` and + `runs.stats` are existing JSON-bearing columns; WP5 adds key/value pairs. + +### 3.1 Finding-write channel (load-bearing distinction) + +The codebase has two distinct "finding" concepts; WP5 emits into one of them. + +- **`HostFinding`** (`crates/clarion-core/src/plugin/host_findings.rs:84`) is + a plugin-host **integrity event**: subcode + message + structured metadata + map. It carries no severity, no `rule_id`, no `(file, line)` slot. It is + drained via `PluginHost::take_findings` and logged through + `log_plugin_findings` (`analyze.rs:1042`). Per the doc-comment at line + 79–82, "for Sprint 1 they are collected only" — they are NOT persisted to + the `findings` table. +- **`FindingRecord`** (`crates/clarion-storage/src/commands.rs:113`) is the + ADR-004 finding shape that the writer-actor persists. It carries + `rule_id`, `severity`, `entity_id`, `message`, `evidence_json`, + `properties_json`, lifecycle status, etc. Write path: send + `WriterCmd::InsertFinding { finding: Box, ack }` through + the existing `send_wait` channel. + +**WP5 emits `FindingRecord` via `WriterCmd::InsertFinding`.** Precedent: +`insert_weak_modularity_finding` at `analyze.rs:907–965` is the existing +pattern for a CLI-side, application-level finding emitted directly through +the writer. WP5's `secret_scan` module follows that pattern verbatim — +build a `FindingRecord`, send via the writer's command channel, await ack. + +`HostFinding` is **not** the channel for `CLA-SEC-*` / `CLA-INFRA-SECRET-*` +because (a) it lacks the severity / rule-ID slots ADR-013 mandates and +(b) it is not persisted. The Thread-1 §2.A.1 reference to "the existing +`HostFinding` → `WriterCmd::InsertEntity` path" is imprecise; the +authoritative channel is `WriterCmd::InsertFinding`. + +For each detection, the `FindingRecord` is populated: + +| Field | Value | +|---|---| +| `id` | `uuid::Uuid::new_v4().to_string()` | +| `tool` | `"clarion"` | +| `tool_version` | `env!("CARGO_PKG_VERSION")` | +| `run_id` | current run's `run_id` | +| `rule_id` | `"CLA-SEC-SECRET-DETECTED"` (or other; see §6 catalogue) | +| `kind` | `"defect"` (per ADR-004; secrets are defects pending remediation) | +| `severity` | `"error"` (or `"info"` for `CLA-INFRA-SECRET-BASELINE-MATCH`) | +| `confidence` | `Some(1.0)` for named-pattern matches; `Some(0.6)` for high-entropy detections (rough heuristic — refine in Task 1) | +| `confidence_basis` | `Some("pattern")` or `Some("entropy")` | +| `entity_id` | empty string (file-level, not entity-bound; the file path is in `evidence_json`) | +| `related_entities_json` | `"[]"` | +| `message` | one-line human-readable, e.g. `"AWS access key detected in src/api/keys.py:42"` (the literal bytes NEVER appear in this string) | +| `evidence_json` | `{"file_path": "...", "line_number": 42, "rule": "AwsAccessKeyId", "hashed_secret_hex": "..."}` | +| `properties_json` | `"{}"` | +| `supports_json` / `supported_by_json` | `"[]"` | +| `created_at` / `updated_at` | run's `started_at` (deterministic) | + +## 4. Architecture at a glance + +``` +clarion analyze + │ + ├─ collect_source_files(...) [unchanged] + │ + ├─ secret_scan::pre_ingest(source_files, baseline) ──► WP5 entry point + │ │ + │ ├─ For each file: + │ │ 1. mmap or read buffer + │ │ 2. Scanner::scan_bytes(&buf) ─► Vec + │ │ 3. baseline.suppress(detections, path) + │ │ 4. partition allowed/suppressed + │ │ + │ ├─ If TTY and allowed.is_empty() == false: + │ │ interactive prompt (per ADR-013 §Override) + │ │ + │ ├─ Returns: + │ │ - blocked: BTreeMap> + │ │ - findings: Vec (SEC-* + INFRA-SECRET-*) + │ │ - override_state: OverrideState + │ │ + │ └─ If override_unconfirmed → return Err → exit 78 (EX_CONFIG) + │ + ├─ run_plugins(..., blocked) ─► plugins still analyse blocked files; + │ host stamps `briefing_blocked` into + │ RawEntity.extra for blocked files + │ + ├─ writer persists entities (properties_json carries the flag) + │ + └─ summary_pass / MCP serve / etc. → read briefing_blocked, + skip LLM dispatch +``` + +Two invariants hold for any reader at any time: + +- The scanner runs **before** any plugin process is spawned. ADR-013 + §"Plugin-boundary interaction" — file buffers reach Anthropic only via + Phase 4–6 summarisation, which is gated by `briefing_blocked`. +- The override path **never** silently bypasses. Non-TTY without confirm + flag exits 78 *before* a `runs` row is created. + +## 5. Task breakdown + +Each task fits one agent pass (≤ ~500 LOC plus tests). Tasks 1 and 2 are +independent and can parallelise; 3 depends on 1 + 2; 4 depends on 3; 5 +depends on 3; 6 + 7 are documentation, parallel with any. + +### Task 1 — Scanner crate, rule registry, entropy + +**Owner**: `crates/clarion-scanner/` +**Estimated size**: 350–500 LOC including tests. + +#### Files + +| Action | Path | +|---|---| +| create | `crates/clarion-scanner/Cargo.toml` | +| create | `crates/clarion-scanner/src/lib.rs` | +| create | `crates/clarion-scanner/src/patterns.rs` | +| create | `crates/clarion-scanner/src/entropy.rs` | +| create | `crates/clarion-scanner/tests/fixtures/*.txt` (positive + negative per rule) | +| create | `crates/clarion-scanner/tests/scanner.rs` | +| modify | `Cargo.toml` (workspace root) — add `crates/clarion-scanner` member | + +#### Scope + +Implement the named-credential regex table from ADR-013 lines 35–38 (AWS +access keys, AWS secret adjacency, GitHub PATs / OAuth tokens, Anthropic +keys, OpenAI keys, Stripe keys, Slack tokens, JWT, RSA/EC/DSA/OpenSSH +private-key headers, contextual-credential `name=value` patterns). +Implement Shannon entropy detection over byte slices with the bounds +ADR-013 §Implementation specifies (base64 ≥ 4.5 entropy over ≥20 chars; hex +≥ 3.0 over ≥40 chars). UUIDs and short tokens must NOT trip the entropy +rule (positive/negative fixtures enforce this). + +#### Public surface + +```rust +pub struct Scanner { + patterns: regex::RegexSet, + pattern_meta: Vec, + entropy_b64: EntropyTuning, + entropy_hex: EntropyTuning, +} + +pub struct Detection { + pub rule_id: &'static str, // e.g. "AwsAccessKeyId", "HighEntropyBase64" + pub category: SecretCategory, + pub byte_offset: usize, + pub line_number: u32, + pub matched_len: usize, // never persist the literal bytes + pub hashed_secret: [u8; 20], // sha1, baseline-compat +} + +pub enum SecretCategory { + CloudCredential, // AWS, GCP, Azure + VcsCredential, // GitHub PATs, OAuth + AiProviderCredential, // Anthropic, OpenAI + PaymentsCredential, // Stripe + MessagingCredential, // Slack + PrivateKey, // RSA/EC/DSA/OpenSSH/PGP + JwtToken, + HighEntropy, + ContextualCredential, // password=, api_key=... +} + +impl Scanner { + pub fn new() -> Self; // default thresholds per ADR-013 + pub fn scan_bytes(&self, buf: &[u8]) -> Vec; +} +``` + +`Detection::hashed_secret` is sha1 over the literal matched bytes, computed +**at detection time**; the literal never lives anywhere downstream. This is +the same convention `detect-secrets` uses so baseline files round-trip. + +#### Dependency budget + +The scanner crate is intentionally a leaf: + +- `regex` (workspace already pulls it for elsewhere) +- `sha1` (new — small, well-maintained) +- No `tokio`, no `rusqlite`, no `serde_norway`. + +Verification at exit: `cargo tree -p clarion-scanner | head -30` shows no +`tokio` or `rusqlite` ancestors. If a downstream crate's `regex` features +leak something heavier, pin the feature set in `clarion-scanner`'s +`Cargo.toml`. + +#### Tests + +`crates/clarion-scanner/tests/scanner.rs`: + +- One positive + one negative fixture per named pattern (AWS access key, + GitHub PAT `ghp_…`, GitHub fine-grained `github_pat_…`, Anthropic + `sk-ant-…`, OpenAI `sk-…`, Stripe `sk_live_…`, Slack `xoxb-…`, JWT + `eyJ…`, RSA private-key header, OpenSSH private-key header). +- High-entropy positives: 32-char random base64, 64-char random hex. +- High-entropy negatives: UUIDs, base64-encoded SHA256 checksums (these + are high-entropy but commonly safe — confirm whether to suppress; if + not, document in the operator doc as a known-baseline candidate). +- Contextual-credential positives: `password = "..."`, `api_key: "…"`, + `SECRET_TOKEN := "…"`. +- Contextual-credential negatives: `password_hash = "…"` (post-hash safe), + `# password placeholder` (comments). + +#### Exit criteria + +- `cargo test -p clarion-scanner` green. +- `cargo clippy -p clarion-scanner -- -D warnings` clean. +- `cargo tree -p clarion-scanner` shows no `tokio` / `rusqlite` / + `serde_norway` deps. +- `cargo build --workspace` green (new crate compiles into the workspace + without changing existing build). + +--- + +### Task 2 — Baseline parser (`.clarion/secrets-baseline.yaml`) + +**Owner**: `crates/clarion-scanner/` +**Estimated size**: 200–300 LOC including tests. +**Parallelisable with Task 1**: yes — touches separate module. + +#### Files + +| Action | Path | +|---|---| +| create | `crates/clarion-scanner/src/baseline.rs` | +| modify | `crates/clarion-scanner/src/lib.rs` (re-export) | +| create | `crates/clarion-scanner/tests/fixtures/baselines/*.yaml` | +| modify | `crates/clarion-scanner/tests/scanner.rs` OR new `tests/baseline.rs` | + +#### Scope + +Parse the `detect-secrets` v1.x baseline schema (ADR-013 §Baseline). Schema +fields required: + +```yaml +version: "1.0" # must equal "1.0" +results: + "": # repository-relative path + - type: "" + hashed_secret: "" + line_number: 42 + is_secret: false # operator declares not-a-secret + justification: "..." # REQUIRED (ADR-013 line 71) +``` + +`justification` missing → emit `CLA-INFRA-SECRET-BASELINE-NO-JUSTIFICATION` +(at load time, surfaces during CLI startup; one finding per offending +entry). + +Match-then-suppress at `Detection` granularity, keyed on +`(file_path, hashed_secret, line_number)`. Both `allowed` and `suppressed` +partitions are retained so the CLI can emit one `CLA-SEC-SECRET-DETECTED` +per unsuppressed detection AND one `CLA-INFRA-SECRET-BASELINE-MATCH` +info-level finding per *actually-fired* baseline entry (the audit-surface +requirement of `NFR-SEC-04`). + +`is_secret: true` in a baseline entry is treated as **not a suppression** — +the operator is declaring the entry IS a real secret they haven't fixed +yet; behaviour is identical to no baseline entry at all. (This mirrors +`detect-secrets`' convention.) + +#### Public surface + +```rust +pub struct Baseline { + version: String, + entries: BTreeMap>, +} + +pub struct BaselineEntry { + pub rule_type: String, // detect-secrets type name + pub hashed_secret: [u8; 20], + pub line_number: u32, + pub is_secret: bool, + pub justification: String, +} + +pub struct SuppressionResult { + pub allowed: Vec, // detections that survive suppression + pub suppressed: Vec, // detections suppressed by a baseline entry + pub fired_entries: Vec, // baseline entries that actually matched +} + +pub fn load_baseline(path: &Path) -> Result; + +impl Baseline { + pub fn empty() -> Self; // path-absent path (NOT an error) + pub fn suppress( + &self, + detections: Vec, + file: &Path, + ) -> SuppressionResult; +} + +#[derive(Debug, thiserror::Error)] +pub enum BaselineError { + #[error("baseline version mismatch: expected 1.0, got {0}")] + UnsupportedVersion(String), + #[error("baseline entry missing required field 'justification' at {file}:{line}")] + MissingJustification { file: PathBuf, line: u32 }, + #[error("baseline parse error: {0}")] + Parse(#[from] serde_norway::Error), + #[error("baseline I/O error: {0}")] + Io(#[from] std::io::Error), +} +``` + +`load_baseline` returns `Ok(Baseline::empty())` if the file does not exist +(this is the common case — operators add a baseline only after the first +false positive). It errors only on parse failure, version mismatch, or +missing required fields. + +#### Tests + +- Fixture baseline + scanner detections → asserts `allowed` and + `suppressed` partitions match expected. +- Missing `justification` → `BaselineError::MissingJustification` returned. +- Baseline file absent → `Baseline::empty()` returned, no error. +- Round-trip: parse → serialise → parse → byte-identical (subject to + YAML serialisation determinism with `serde_norway`). +- Path-relativity test: baseline keys are repository-relative; scanner + passes absolute paths. The suppression layer normalises before matching. + Document the normalisation rule and assert it. + +#### Exit criteria + +- `cargo test -p clarion-scanner` (combined with Task 1) green. +- Round-trip baseline test passes. +- `BaselineError` is the only error type leaking out of the module; no + bare `anyhow::Error` at the public surface. + +--- + +### Task 3 — CLI wiring: pre-ingest hook in `analyze::run` + +**Owner**: `crates/clarion-cli/` +**Estimated size**: 300–400 LOC including the new orchestration module + tests. +**Depends on**: Task 1, Task 2. + +#### Files + +| Action | Path | +|---|---| +| create | `crates/clarion-cli/src/secret_scan.rs` (orchestration module — keep `analyze.rs` from growing further per arch-analysis H-1) | +| modify | `crates/clarion-cli/src/analyze.rs` | +| modify | `crates/clarion-cli/Cargo.toml` (add `clarion-scanner` dep) | +| modify | `crates/clarion-core/src/plugin/host.rs` (stamp `briefing_blocked` into RawEntity.extra for blocked files — see "Host integration" below) | +| create | `crates/clarion-cli/tests/secret_scan.rs` (integration tests) | +| create | `crates/clarion-cli/tests/fixtures/secret-project/...` (fixture trees) | + +#### Scope + +Between `collect_source_files` (`analyze.rs:1740`) and the per-plugin +processing loop, insert a pre-ingest scan pass: + +1. Load `.clarion/secrets-baseline.yaml` via Task 2's `load_baseline`. +2. For each file in `source_files`: + a. Read the file buffer. + b. `Scanner::scan_bytes(&buf)` → `Vec`. + c. `baseline.suppress(detections, path)` → `SuppressionResult`. +3. Build a `BlockSet: BTreeMap` of files with + non-empty `allowed`. `BlockReason` is an enum with a single variant at + v0.1: + + ```rust + pub enum BlockReason { + SecretPresent, // ADR-013: rendered as "secret_present" in properties_json + } + ``` + + The string-on-the-wire form is `"secret_present"`. Future variants + (e.g. `SizeCapExceeded`) would add new string values; readers treat + `briefing_blocked` as open-vocabulary. + +4. Emit findings via `WriterCmd::InsertFinding` (see §3.1): + - One `CLA-SEC-SECRET-DETECTED` per `allowed` detection (severity + `error`). + - One `CLA-INFRA-SECRET-BASELINE-MATCH` per `fired_entries` element + (severity `info`). + - One `CLA-INFRA-SECRET-BASELINE-NO-JUSTIFICATION` per + `BaselineError::MissingJustification` surfaced at load time. +5. Pass the `BlockSet` through to the per-plugin processing loop so the + plugin host can stamp `briefing_blocked: secret_present` into the + `RawEntity.extra` for each entity coming out of those files. + +#### Host integration + +The plugin host (`crates/clarion-core/src/plugin/host.rs`) currently +accepts `RawEntity` from the plugin and translates `extra` into +`properties_json` downstream (host.rs:74, 222). WP5 adds one path: + +- The host receives a `BTreeMap` at spawn time (via + a new field on the existing host configuration struct, or via a setter — + pick the smaller diff). For each entity whose `source.path` is in the + block map, the host inserts `"briefing_blocked": "secret_present"` into + the entity's `extra` map **after** the plugin returns, **before** + serialisation to `properties_json`. + +This keeps the per-plugin code untouched. Plugins do not need to know +about secret scanning; the host stamps the flag on its way through. + +Rationale: the alternative (passing block info to the plugin so the plugin +emits the flag) would require changing the plugin protocol — out of scope +for v0.1. + +#### Tests + +`crates/clarion-cli/tests/secret_scan.rs`: + +1. **Happy path — clean project**: fixture with no secrets → `analyze` + exits 0; no `CLA-SEC-*` findings persisted. +2. **One file with AWS key in `.env`**: fixture with a committed `.env` + containing `AKIAIOSFODNN7EXAMPLE` → + - `analyze` exits 0 with `RunOutcome::Completed`. Secret detection + produces findings but does **not** affect the run outcome — the + structural extraction succeeded, the security finding is the + audit signal, and `SoftFailed` is reserved for actual plugin + failures. + - `entities` table has rows for the `.env` file's structural entities. + - Those entities' `properties_json` contains + `"briefing_blocked":"secret_present"`. + - `findings` table has one `CLA-SEC-SECRET-DETECTED` row referencing + the file. +3. **Baseline suppression**: same `.env` fixture + a `.clarion/secrets-baseline.yaml` + with the matching entry → no `CLA-SEC-SECRET-DETECTED` finding; one + `CLA-INFRA-SECRET-BASELINE-MATCH` info-level finding; no + `briefing_blocked` flag on entities. +4. **Baseline missing justification**: malformed baseline → + `CLA-INFRA-SECRET-BASELINE-NO-JUSTIFICATION` finding; analyze + completes with `RunOutcome::Completed` (load errors degrade to + "treat baseline as empty + surface finding", they do not abort the + run and do not promote it to `SoftFailed`). +5. **Multi-file project**: secret in 1 of 10 files → 9 unblocked, 1 + blocked; entity counts match expectation. + +#### Exit criteria + +- All five integration tests green. +- `crates/clarion-cli/src/secret_scan.rs` ≤ 250 LOC (the orchestration + module; arch-analysis H-1 ceiling). +- `analyze.rs` net growth ≤ 30 LOC (the new module owns the body; analyze + delegates). +- The walking-skeleton E2E (`tests/e2e/sprint_1_walking_skeleton.sh`) + still passes — clean fixture must not regress. +- `cargo clippy --workspace --all-targets --all-features -- -D warnings` + clean. + +--- + +### Task 4 — Override semantics: `--allow-unredacted-secrets` + +**Owner**: `crates/clarion-cli/` +**Estimated size**: 250–350 LOC including tests. +**Depends on**: Task 3. + +#### Files + +| Action | Path | +|---|---| +| modify | `crates/clarion-cli/src/cli.rs` (clap definition) | +| modify | `crates/clarion-cli/src/secret_scan.rs` (override logic) | +| modify | `crates/clarion-cli/tests/secret_scan.rs` (override tests) | + +#### Scope + +Implement the override path per ADR-013 §Override. + +CLI surface (additions to the `analyze` subcommand): + +```rust +#[derive(Args)] +pub struct AnalyzeArgs { + // ... existing fields ... + /// Allow analysis of files containing unredacted secrets. REQUIRES a + /// confirmation step (interactive prompt or --confirm-allow-unredacted-secrets). + #[arg(long)] + pub allow_unredacted_secrets: bool, + + /// Non-TTY confirmation token for --allow-unredacted-secrets. + /// Must be the literal string "yes-i-understand". + #[arg(long, value_name = "TOKEN", requires = "allow_unredacted_secrets")] + pub confirm_allow_unredacted_secrets: Option, +} +``` + +Behaviour: + +- **No detections**: flag is a no-op (do NOT prompt; do NOT emit override + findings). Document this in the operator doc so operators can safely + leave the flag set in CI configurations. +- **Detections present, no override flag**: existing block behaviour + (Task 3) — `briefing_blocked` stamped, `CLA-SEC-SECRET-DETECTED` + emitted, analyze continues for structural pass. +- **Detections present, `--allow-unredacted-secrets` only, TTY**: + - Print detection summary to stderr (file path, line, rule-ID, but + NEVER the matched bytes). + - Prompt: `Type 'yes-i-understand' to proceed: `. + - Match → proceed without blocking; emit + `CLA-SEC-UNREDACTED-SECRETS-ALLOWED` per affected file; record + `override_used: true, files_affected: [...]` in `runs.stats`. + - Anything else (EOF, mismatched string, `^C`) → exit 78 (`EX_CONFIG`); + no `runs` row started. +- **Detections present, `--allow-unredacted-secrets` only, non-TTY**: + exit 78 immediately with + `CLA-INFRA-SECRET-OVERRIDE-UNCONFIRMED` rule-ID on stderr (the rule-ID + is the operator's grep target; no `findings` row is persisted because + the run never started). +- **Detections present, both flags, non-TTY, `--confirm-allow-unredacted-secrets=yes-i-understand`**: + proceed; emit `CLA-SEC-UNREDACTED-SECRETS-ALLOWED` findings; record + `runs.stats` override entry. +- **Detections present, both flags, wrong confirm value**: exit 78; + rule-ID on stderr; no run row. + +`std::io::IsTerminal` on stdin is the TTY detector (stable since Rust +1.70; workspace MSRV 1.88). + +#### Override state in `runs.stats` + +The existing `runs.stats` column is a TEXT-affinity JSON blob. WP5 appends +two keys: + +```json +{ + "secret_override_used": true, + "secret_override_files_affected": [ + "src/api/keys.py", + "tests/fixtures/secrets.yaml" + ] +} +``` + +These keys are absent when no override fires. Existing readers ignore +unknown keys (verify by grepping for `runs.stats` JSON deserialisation +sites). + +#### Tests + +1. **Non-TTY override-confirmed**: fixture with one AWS key, both flags + set, confirm value correct → exits 0, no `briefing_blocked`, one + `CLA-SEC-UNREDACTED-SECRETS-ALLOWED` per file, `runs.stats` carries + the override keys. +2. **Non-TTY override-unconfirmed**: only `--allow-unredacted-secrets`, + no confirm → exit 78, stderr contains + `CLA-INFRA-SECRET-OVERRIDE-UNCONFIRMED`, no `runs` row in the DB. +3. **Non-TTY override-wrong-confirm**: both flags but + `--confirm-allow-unredacted-secrets=oops` → exit 78, stderr contains + the rule-ID. +4. **Override + no detections**: flag set but no secrets present → no + override findings, no `runs.stats` override keys (the operator's CI + stays clean on clean repos). +5. **TTY path**: marked `#[ignore]` with a doc comment; manual + verification belongs to Workstream D's smoke test step 8. + +#### Exit criteria + +- All non-TTY tests green. +- `cli.rs` exit codes documented in a module-level doc comment: + - 0: success (with or without override). + - 1: hard failure (existing behaviour). + - 78 (`EX_CONFIG`): override misconfigured. +- Operator doc (Task 7) documents the exit-code contract and the + `--confirm-allow-unredacted-secrets` syntax. + +--- + +### Task 5 — MCP-side awareness of `briefing_blocked` + +**Owner**: `crates/clarion-mcp/` +**Estimated size**: 150–250 LOC including tests. +**Depends on**: Task 3 (writes the flag); independent of Task 4. +**Parallelisable with Task 4**: yes. + +#### Files + +| Action | Path | +|---|---| +| modify | `crates/clarion-mcp/src/lib.rs` (summary tool dispatch) | +| modify | `crates/clarion-mcp/tests/storage_tools.rs` (or split into a new test file) | + +#### Scope + +When `summary(id)` is called on an entity whose +`properties.briefing_blocked == "secret_present"`, the MCP server must: + +1. **Not** invoke the LLM provider. +2. **Not** consume any budget ledger. +3. **Not** write a row to `summary_cache`. +4. Return an envelope shape that tells the consult-mode agent the absence + is policy, not pipeline failure. + +#### Envelope shape + +```json +{ + "entity_id": "python:function:demo.foo|function", + "summary": null, + "briefing_blocked": "secret_present", + "remediation": "File flagged by pre-ingest secret scan. Fix the secret or whitelist via .clarion/secrets-baseline.yaml. See ADR-013." +} +``` + +This is parallel to the existing `summary_scope_deferred` envelope +(`crates/clarion-mcp/src/lib.rs:2341`) and the four `issues_unavailable` +envelopes in `clarion-mcp::filigree`. Add a `summary_briefing_blocked` +helper next to `summary_scope_deferred`. + +#### Hook point + +The summary tool dispatch reads `entity_properties_json` at line 2382 of +`lib.rs`. The branch on `briefing_blocked` happens **before** any cache +lookup or provider invocation. Recommended placement: right after the +existing `summary_scope_deferred` check, since both are +"return-immediately" policy branches. + +#### Tests + +1. **Storage-only**: fixture entity with + `properties.briefing_blocked == "secret_present"` → `summary` tool + returns the envelope; no row in `summary_cache`. +2. **Recording-provider isolation**: with `RecordingProvider` configured + in recording mode and a counter wrapper around the inner provider, + call `summary` on a blocked entity → the inner-provider call counter + stays at zero. (The exact assertion form depends on the + `RecordingProvider` fixture layout — either "no fixture file + created" or "no new entry appended"; the load-bearing claim is "no + outbound LLM call." Pick whichever assertion the existing + `RecordingProvider` test harness already supports.) +3. **Budget untouched**: read the session's budget ledger before and + after — bytes identical. +4. **Other tools unaffected**: `entity_at`, `find_entity`, `callers_of`, + `execution_paths_from`, `neighborhood`, `issues_for` all return their + normal envelopes for blocked entities. The block only affects LLM + dispatch; structural navigation still works (this is the whole point + of "block briefings, not analysis"). + +#### Exit criteria + +- Storage-tools tests green. +- Manual log inspection on a fixture with one blocked entity confirms + zero outbound LLM calls. + +--- + +### Task 6 — Rule-ID catalogue in `detailed-design.md` + +**Owner**: docs +**Estimated size**: ~100 lines of doc changes. +**Parallelisable with any task**. + +#### Files + +| Action | Path | +|---|---| +| modify | `docs/clarion/v0.1/detailed-design.md` (§5 rule catalogue) | + +#### Scope + +Append rule rows for each of the five new rule-IDs WP5 introduces. Use the +existing table shape in §5 (do not invent a new format). + +| Rule-ID | Severity | Category | Description (one sentence) | Remediation (one sentence) | ADR | +|---|---|---|---|---|---| +| `CLA-SEC-SECRET-DETECTED` | error | security | Pre-ingest secret scanner detected a credential pattern in a file slated for LLM dispatch. | Remove the secret, rotate the credential, or whitelist via `.clarion/secrets-baseline.yaml` with a justification. | [ADR-013](../adr/ADR-013-pre-ingest-secret-scanner.md) | +| `CLA-SEC-UNREDACTED-SECRETS-ALLOWED` | error | security | Operator invoked `--allow-unredacted-secrets`; file content reached the LLM provider with secrets intact. | Audit override usage via `filigree list --rule-id=CLA-SEC-UNREDACTED-SECRETS-ALLOWED --since 30d`. | [ADR-013](../adr/ADR-013-pre-ingest-secret-scanner.md) | +| `CLA-INFRA-SECRET-BASELINE-NO-JUSTIFICATION` | error | infra | Baseline entry missing required `justification` field; entry not honoured. | Add a `justification` string explaining why the match is safe. | [ADR-013](../adr/ADR-013-pre-ingest-secret-scanner.md) | +| `CLA-INFRA-SECRET-BASELINE-MATCH` | info | infra | Baseline entry suppressed a scanner detection (audit surface). | None — informational, retained for `NFR-SEC-04` audit. | [ADR-013](../adr/ADR-013-pre-ingest-secret-scanner.md) | +| `CLA-INFRA-SECRET-OVERRIDE-UNCONFIRMED` | error | infra | `--allow-unredacted-secrets` supplied without confirmation; run aborted before start. | Supply `--confirm-allow-unredacted-secrets=yes-i-understand` in non-TTY contexts or run interactively. | [ADR-013](../adr/ADR-013-pre-ingest-secret-scanner.md) | + +These IDs make the WP9-B Filigree-emission story (v0.2) able to +round-trip without a separate spec pass. + +#### Exit criteria + +- `detailed-design.md` lints clean (existing markdown / cross-reference + checks in CI). +- ADR-013 retains canonical authority on *behaviour*; the design doc + carries the *catalogue*. No behaviour is restated in the design doc + beyond the one-sentence description and remediation. + +--- + +### Task 7 — Operator documentation + +**Owner**: docs +**Estimated size**: ≤ 250 lines. +**Parallelisable with any task**. + +#### Files + +| Action | Path | +|---|---| +| create | `docs/operator/secret-scanning.md` | +| modify | `docs/operator/README.md` (add link to the new page) | +| modify | `README.md` (Workstream B Task 1; cross-link) — coordinated with WS B, do not duplicate | + +If `docs/operator/README.md` does not exist yet, create a minimal index +listing this and the (forthcoming) `getting-started.md` from WS-B Task 2. +Coordinate with WS-B to avoid duplicate index creation. + +#### Scope + +A single page that lets a non-engineer resolve a baseline false-positive +without reading ADR-013. Sections: + +1. **What the scanner does** (3 sentences). +2. **What gets blocked** (file-level; structural extraction continues; + summaries don't). +3. **How to whitelist a false positive** (edit + `.clarion/secrets-baseline.yaml`; example entry; commit it). +4. **The override flag** (`--allow-unredacted-secrets` + + `--confirm-allow-unredacted-secrets=yes-i-understand`; when to use it; + what gets audited). +5. **Exit codes** (0 / 1 / 78). +6. **Finding the audit trail** ( + `select * from findings where rule_id like 'CLA-SEC-%'`; + forward-pointer to Filigree integration in v0.2). +7. **Limitations** (pattern-based scanning has false negatives; novel + secret shapes; air-gapped alternatives for truly high-risk repos — + `--no-llm`). + +#### Exit criteria + +- A non-engineer can read the doc and resolve a baseline false-positive + end-to-end (verified in WS-D smoke test). +- The doc is ≤ 250 lines. +- Cross-links to ADR-013 for the operator who wants the design rationale, + but does not require reading the ADR to act. + +--- + +## 6. Schema additions + +WP5 does not create new tables or columns. It adds two well-known keys to +existing JSON-bearing columns: + +### `entities.properties` (existing TEXT column, JSON object) + +- `briefing_blocked: "secret_present"` — stamped by the host on entities + whose source file was flagged by the pre-ingest scanner. + +Other plausible `briefing_blocked` reasons in the future (e.g. +`"size_cap_exceeded"`, `"operator_excluded"`) are NOT introduced by WP5; +the schema is open-vocabulary on the value side. ADR-013 names +`secret_present` as the canonical first reason. + +### `runs.stats` (existing TEXT column, JSON object) + +- `secret_override_used: true` — present only when the override fired. +- `secret_override_files_affected: ["…", "…"]` — present only when + `secret_override_used == true`. + +Both keys absent on runs with no override → reader code defaults to "no +override happened." + +### Rule-ID grammar + +The five new rule-IDs all match the existing ADR-022 manifest rule-ID +grammar (see `plugin/manifest` correction in commit `0cb61b4`). No +manifest changes are needed because these are *core-emitted* findings, +not plugin-emitted. + +--- + +## 7. Test strategy across layers + +| Layer | Tests | Location | +|---|---|---| +| Unit | Scanner pattern fixtures (positive + negative per rule); entropy bounds; baseline parse / suppress / round-trip | `crates/clarion-scanner/tests/` | +| Unit | Override CLI exit codes; `runs.stats` JSON shape | `crates/clarion-cli/tests/secret_scan.rs` | +| Unit | MCP `summary` envelope on blocked entity | `crates/clarion-mcp/tests/storage_tools.rs` | +| Integration | `analyze` over fixture project with one secret-bearing file → entities + findings + properties_json shape | `crates/clarion-cli/tests/secret_scan.rs` | +| Integration | Baseline suppression; baseline missing-justification | same | +| Integration | Non-TTY override paths (3 cases) | same | +| E2E | `clarion install && clarion analyze` against a fixture project with one known secret; assert exit 0, entities present, briefing_blocked flagged, finding persisted; assert walking-skeleton fixture still green | `tests/e2e/wp5_secret_scan.sh` (new), parallels `tests/e2e/sprint_1_walking_skeleton.sh` | +| Manual | TTY interactive prompt path | WS-D smoke test step 8 | + +### CI gates (ADR-023 floor, unchanged) + +WP5 must keep all of these green: + +- `cargo fmt --all -- --check` +- `cargo clippy --workspace --all-targets --all-features -- -D warnings` +- `cargo build --workspace --bins` +- `cargo nextest run --workspace --all-features` +- `RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --all-features` +- `cargo deny check` +- `plugins/python/.venv/bin/ruff check plugins/python` +- `plugins/python/.venv/bin/ruff format --check plugins/python` +- `plugins/python/.venv/bin/mypy --strict plugins/python` +- `plugins/python/.venv/bin/pytest plugins/python` +- `bash tests/e2e/sprint_1_walking_skeleton.sh` + +A new E2E (`wp5_secret_scan.sh`) is added to the CI matrix's +`walking-skeleton` job. + +--- + +## 8. Risks and open questions + +| ID | Risk / question | Mitigation | +|---|---|---| +| R-1 | High-entropy detection on UUIDs and base64 checksums creates false-positive flood on real repos | Bound length thresholds per ADR-013 (≥20 chars base64, ≥40 chars hex); document baseline workflow prominently in Task 7; Workstream D smoke test on `requests` will surface real-world false-positive rate before publish | +| R-2 | sha1 hashing of literal secrets means a baseline rebuilt from a different `detect-secrets` version may mismatch | Document the exact hash function (sha1 over the literal matched bytes, no normalisation); migration story for `detect-secrets` v2.x is a v0.2 problem | +| R-3 | The override flag could become normalised in CI configurations ("we always pass it because…") | Audit-surface design: every override is a finding; v0.2 Filigree integration makes them visible; operator doc explicitly names this anti-pattern | +| R-4 | Plugin-host integration to stamp `briefing_blocked` adds a code path through a 3 126-LOC file flagged by arch-analysis §5.4 A-3 | Keep the host-side change small (≤ 30 LOC, a single map lookup); the orchestration lives in `clarion-cli/src/secret_scan.rs`, not in the host | +| R-5 | Baseline format compatibility with `detect-secrets` v1.x must be exact; an operator running `detect-secrets scan --baseline` and dropping the file in `.clarion/` must work | Round-trip test in Task 2; smoke-test the workflow with a real `detect-secrets`-generated baseline in WS-D | +| Q-1 | Should baseline entries carry an `expires_at` field so stale entries are surfaced? | Out of scope for v0.1; surface as a v0.2 enhancement if the override-monitoring loop also lands then | +| Q-2 | Does `briefing_blocked: secret_present` propagate up to module / subsystem summaries when those are aggregated (deferred to v0.2 per ADR-030)? | Defer to v0.2; the leaf summary skip is sufficient for v0.1; document in Task 7 that "summaries of containing modules may still be generated and may infer secret content indirectly — fix the underlying file" | +| Q-3 | Should the scanner also redact secrets from log lines emitted by Clarion itself (the `runs//log.jsonl` ADR-013 line 16 cites)? | **Resolved at planning time**: `runs//log.jsonl` is referenced only in `clarion-cli/src/install.rs:74` (the gitignore template) and `tests/install.rs:40` (the test asserting it's ignored). No writer code exists in `crates/` that emits to that path today. WP5 has nothing to redact. If a per-run JSONL log lands later (likely WP6 batched pipeline or WP9-B), that work must include a redaction pass for any file content emitted from `briefing_blocked` files; cite this row when the log writer is authored. | + +All three questions resolved at planning time. None blocks task kickoff. + +--- + +## 9. Workstream exit criteria (gate to Workstream D) + +The workstream is signed-off when ALL of the following hold: + +- [ ] All seven tasks closed. +- [ ] `cargo test --workspace --all-features` green. +- [ ] Existing CI gates (§7) unchanged in pass status. +- [ ] `tests/e2e/wp5_secret_scan.sh` added and green; included in the + `walking-skeleton` CI job. +- [ ] The walking-skeleton E2E continues to pass on the clean-fixture + path (no regression). +- [ ] `docs/operator/secret-scanning.md` reviewed by a non-author or + verified during WS-D smoke test step 8. +- [ ] All five rule-IDs appear in `detailed-design.md` §5. +- [ ] No new clippy warnings; no `unsafe` blocks introduced. +- [ ] `cargo tree -p clarion-scanner` shows no `tokio` / `rusqlite` / + `serde_norway` ancestors. + +When this gate closes, Workstream D's smoke-test step 8 (planted-`.env`) +becomes the publish-gate proof point for WP5. + +--- + +## 10. Filigree seeding + +Issues to create at workstream kickoff (umbrella + seven tasks). Set +dependencies as shown. + +```bash +# Umbrella +filigree create --type=work_package \ + --title="WP5 — Pre-ingest secret scanner (ADR-013)" \ + --labels="release:v0.1,sprint:3,wp:5,adr:013,tier:a" \ + --priority=1 +# capture the returned id as $WP5_UMBRELLA + +# Task 1 +filigree create --type=task \ + --title="WP5 Task 1 — Scanner crate + rule registry + entropy" \ + --labels="release:v0.1,sprint:3,wp:5,adr:013,crate:scanner" \ + --priority=1 +# capture as $T1 + +# Task 2 +filigree create --type=task \ + --title="WP5 Task 2 — Baseline parser (.clarion/secrets-baseline.yaml)" \ + --labels="release:v0.1,sprint:3,wp:5,adr:013,crate:scanner" \ + --priority=1 +# capture as $T2 + +# Task 3 +filigree create --type=task \ + --title="WP5 Task 3 — CLI wiring: pre-ingest hook in analyze::run" \ + --labels="release:v0.1,sprint:3,wp:5,adr:013,crate:cli,crate:core" \ + --priority=1 +# capture as $T3 + +# Task 4 +filigree create --type=task \ + --title="WP5 Task 4 — Override semantics: --allow-unredacted-secrets" \ + --labels="release:v0.1,sprint:3,wp:5,adr:013,crate:cli" \ + --priority=1 +# capture as $T4 + +# Task 5 +filigree create --type=task \ + --title="WP5 Task 5 — MCP-side awareness of briefing_blocked" \ + --labels="release:v0.1,sprint:3,wp:5,adr:013,crate:mcp" \ + --priority=1 +# capture as $T5 + +# Task 6 +filigree create --type=task \ + --title="WP5 Task 6 — Rule-ID catalogue entries in detailed-design.md" \ + --labels="release:v0.1,sprint:3,wp:5,adr:013,docs" \ + --priority=2 +# capture as $T6 + +# Task 7 +filigree create --type=task \ + --title="WP5 Task 7 — Operator documentation: secret-scanning.md" \ + --labels="release:v0.1,sprint:3,wp:5,adr:013,docs" \ + --priority=2 +# capture as $T7 + +# Dependencies +filigree add-dep $T3 $T1 # T3 depends on T1 +filigree add-dep $T3 $T2 # T3 depends on T2 +filigree add-dep $T4 $T3 # T4 depends on T3 +filigree add-dep $T5 $T3 # T5 depends on T3 (writes the flag T5 reads) + +# Umbrella rolls up +for t in $T1 $T2 $T3 $T4 $T5 $T6 $T7; do + filigree add-dep $WP5_UMBRELLA $t +done + +# Body for each issue should link back to the matching section of this doc: +# docs/implementation/v0.1-publish/ws-a-secret-scanner.md#task-N-... +``` + +The umbrella is `done` when all seven tasks close; the workstream signs +off via the criteria in §9. + +--- + +## 11. References + +- [ADR-013 — Pre-Ingest Secret Scanner with LLM-Dispatch Block](../../clarion/adr/ADR-013-pre-ingest-secret-scanner.md) — canonical spec. +- [ADR-007 — Summary cache key](../../clarion/adr/ADR-007-summary-cache-key.md) — `briefing_blocked` interaction. +- [ADR-017 — Severity and dedup](../../clarion/adr/ADR-017-severity-and-dedup.md) — `CLA-SEC-*` namespace ownership. +- [ADR-021 — Plugin authority hybrid](../../clarion/adr/ADR-021-plugin-authority-hybrid.md) — path-jail upstream of scanner. +- [ADR-022 — Core/plugin ontology boundary](../../clarion/adr/ADR-022-core-plugin-ontology.md) — secret detection as a core-owned algorithm. +- [ADR-023 — Tooling baseline](../../clarion/adr/ADR-023-tooling-baseline.md) — CI floor every task in this workstream must clear. +- [Requirements — NFR-SEC-01, NFR-SEC-04, NFR-OPS-01, NFR-OPS-04](../../clarion/v0.1/requirements.md) — requirement floor. +- [Thread 1 — Pre-publish blockers (program of work)](./thread-1-pre-publish-blockers.md) — the umbrella program. +- [v0.1-plan.md — WP5 scope](../v0.1-plan.md#wp5--pre-ingest-secret-scanner) — original work-package definition. +- [Sprint 2 scope amendment — WP5 deferral rationale](../sprint-2/scope-amendment-2026-05.md) — "production deployment against unknown corpora gates on this returning." +- [Arch-analysis final report §7 follow-ups](../../arch-analysis-2026-05-18-1244/04-final-report.md) — H-1 (`SoftFailed` coverage) coverage now closed via `clarion-141ca7de30`; relevant precondition for Task 3. +- [`detect-secrets` baseline format](https://github.com/Yelp/detect-secrets/blob/master/README.md#baseline-file) — the format Task 2 matches. diff --git a/docs/operator/README.md b/docs/operator/README.md index 225676df..145cf4d6 100644 --- a/docs/operator/README.md +++ b/docs/operator/README.md @@ -6,3 +6,5 @@ Practical notes for configuring and running Clarion. headers, and token-ceiling configuration for v0.1. - [Runtime topology](./runtime-topology.md) — supported `clarion serve` and `clarion analyze` concurrency against one `.clarion/clarion.db`. +- [Secret scanning](./secret-scanning.md) — pre-ingest scanner behavior, + baseline false-positive workflow, override confirmation, and audit queries. diff --git a/docs/operator/secret-scanning.md b/docs/operator/secret-scanning.md new file mode 100644 index 00000000..bd1b66ae --- /dev/null +++ b/docs/operator/secret-scanning.md @@ -0,0 +1,74 @@ +# Secret Scanning + +Clarion scans source files before any file content can be used for LLM summaries. A detected credential creates a finding and marks entities from that file with `briefing_blocked: secret_present`. Structural analysis still runs, but summaries for that file do not. + +## What Gets Blocked + +Blocking is file-level. If `src/config.py` contains a detected key, entities from that file remain queryable through structural tools, but the `summary` tool returns a policy envelope instead of calling the LLM provider or writing `summary_cache`. + +Files outside the analyzed plugin extension set are not scanned because they are not candidates for plugin dispatch in that run. + +## Whitelist A False Positive + +Add `.clarion/secrets-baseline.yaml` and commit it with the source change: + +```yaml +version: "1.0" +results: + "src/auth/fixtures.py": + - type: "AWS Access Key" + hashed_secret: "25910f981e85ca04baf359199dd0bd4a3ae738b6" + line_number: 42 + is_secret: false + justification: "AWS documentation example key used in a test fixture." +``` + +The hash is SHA-1 over the matched literal bytes, matching `detect-secrets` v1.x baseline conventions. `justification` is required; entries without it are ignored and produce `CLA-INFRA-SECRET-BASELINE-NO-JUSTIFICATION`. + +A matching baseline entry suppresses the block and records `CLA-INFRA-SECRET-BASELINE-MATCH` for audit. + +## Override Flag + +Use `--allow-unredacted-secrets` only when you deliberately accept that detected secrets may reach the LLM provider. + +Interactive runs prompt for: + +```text +yes-i-understand +``` + +Non-TTY runs must pass both flags: + +```bash +clarion analyze --allow-unredacted-secrets \ + --confirm-allow-unredacted-secrets=yes-i-understand . +``` + +Confirmed overrides do not set `briefing_blocked`; they emit `CLA-SEC-UNREDACTED-SECRETS-ALLOWED` and add `secret_override_used` plus `secret_override_files_affected` to `runs.stats`. Passing `--allow-unredacted-secrets` on a clean repo is a no-op. + +## Exit Codes + +| Code | Meaning | +|---|---| +| 0 | Analysis completed, with or without secret findings or a confirmed override. | +| 1 | Hard failure in the normal analysis path. | +| 78 | Secret override was requested but not confirmed; no run row is started. | + +## Audit Trail + +Local SQLite: + +```sql +select rule_id, severity, message, evidence +from findings +where rule_id like 'CLA-SEC-%' + or rule_id like 'CLA-INFRA-SECRET-%'; +``` + +Filigree integration for scanner findings is planned for v0.2. Until then, the local `findings` table is the authoritative WP5 audit surface. + +## Limitations + +The scanner is pattern-based. It can miss novel internal key formats and it can flag high-entropy test data. Use a justified baseline for reviewed false positives, and prefer `--no-llm` or an air-gapped workflow for repos where any source disclosure would be unacceptable. + +See [ADR-013](../clarion/adr/ADR-013-pre-ingest-secret-scanner.md) for design rationale. diff --git a/tests/e2e/wp5_secret_scan.sh b/tests/e2e/wp5_secret_scan.sh new file mode 100755 index 00000000..13b87935 --- /dev/null +++ b/tests/e2e/wp5_secret_scan.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# WP5 pre-ingest secret-scanner end-to-end smoke. + +set -euo pipefail + +REPO_ROOT="${REPO_ROOT:-$(git rev-parse --show-toplevel)}" +CARGO_BUILD="${CARGO_BUILD:-1}" + +log() { printf '[wp5-secret-scan] %s\n' "$*" >&2; } +fail() { printf '[wp5-secret-scan] FAIL: %s\n' "$*" >&2; exit 1; } + +cd "$REPO_ROOT" + +if [ "$CARGO_BUILD" = "1" ]; then + log "building clarion (release) ..." + cargo build --workspace --release +fi + +CLARION_BIN="$REPO_ROOT/target/release/clarion" +[ -x "$CLARION_BIN" ] || fail "clarion binary missing at $CLARION_BIN" + +DEMO_DIR="$(mktemp -d -t clarion-wp5-demo-XXXXXX)" +PLUGIN_DIR="$(mktemp -d -t clarion-wp5-plugin-XXXXXX)" +trap 'rm -rf "$DEMO_DIR" "$PLUGIN_DIR"' EXIT + +cat > "$PLUGIN_DIR/clarion-plugin-secretfixture" <<'PY' +#!/usr/bin/python3 +import json +import pathlib +import re +import sys + +def read_frame(): + headers = {} + while True: + line = sys.stdin.buffer.readline() + if line in (b"", b"\r\n"): + break + name, value = line.decode("ascii").strip().split(":", 1) + headers[name.lower()] = value.strip() + length = int(headers["content-length"]) + return json.loads(sys.stdin.buffer.read(length)) + +def write_frame(message): + body = json.dumps(message, separators=(",", ":")).encode("utf-8") + sys.stdout.buffer.write(b"Content-Length: " + str(len(body)).encode("ascii") + b"\r\n\r\n") + sys.stdout.buffer.write(body) + sys.stdout.buffer.flush() + +while True: + msg = read_frame() + method = msg.get("method") + if method == "initialized": + continue + if method == "exit": + raise SystemExit(0) + ident = msg["id"] + if method == "initialize": + write_frame({"jsonrpc":"2.0","id":ident,"result":{"name":"clarion-plugin-secretfixture","version":"0.1.0","ontology_version":"0.1.0","capabilities":{}}}) + elif method == "analyze_file": + path = msg["params"]["file_path"] + name = "file_" + re.sub(r"[^A-Za-z0-9_]", "_", pathlib.Path(path).name) + write_frame({"jsonrpc":"2.0","id":ident,"result":{"entities":[{"id":"secretfixture:module:"+name,"kind":"module","qualified_name":name,"source":{"file_path":path}}],"edges":[]}}) + elif method == "shutdown": + write_frame({"jsonrpc":"2.0","id":ident,"result":{}}) + else: + raise SystemExit(1) +PY +chmod 755 "$PLUGIN_DIR/clarion-plugin-secretfixture" + +cat > "$PLUGIN_DIR/plugin.toml" <<'TOML' +[plugin] +name = "clarion-plugin-secretfixture" +plugin_id = "secretfixture" +version = "0.1.0" +protocol_version = "1.0" +executable = "clarion-plugin-secretfixture" +language = "secretfixture" +extensions = ["sec"] + +[capabilities.runtime] +expected_max_rss_mb = 256 +expected_entities_per_file = 100 +wardline_aware = false +reads_outside_project_root = false + +[ontology] +entity_kinds = ["module"] +edge_kinds = [] +rule_id_prefix = "CLA-SECRET-FIXTURE-" +ontology_version = "0.1.0" +TOML + +log "scratch project: $DEMO_DIR" +"$CLARION_BIN" install --path "$DEMO_DIR" +printf "aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'\n" > "$DEMO_DIR/leaky.sec" + +PATH="$PLUGIN_DIR" "$CLARION_BIN" analyze "$DEMO_DIR" + +DB="$DEMO_DIR/.clarion/clarion.db" +BLOCKED=$(sqlite3 "$DB" "select json_extract(properties, '\$.briefing_blocked') from entities;") +[ "$BLOCKED" = "secret_present" ] || fail "expected briefing_blocked secret_present, got $BLOCKED" + +FINDINGS=$(sqlite3 "$DB" "select count(*) from findings where rule_id = 'CLA-SEC-SECRET-DETECTED';") +[ "$FINDINGS" = "1" ] || fail "expected one CLA-SEC-SECRET-DETECTED finding, got $FINDINGS" + +RUN_STATUS=$(sqlite3 "$DB" "select status from runs;") +[ "$RUN_STATUS" = "completed" ] || fail "expected completed run, got $RUN_STATUS" + +log "PASS: WP5 secret scan blocks summaries and records finding" From 2117118c45bdea5c50d061fd168089b0e700dfce Mon Sep 17 00:00:00 2001 From: John Morrissey <544926+tachyon-beep@users.noreply.github.com> Date: Tue, 19 May 2026 02:31:16 +1000 Subject: [PATCH 2/4] fix(wp5): close secret scanner review gaps --- crates/clarion-cli/src/analyze.rs | 158 +++++++++++++++++++++++- crates/clarion-cli/src/secret_scan.rs | 59 ++++++--- crates/clarion-cli/tests/secret_scan.rs | 112 ++++++++++++++++- crates/clarion-core/src/plugin/host.rs | 24 +++- crates/clarion-mcp/src/lib.rs | 7 +- crates/clarion-scanner/src/baseline.rs | 37 ++++-- crates/clarion-scanner/src/lib.rs | 3 +- crates/clarion-scanner/tests/scanner.rs | 36 +++++- docs/operator/secret-scanning.md | 2 +- tests/e2e/wp5_secret_scan.sh | 10 +- 10 files changed, 407 insertions(+), 41 deletions(-) diff --git a/crates/clarion-cli/src/analyze.rs b/crates/clarion-cli/src/analyze.rs index 5ad5fa5b..db47b7d5 100644 --- a/crates/clarion-cli/src/analyze.rs +++ b/crates/clarion-cli/src/analyze.rs @@ -207,8 +207,13 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti let source_files = collect_source_files(&project_root, &wanted_extensions); tracing::info!(file_count = source_files.len(), "source tree walk complete"); + let secret_scan_files = collect_secret_scan_files(&project_root, &source_files); + tracing::info!( + file_count = secret_scan_files.len(), + "secret scan file walk complete" + ); let secret_scan_outcome = - crate::secret_scan::pre_ingest(&project_root, &source_files, &options.secret_scan)?; + crate::secret_scan::pre_ingest(&project_root, &secret_scan_files, &options.secret_scan)?; begin_run(&writer, &run_id, &analyze_config_json, &started_at).await?; // ── Per-plugin processing ───────────────────────────────────────────────── @@ -280,6 +285,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti let exec_clone = plugin.executable.clone(); let files_clone = plugin_files.clone(); let briefing_blocks_clone = secret_scan_outcome.briefing_blocks.clone(); + let scanned_files_clone = secret_scan_outcome.scanned_files().clone(); // A JoinError here means the blocking task panicked (OOM, stack // overflow, internal unwrap, abort — anything that unwinds past the @@ -298,6 +304,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti &exec_clone, &files_clone, &briefing_blocks_clone, + &scanned_files_clone, ) }) .await, @@ -435,6 +442,22 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti } } + if !matches!(run_outcome, RunOutcome::HardFailed { .. }) + && let Err(e) = ensure_secret_finding_anchors( + &writer, + &project_root, + &started_at, + &secret_scan_outcome, + &mut secret_finding_anchors, + ) + .await + { + tracing::error!(run_id = %run_id, error = %e, "secret finding anchor persistence failed"); + run_outcome = RunOutcome::HardFailed { + reason: format!("secret finding anchor persistence failed: {e:#}"), + }; + } + if !matches!(run_outcome, RunOutcome::HardFailed { .. }) && let Err(e) = crate::secret_scan::emit_findings( &writer, @@ -1207,6 +1230,7 @@ fn run_plugin_blocking( executable: &Path, files: &[PathBuf], briefing_blocks: &BTreeMap, + scanned_source_files: &BTreeSet, ) -> Result { use clarion_core::PluginHost; @@ -1223,6 +1247,7 @@ fn run_plugin_blocking( } })?; host.set_briefing_blocks(briefing_blocks.clone()); + host.set_scanned_source_files(scanned_source_files.clone()); let work_result: Result = (|| { let mut collected_entities: Vec<(String, EntityRecord)> = Vec::new(); @@ -1551,6 +1576,91 @@ fn remember_secret_finding_anchors( } } +async fn ensure_secret_finding_anchors( + writer: &Writer, + project_root: &Path, + started_at: &str, + outcome: &crate::secret_scan::SecretScanOutcome, + anchors: &mut BTreeMap, +) -> Result<()> { + for file in outcome.finding_files() { + let key = canonical_or_original(&file); + if anchors.contains_key(&key) { + continue; + } + let id = secret_finding_anchor_id(project_root, &key); + let relative = display_relative_path(project_root, &key); + let short_name = key + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(&relative) + .to_owned(); + let mut properties = serde_json::Map::new(); + properties.insert("finding_anchor".to_owned(), serde_json::json!(true)); + if let Some(reason) = outcome.briefing_blocks.get(&key) { + properties.insert( + "briefing_blocked".to_owned(), + serde_json::Value::String(reason.clone()), + ); + } + let record = EntityRecord { + id: id.clone(), + plugin_id: "core".to_owned(), + kind: "file".to_owned(), + name: relative, + short_name, + parent_id: None, + source_file_id: None, + source_file_path: Some(key.display().to_string()), + source_byte_start: None, + source_byte_end: None, + source_line_start: None, + source_line_end: None, + properties_json: serde_json::Value::Object(properties).to_string(), + content_hash: file_content_hash(&key), + summary_json: None, + wardline_json: None, + first_seen_commit: None, + last_seen_commit: None, + created_at: started_at.to_owned(), + updated_at: started_at.to_owned(), + }; + writer + .send_wait(|ack| WriterCmd::InsertEntity { + entity: Box::new(record), + ack, + }) + .await + .map_err(|e| anyhow::anyhow!("{e}")) + .with_context(|| format!("InsertEntity for secret finding anchor {id}"))?; + anchors.insert(key, id); + } + Ok(()) +} + +fn secret_finding_anchor_id(project_root: &Path, file: &Path) -> String { + let relative = display_relative_path(project_root, file); + let digest = blake3::hash(relative.as_bytes()).to_hex().to_string(); + format!("core:file:{digest}") +} + +fn canonical_or_original(path: &Path) -> PathBuf { + path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) +} + +fn display_relative_path(root: &Path, path: &Path) -> String { + path.strip_prefix(root) + .unwrap_or(path) + .display() + .to_string() +} + +fn file_content_hash(path: &Path) -> Option { + fs::read(path) + .ok() + .map(|bytes| blake3::hash(&bytes).to_hex().to_string()) +} + /// Map an `AcceptedEntity` to an `EntityRecord` for the writer-actor. fn map_entity_to_record( entity: &AcceptedEntity, @@ -1846,6 +1956,52 @@ fn collect_source_files(root: &Path, wanted_extensions: &BTreeSet) -> Ve out } +fn collect_secret_scan_files(root: &Path, source_files: &[PathBuf]) -> Vec { + let mut out: BTreeSet = source_files + .iter() + .map(|path| canonical_or_original(path)) + .collect(); + for path in collect_secret_scan_sidecars(root) { + out.insert(canonical_or_original(&path)); + } + out.into_iter().collect() +} + +fn collect_secret_scan_sidecars(root: &Path) -> Vec { + let mut out = Vec::new(); + let mut builder = WalkBuilder::new(root); + builder + .follow_links(false) + .hidden(false) + .ignore(false) + .git_ignore(false) + .git_global(false) + .git_exclude(false) + .parents(false) + .require_git(false) + .filter_entry(|entry| !is_skipped_dir(entry)); + + for entry in builder.build().filter_map(std::result::Result::ok) { + let Some(file_type) = entry.file_type() else { + continue; + }; + let path = entry.into_path(); + if file_type.is_file() && is_secret_scan_sidecar(&path) { + out.push(path); + } + } + out +} + +fn is_secret_scan_sidecar(path: &Path) -> bool { + let file_name = path.file_name().and_then(|name| name.to_str()); + file_name.is_some_and(|name| name == ".env" || name.starts_with(".env.")) + || path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("env")) +} + fn is_skipped_dir(entry: &DirEntry) -> bool { entry .file_type() diff --git a/crates/clarion-cli/src/secret_scan.rs b/crates/clarion-cli/src/secret_scan.rs index 0e0e08cb..3391bd28 100644 --- a/crates/clarion-cli/src/secret_scan.rs +++ b/crates/clarion-cli/src/secret_scan.rs @@ -38,9 +38,21 @@ pub(crate) struct SecretScanOutcome { pub(crate) briefing_blocks: BTreeMap, findings: Vec, override_files: Vec, + scanned_files: BTreeSet, } impl SecretScanOutcome { + pub(crate) fn scanned_files(&self) -> &BTreeSet { + &self.scanned_files + } + + pub(crate) fn finding_files(&self) -> BTreeSet { + self.findings + .iter() + .map(|finding| finding.file_path.clone()) + .collect() + } + pub(crate) fn augment_stats(&self, stats: &mut serde_json::Value) { if self.override_files.is_empty() { return; @@ -65,8 +77,11 @@ pub(crate) fn pre_ingest( let mut per_file = Vec::new(); let mut all_allowed = Vec::new(); let mut baseline_matches = Vec::new(); + let mut scanned_files = BTreeSet::new(); for file in source_files { + let canonical_file = canonical_or_original(file); + scanned_files.insert(canonical_file.clone()); let buf = fs::read(file).with_context(|| format!("read {}", file.display()))?; let detections = scanner.scan_bytes(&buf); let SuppressionResult { @@ -79,11 +94,11 @@ pub(crate) fn pre_ingest( allowed .iter() .cloned() - .map(|detection| (canonical_or_original(file), detection)), + .map(|detection| (canonical_file.clone(), detection)), ); } baseline_matches.extend(fired_entries.into_iter().map(|entry| PendingFinding { - file_path: entry.file_path.clone(), + file_path: normalize_project_path(project_root, &entry.file_path), rule_id: BASELINE_MATCH, kind: "fact", severity: "INFO", @@ -100,7 +115,7 @@ pub(crate) fn pre_ingest( "rule": entry.entry.rule_type, }), })); - per_file.push((canonical_or_original(file), allowed)); + per_file.push((canonical_file, allowed)); } let override_confirmed = confirm_override_if_needed(&all_allowed, options); @@ -147,6 +162,7 @@ pub(crate) fn pre_ingest( briefing_blocks, findings, override_files: override_files.into_iter().collect(), + scanned_files, }) } @@ -154,22 +170,25 @@ fn load_baseline_for_scan(project_root: &Path) -> Result<(Baseline, Vec Ok((baseline, Vec::new())), - Err(BaselineError::MissingJustification { file, line }) => Ok(( + Err(BaselineError::MissingJustifications { entries }) => Ok(( Baseline::empty(), - vec![PendingFinding { - file_path: project_root.join(file.clone()), - rule_id: BASELINE_NO_JUSTIFICATION, - kind: "defect", - severity: "ERROR", - confidence: Some(1.0), - confidence_basis: Some("baseline_schema"), - message: format!( - "Secret baseline entry missing justification at {}:{}", - file.display(), - line - ), - evidence: json!({"file_path": file, "line_number": line}), - }], + entries + .into_iter() + .map(|entry| PendingFinding { + file_path: normalize_project_path(project_root, &entry.file), + rule_id: BASELINE_NO_JUSTIFICATION, + kind: "defect", + severity: "ERROR", + confidence: Some(1.0), + confidence_basis: Some("baseline_schema"), + message: format!( + "Secret baseline entry missing justification at {}:{}", + entry.file.display(), + entry.line + ), + evidence: json!({"file_path": entry.file, "line_number": entry.line}), + }) + .collect(), )), Err(err) => Err(err).context("load secret baseline"), } @@ -216,6 +235,10 @@ fn canonical_or_original(path: &Path) -> PathBuf { path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) } +fn normalize_project_path(root: &Path, path: &Path) -> PathBuf { + canonical_or_original(&root.join(path)) +} + fn display_relative(root: &Path, path: &Path) -> String { path.strip_prefix(root) .unwrap_or(path) diff --git a/crates/clarion-cli/tests/secret_scan.rs b/crates/clarion-cli/tests/secret_scan.rs index 0399b758..ad3bc7d1 100644 --- a/crates/clarion-cli/tests/secret_scan.rs +++ b/crates/clarion-cli/tests/secret_scan.rs @@ -9,6 +9,7 @@ fn clarion_bin() -> Command { const PLUGIN_SCRIPT: &str = r#"#!/usr/bin/python3 import json +import os import pathlib import re import sys @@ -54,6 +55,7 @@ while True: }) elif method == "analyze_file": path = msg["params"]["file_path"] + source_path = os.environ.get("SECRETFIXTURE_SOURCE_OVERRIDE", path) name = "file_" + re.sub(r"[^A-Za-z0-9_]", "_", pathlib.Path(path).name) write_frame({ "jsonrpc": "2.0", @@ -64,7 +66,7 @@ while True: "id": "secretfixture:module:" + name, "kind": "module", "qualified_name": name, - "source": {"file_path": path}, + "source": {"file_path": source_path}, } ], "edges": [], @@ -192,6 +194,77 @@ fn secret_file_persists_finding_and_briefing_block() { assert_eq!(count, 1); } +#[test] +fn dotenv_sidecar_persists_finding_with_core_file_anchor() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write(project.path().join("clean.sec"), b"nothing to see\n").unwrap(); + std::fs::write( + project.path().join(".env"), + b"aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'\n", + ) + .unwrap(); + + clarion_bin() + .arg("analyze") + .arg(project.path()) + .env("PATH", plugin_path(plugin.path())) + .assert() + .success(); + + let db = conn(project.path()); + let finding_count: i64 = db + .query_row( + "SELECT COUNT(*) FROM findings WHERE rule_id = 'CLA-SEC-SECRET-DETECTED'", + [], + |row| row.get(0), + ) + .unwrap(); + let anchor_count: i64 = db + .query_row( + "SELECT COUNT(*) FROM entities \ + WHERE plugin_id = 'core' \ + AND kind = 'file' \ + AND source_file_path LIKE '%.env' \ + AND json_extract(properties, '$.briefing_blocked') = 'secret_present'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(finding_count, 1); + assert_eq!(anchor_count, 1); +} + +#[test] +fn plugin_entity_for_unscanned_source_is_briefing_blocked() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write(project.path().join("clean.sec"), b"nothing to see\n").unwrap(); + let unscanned = project.path().join("notes.txt"); + std::fs::write(&unscanned, b"ordinary notes\n").unwrap(); + + clarion_bin() + .arg("analyze") + .arg(project.path()) + .env("PATH", plugin_path(plugin.path())) + .env("SECRETFIXTURE_SOURCE_OVERRIDE", &unscanned) + .assert() + .success(); + + let blocked: String = conn(project.path()) + .query_row( + "SELECT json_extract(properties, '$.briefing_blocked') FROM entities", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(blocked, "unscanned_source"); +} + #[test] fn baseline_suppresses_secret_and_emits_audit_match() { let project = tempfile::tempdir().unwrap(); @@ -273,6 +346,11 @@ results: hashed_secret: "25910f981e85ca04baf359199dd0bd4a3ae738b6" line_number: 1 is_secret: false + "stale.sec": + - type: "AWS Access Key" + hashed_secret: "25910f981e85ca04baf359199dd0bd4a3ae738b6" + line_number: 9 + is_secret: false "#, ) .unwrap(); @@ -291,7 +369,7 @@ results: |row| row.get(0), ) .unwrap(); - assert_eq!(count, 1); + assert_eq!(count, 2); } #[test] @@ -370,6 +448,36 @@ fn non_tty_override_without_confirmation_exits_78_before_run_start() { assert_eq!(run_count, 0); } +#[test] +fn non_tty_override_with_wrong_confirmation_exits_78_before_run_start() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write( + project.path().join("leaky.sec"), + b"aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'\n", + ) + .unwrap(); + + let assert = clarion_bin() + .args([ + "analyze", + "--allow-unredacted-secrets", + "--confirm-allow-unredacted-secrets=oops", + ]) + .arg(project.path()) + .env("PATH", plugin_path(plugin.path())) + .assert() + .code(78); + let stderr = String::from_utf8_lossy(&assert.get_output().stderr); + assert!(stderr.contains("CLA-INFRA-SECRET-OVERRIDE-UNCONFIRMED")); + let run_count: i64 = conn(project.path()) + .query_row("SELECT COUNT(*) FROM runs", [], |row| row.get(0)) + .unwrap(); + assert_eq!(run_count, 0); +} + #[test] #[ignore = "TTY override confirmation needs an interactive terminal; WS-D owns the manual smoke."] fn tty_override_confirmation_manual_smoke() {} diff --git a/crates/clarion-core/src/plugin/host.rs b/crates/clarion-core/src/plugin/host.rs index 2afed830..f476a586 100644 --- a/crates/clarion-core/src/plugin/host.rs +++ b/crates/clarion-core/src/plugin/host.rs @@ -416,6 +416,8 @@ where stderr_tail: Option>>>, /// Canonical source paths whose entities must not be sent for LLM briefing. briefing_blocks: BTreeMap, + /// Canonical source paths that were covered by the core pre-ingest scanner. + scanned_source_files: BTreeSet, } /// Size of the stderr ring buffer kept for diagnostics. 64 KiB holds @@ -664,6 +666,7 @@ impl PluginHost { ontology_version: None, stderr_tail: None, briefing_blocks: BTreeMap::new(), + scanned_source_files: BTreeSet::new(), } } @@ -698,6 +701,15 @@ impl PluginHost { self.briefing_blocks = blocks; } + /// Install canonical source paths covered by the pre-ingest scanner. + /// + /// If a plugin returns an entity for a jailed in-project path that is not in + /// this set, the entity is policy-blocked from LLM briefing because its file + /// bytes have not passed the scanner. + pub fn set_scanned_source_files(&mut self, files: BTreeSet) { + self.scanned_source_files = files; + } + /// Perform the `initialize` → `initialized` handshake. /// /// Steps: @@ -955,10 +967,18 @@ impl PluginHost { } fn apply_briefing_block(&self, raw: &mut RawEntity, jailed: &str) { - if let Some(reason) = self.briefing_blocks.get(Path::new(jailed)) { + let jailed_path = Path::new(jailed); + let reason = self + .briefing_blocks + .get(jailed_path) + .map(String::as_str) + .or_else(|| { + (!self.scanned_source_files.contains(jailed_path)).then_some("unscanned_source") + }); + if let Some(reason) = reason { raw.extra.insert( "briefing_blocked".to_owned(), - serde_json::Value::String(reason.clone()), + serde_json::Value::String(reason.to_owned()), ); } } diff --git a/crates/clarion-mcp/src/lib.rs b/crates/clarion-mcp/src/lib.rs index 78cfc2bd..3bc3d9ef 100644 --- a/crates/clarion-mcp/src/lib.rs +++ b/crates/clarion-mcp/src/lib.rs @@ -2353,13 +2353,18 @@ fn summary_scope_deferred(entity: &EntityRow) -> Value { } fn summary_briefing_blocked(entity: &EntityRow, reason: &str) -> Value { + let remediation = if reason == "unscanned_source" { + "Entity source file was not covered by the pre-ingest secret scan. Re-run with scanner coverage for that path or fix the plugin source path before requesting a summary." + } else { + "File flagged by pre-ingest secret scan. Fix the secret or whitelist via .clarion/secrets-baseline.yaml. See ADR-013." + }; success_envelope(json!({ "available": false, "entity_id": entity.id, "entity": entity_json(entity), "summary": null, "briefing_blocked": reason, - "remediation": "File flagged by pre-ingest secret scan. Fix the secret or whitelist via .clarion/secrets-baseline.yaml. See ADR-013." + "remediation": remediation })) } diff --git a/crates/clarion-scanner/src/baseline.rs b/crates/clarion-scanner/src/baseline.rs index ef1ea383..25a80c1c 100644 --- a/crates/clarion-scanner/src/baseline.rs +++ b/crates/clarion-scanner/src/baseline.rs @@ -29,6 +29,12 @@ pub struct BaselineMatch { pub entry: BaselineEntry, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct BaselineEntryIssue { + pub file: PathBuf, + pub line: u32, +} + #[derive(Debug, Clone, PartialEq, Eq)] pub struct SuppressionResult { pub allowed: Vec, @@ -40,8 +46,8 @@ pub struct SuppressionResult { pub enum BaselineError { #[error("baseline version mismatch: expected 1.0, got {0}")] UnsupportedVersion(String), - #[error("baseline entry missing required field 'justification' at {file}:{line}")] - MissingJustification { file: PathBuf, line: u32 }, + #[error("baseline entries missing required field 'justification'")] + MissingJustifications { entries: Vec }, #[error("baseline entry has invalid hashed_secret at {file}:{line}: {details}")] InvalidHash { file: PathBuf, @@ -131,17 +137,32 @@ impl Baseline { if raw.version != "1.0" { return Err(BaselineError::UnsupportedVersion(raw.version)); } + let mut missing_justifications = Vec::new(); + for (file, raw_entries) in &raw.results { + for entry in raw_entries { + if entry + .justification + .as_ref() + .is_none_or(|value| value.trim().is_empty()) + { + missing_justifications.push(BaselineEntryIssue { + file: file.clone(), + line: entry.line_number, + }); + } + } + } + if !missing_justifications.is_empty() { + return Err(BaselineError::MissingJustifications { + entries: missing_justifications, + }); + } + let mut entries = BTreeMap::new(); for (file, raw_entries) in raw.results { let mut converted = Vec::new(); for entry in raw_entries { let justification = entry.justification.unwrap_or_default(); - if justification.trim().is_empty() { - return Err(BaselineError::MissingJustification { - file, - line: entry.line_number, - }); - } let hashed_secret = hex_decode_20(&entry.hashed_secret).map_err(|details| { BaselineError::InvalidHash { file: file.clone(), diff --git a/crates/clarion-scanner/src/lib.rs b/crates/clarion-scanner/src/lib.rs index abcb6924..94d939bd 100644 --- a/crates/clarion-scanner/src/lib.rs +++ b/crates/clarion-scanner/src/lib.rs @@ -9,7 +9,8 @@ mod entropy; mod patterns; pub use baseline::{ - Baseline, BaselineEntry, BaselineError, BaselineMatch, SuppressionResult, load_baseline, + Baseline, BaselineEntry, BaselineEntryIssue, BaselineError, BaselineMatch, SuppressionResult, + load_baseline, }; pub use entropy::EntropyTuning; pub use patterns::{PatternMeta, Scanner}; diff --git a/crates/clarion-scanner/tests/scanner.rs b/crates/clarion-scanner/tests/scanner.rs index c031728a..9341432e 100644 --- a/crates/clarion-scanner/tests/scanner.rs +++ b/crates/clarion-scanner/tests/scanner.rs @@ -189,10 +189,38 @@ results: "#, ) .expect_err("missing justification should fail"); - assert!(matches!( - err, - BaselineError::MissingJustification { line: 42, .. } - )); + let BaselineError::MissingJustifications { entries } = err else { + panic!("expected MissingJustifications, got {err:?}"); + }; + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].line, 42); +} + +#[test] +fn baseline_missing_justification_reports_all_entries() { + let err = Baseline::from_yaml_str( + r#" +version: "1.0" +results: + "src/one.py": + - type: "AWS Access Key" + hashed_secret: "0123456789abcdef0123456789abcdef01234567" + line_number: 4 + is_secret: false + "src/two.py": + - type: "AWS Access Key" + hashed_secret: "0123456789abcdef0123456789abcdef01234567" + line_number: 8 + is_secret: false +"#, + ) + .expect_err("all missing justifications should be reported"); + let BaselineError::MissingJustifications { entries } = err else { + panic!("expected MissingJustifications, got {err:?}"); + }; + assert_eq!(entries.len(), 2); + assert_eq!(entries[0].line, 4); + assert_eq!(entries[1].line, 8); } #[test] diff --git a/docs/operator/secret-scanning.md b/docs/operator/secret-scanning.md index bd1b66ae..63410505 100644 --- a/docs/operator/secret-scanning.md +++ b/docs/operator/secret-scanning.md @@ -6,7 +6,7 @@ Clarion scans source files before any file content can be used for LLM summaries Blocking is file-level. If `src/config.py` contains a detected key, entities from that file remain queryable through structural tools, but the `summary` tool returns a policy envelope instead of calling the LLM provider or writing `summary_cache`. -Files outside the analyzed plugin extension set are not scanned because they are not candidates for plugin dispatch in that run. +Plugin source files and `.env` sidecars are scanned. If a plugin reports an entity for some other in-project path that was not covered by the scanner, Clarion marks that entity `briefing_blocked: unscanned_source` so source bytes cannot reach the LLM provider without a prior scan. ## Whitelist A False Positive diff --git a/tests/e2e/wp5_secret_scan.sh b/tests/e2e/wp5_secret_scan.sh index 13b87935..e50ab06b 100755 --- a/tests/e2e/wp5_secret_scan.sh +++ b/tests/e2e/wp5_secret_scan.sh @@ -94,15 +94,19 @@ TOML log "scratch project: $DEMO_DIR" "$CLARION_BIN" install --path "$DEMO_DIR" printf "aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'\n" > "$DEMO_DIR/leaky.sec" +printf "aws_access_key_id = 'AKIAIOSFODNN7EXAMPLE'\n" > "$DEMO_DIR/.env" PATH="$PLUGIN_DIR" "$CLARION_BIN" analyze "$DEMO_DIR" DB="$DEMO_DIR/.clarion/clarion.db" -BLOCKED=$(sqlite3 "$DB" "select json_extract(properties, '\$.briefing_blocked') from entities;") -[ "$BLOCKED" = "secret_present" ] || fail "expected briefing_blocked secret_present, got $BLOCKED" +BLOCKED=$(sqlite3 "$DB" "select count(*) from entities where json_extract(properties, '\$.briefing_blocked') = 'secret_present';") +[ "$BLOCKED" = "2" ] || fail "expected two briefing_blocked secret_present entities, got $BLOCKED" FINDINGS=$(sqlite3 "$DB" "select count(*) from findings where rule_id = 'CLA-SEC-SECRET-DETECTED';") -[ "$FINDINGS" = "1" ] || fail "expected one CLA-SEC-SECRET-DETECTED finding, got $FINDINGS" +[ "$FINDINGS" = "2" ] || fail "expected two CLA-SEC-SECRET-DETECTED findings, got $FINDINGS" + +DOTENV_ANCHOR=$(sqlite3 "$DB" "select count(*) from entities where plugin_id = 'core' and kind = 'file' and source_file_path = '$DEMO_DIR/.env' and json_extract(properties, '\$.briefing_blocked') = 'secret_present';") +[ "$DOTENV_ANCHOR" = "1" ] || fail "expected .env core file anchor with briefing_blocked secret_present, got $DOTENV_ANCHOR" RUN_STATUS=$(sqlite3 "$DB" "select status from runs;") [ "$RUN_STATUS" = "completed" ] || fail "expected completed run, got $RUN_STATUS" From d2d15b1263dc52e75f5588f842a915007168fb34 Mon Sep 17 00:00:00 2001 From: John Morrissey <544926+tachyon-beep@users.noreply.github.com> Date: Tue, 19 May 2026 03:18:48 +1000 Subject: [PATCH 3/4] fix(wp5): harden secret scanner review gaps --- crates/clarion-cli/src/main.rs | 14 +- crates/clarion-cli/src/secret_scan.rs | 7 +- crates/clarion-cli/tests/secret_scan.rs | 28 ++++ crates/clarion-mcp/src/lib.rs | 4 + crates/clarion-mcp/tests/storage_tools.rs | 55 +++++++ crates/clarion-scanner/src/baseline.rs | 39 ++++- crates/clarion-scanner/src/lib.rs | 1 + crates/clarion-scanner/src/patterns.rs | 44 +++-- crates/clarion-scanner/tests/scanner.rs | 186 +++++++++++++++++++++- 9 files changed, 349 insertions(+), 29 deletions(-) diff --git a/crates/clarion-cli/src/main.rs b/crates/clarion-cli/src/main.rs index 9b3c1a5a..71604599 100644 --- a/crates/clarion-cli/src/main.rs +++ b/crates/clarion-cli/src/main.rs @@ -11,14 +11,16 @@ use anyhow::Result; use clap::Parser; fn main() -> Result<()> { - // Load .env from CWD or any ancestor directory, before tracing setup so a + let cli = cli::Cli::parse(); + // Load .env before tracing setup for operator-facing commands so a // .env-supplied RUST_LOG is in effect by the time the filter is built. - // Existing process env vars win over .env values (dotenvy default), so an - // explicit `OPENROUTER_API_KEY=… clarion serve` still beats a checked-in - // dev .env. Missing .env is not an error — silently skip. - let _ = dotenvy::dotenv(); + // `analyze` is deliberately excluded: project .env contents are scanned + // as source sidecars and must not be imported into plugin subprocess + // environments before WP5's pre-ingest secret gate runs. + if !matches!(&cli.command, cli::Command::Analyze { .. }) { + let _ = dotenvy::dotenv(); + } init_tracing(); - let cli = cli::Cli::parse(); match cli.command { cli::Command::Install { force, path } => install::run(&path, force), cli::Command::Analyze { diff --git a/crates/clarion-cli/src/secret_scan.rs b/crates/clarion-cli/src/secret_scan.rs index 3391bd28..4dee1f48 100644 --- a/crates/clarion-cli/src/secret_scan.rs +++ b/crates/clarion-cli/src/secret_scan.rs @@ -84,11 +84,12 @@ pub(crate) fn pre_ingest( scanned_files.insert(canonical_file.clone()); let buf = fs::read(file).with_context(|| format!("read {}", file.display()))?; let detections = scanner.scan_bytes(&buf); + let baseline_file = project_relative_path(project_root, &canonical_file); let SuppressionResult { allowed, fired_entries, .. - } = baseline.suppress(detections, file); + } = baseline.suppress(detections, &baseline_file); if !allowed.is_empty() { all_allowed.extend( allowed @@ -239,6 +240,10 @@ fn normalize_project_path(root: &Path, path: &Path) -> PathBuf { canonical_or_original(&root.join(path)) } +fn project_relative_path(root: &Path, path: &Path) -> PathBuf { + path.strip_prefix(root).unwrap_or(path).to_path_buf() +} + fn display_relative(root: &Path, path: &Path) -> String { path.strip_prefix(root) .unwrap_or(path) diff --git a/crates/clarion-cli/tests/secret_scan.rs b/crates/clarion-cli/tests/secret_scan.rs index ad3bc7d1..91ebefd4 100644 --- a/crates/clarion-cli/tests/secret_scan.rs +++ b/crates/clarion-cli/tests/secret_scan.rs @@ -54,6 +54,11 @@ while True: }, }) elif method == "analyze_file": + if ( + os.environ.get("SECRETFIXTURE_ASSERT_ENV_ABSENT") + and os.environ.get("CLARION_DOTENV_SENTINEL") is not None + ): + raise SystemExit(42) path = msg["params"]["file_path"] source_path = os.environ.get("SECRETFIXTURE_SOURCE_OVERRIDE", path) name = "file_" + re.sub(r"[^A-Za-z0-9_]", "_", pathlib.Path(path).name) @@ -237,6 +242,29 @@ fn dotenv_sidecar_persists_finding_with_core_file_anchor() { assert_eq!(anchor_count, 1); } +#[test] +fn analyze_does_not_load_dotenv_into_plugin_environment() { + let project = tempfile::tempdir().unwrap(); + let plugin = tempfile::tempdir().unwrap(); + write_secret_fixture_plugin(plugin.path()); + install_project(project.path()); + std::fs::write(project.path().join("clean.sec"), b"nothing to see\n").unwrap(); + std::fs::write( + project.path().join(".env"), + b"CLARION_DOTENV_SENTINEL=ordinaryvalue\n", + ) + .unwrap(); + + clarion_bin() + .arg("analyze") + .arg(".") + .current_dir(project.path()) + .env("PATH", plugin_path(plugin.path())) + .env("SECRETFIXTURE_ASSERT_ENV_ABSENT", "1") + .assert() + .success(); +} + #[test] fn plugin_entity_for_unscanned_source_is_briefing_blocked() { let project = tempfile::tempdir().unwrap(); diff --git a/crates/clarion-mcp/src/lib.rs b/crates/clarion-mcp/src/lib.rs index 3bc3d9ef..6c735749 100644 --- a/crates/clarion-mcp/src/lib.rs +++ b/crates/clarion-mcp/src/lib.rs @@ -856,6 +856,10 @@ impl ServerState { return Ok(InferredDispatchStats::default()); }; + if briefing_block_reason(&read.caller).is_some() { + return Ok(InferredDispatchStats::default()); + } + if let Some(cached) = read.cached.clone() { return self.materialize_cached_inferred(read, cached).await; } diff --git a/crates/clarion-mcp/tests/storage_tools.rs b/crates/clarion-mcp/tests/storage_tools.rs index 43c3dcc5..069f842a 100644 --- a/crates/clarion-mcp/tests/storage_tools.rs +++ b/crates/clarion-mcp/tests/storage_tools.rs @@ -1707,6 +1707,61 @@ async fn callers_of_inferred_dispatches_and_materializes_recording_result() { handle.await.unwrap().unwrap(); } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn callers_of_inferred_skips_briefing_blocked_callers_without_llm() { + let (project, db_path) = open_project(); + let conn = Connection::open(&db_path).unwrap(); + add_dynamic_source(project.path()); + let source_path = project.path().join("demo.py"); + insert_entity( + &conn, + "python:function:demo.dynamic", + "function", + &source_path, + Some((9, 10)), + Some("python:module:demo"), + ); + insert_unresolved_call_site( + &conn, + "python:function:demo.entry", + "site-dynamic", + "dynamic", + ); + conn.execute( + "UPDATE entities SET properties = ?1 WHERE id = 'python:function:demo.entry'", + params![json!({"briefing_blocked": "secret_present"}).to_string()], + ) + .expect("mark caller briefing-blocked"); + drop(conn); + + let (writer, handle) = Writer::spawn(db_path.clone(), 50, 256).unwrap(); + let provider = Arc::new(AnyInferredProvider::new( + r#"{"edges":[{"site_key":"site-dynamic","target_id":"python:function:demo.dynamic","confidence":0.91,"rationale":"name match"}]}"#, + )); + let state = state_for_summary( + project.path(), + &db_path, + &writer, + provider.clone(), + llm_config(), + ); + + let envelope = call_tool( + &state, + "callers_of", + json!({"id": "python:function:demo.dynamic", "confidence": "inferred"}), + ) + .await; + + assert_eq!(envelope["ok"], true); + assert_eq!(envelope["result"]["callers"].as_array().unwrap().len(), 0); + assert!(provider.invocations().is_empty()); + + drop(state); + drop(writer); + handle.await.unwrap().unwrap(); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn inferred_dispatch_prompt_uses_caller_source_range_not_whole_file() { let (project, db_path) = open_project(); diff --git a/crates/clarion-scanner/src/baseline.rs b/crates/clarion-scanner/src/baseline.rs index 25a80c1c..ddf2f20e 100644 --- a/crates/clarion-scanner/src/baseline.rs +++ b/crates/clarion-scanner/src/baseline.rs @@ -1,7 +1,7 @@ use std::{ collections::{BTreeMap, BTreeSet}, fs, - path::{Path, PathBuf}, + path::{Component, Path, PathBuf}, }; use serde::{Deserialize, Serialize}; @@ -48,6 +48,8 @@ pub enum BaselineError { UnsupportedVersion(String), #[error("baseline entries missing required field 'justification'")] MissingJustifications { entries: Vec }, + #[error("baseline path must be repository-relative and stay within the project: {file}")] + InvalidPath { file: PathBuf }, #[error("baseline entry has invalid hashed_secret at {file}:{line}: {details}")] InvalidHash { file: PathBuf, @@ -107,9 +109,11 @@ impl Baseline { } if entry.hashed_secret == detection.hashed_secret && entry.line_number == detection.line_number + && entry.rule_type == detection.detect_secrets_type { let key = ( (*baseline_path).clone(), + entry.rule_type.clone(), entry.hashed_secret, entry.line_number, ); @@ -137,6 +141,9 @@ impl Baseline { if raw.version != "1.0" { return Err(BaselineError::UnsupportedVersion(raw.version)); } + for file in raw.results.keys() { + validate_baseline_path(file)?; + } let mut missing_justifications = Vec::new(); for (file, raw_entries) in &raw.results { for entry in raw_entries { @@ -189,12 +196,30 @@ impl Baseline { fn entries_for(&self, file: &Path) -> Vec<(&PathBuf, &BaselineEntry)> { self.entries .iter() - .filter(|(candidate, _)| baseline_path_matches(file, candidate)) + .filter(|(candidate, _)| candidate.as_path() == file) .flat_map(|(path, entries)| entries.iter().map(move |entry| (path, entry))) .collect() } } +fn validate_baseline_path(file: &Path) -> Result<(), BaselineError> { + let invalid = file.as_os_str().is_empty() + || file.is_absolute() + || file.components().any(|component| { + matches!( + component, + Component::ParentDir | Component::RootDir | Component::Prefix(_) + ) + }); + if invalid { + Err(BaselineError::InvalidPath { + file: file.to_path_buf(), + }) + } else { + Ok(()) + } +} + #[derive(Debug, Serialize, Deserialize)] struct RawBaseline { version: String, @@ -208,11 +233,15 @@ struct RawBaselineEntry { rule_type: String, hashed_secret: String, line_number: u32, - #[serde(default)] + #[serde(default = "default_is_secret")] is_secret: bool, justification: Option, } +fn default_is_secret() -> bool { + true +} + impl From<&Baseline> for RawBaseline { fn from(value: &Baseline) -> Self { let results = value @@ -240,7 +269,3 @@ impl From<&Baseline> for RawBaseline { } } } - -fn baseline_path_matches(file: &Path, baseline_path: &Path) -> bool { - file == baseline_path || file.ends_with(baseline_path) -} diff --git a/crates/clarion-scanner/src/lib.rs b/crates/clarion-scanner/src/lib.rs index 94d939bd..a9e64b53 100644 --- a/crates/clarion-scanner/src/lib.rs +++ b/crates/clarion-scanner/src/lib.rs @@ -19,6 +19,7 @@ pub use patterns::{PatternMeta, Scanner}; #[derive(Debug, Clone, PartialEq, Eq)] pub struct Detection { pub rule_id: &'static str, + pub detect_secrets_type: &'static str, pub category: SecretCategory, pub byte_offset: usize, pub line_number: u32, diff --git a/crates/clarion-scanner/src/patterns.rs b/crates/clarion-scanner/src/patterns.rs index 9e802cd0..cacd82bc 100644 --- a/crates/clarion-scanner/src/patterns.rs +++ b/crates/clarion-scanner/src/patterns.rs @@ -63,7 +63,7 @@ impl Scanner { compiled_patterns, entropy_b64: EntropyTuning::BASE64, entropy_hex: EntropyTuning::HEX, - entropy_b64_re: Regex::new(r"\b[A-Za-z0-9+/]{20,}={0,2}\b") + entropy_b64_re: Regex::new(r"[A-Za-z0-9+/]{20,}={0,2}") .expect("base64 candidate regex compiles"), entropy_hex_re: Regex::new(r"\b[a-fA-F0-9]{40,}\b") .expect("hex candidate regex compiles"), @@ -133,14 +133,16 @@ impl Scanner { let source = String::from_utf8_lossy(bytes); for candidate in self.entropy_b64_re.find_iter(&source) { let candidate_bytes = &source.as_bytes()[candidate.start()..candidate.end()]; - if looks_like_sha256_base64(&source, candidate.start(), candidate.end()) { - continue; - } - if !range_overlaps(candidate.start(), candidate.end(), named_ranges) + if base64_candidate_has_boundaries( + source.as_bytes(), + candidate.start(), + candidate.end(), + ) && !range_overlaps(candidate.start(), candidate.end(), named_ranges) && self.entropy_b64.accepts(candidate_bytes) { detections.push(entropy_detection( "HighEntropyBase64", + "Base64 High Entropy String", bytes, candidate.start(), candidate.end(), @@ -154,6 +156,7 @@ impl Scanner { { detections.push(entropy_detection( "HighEntropyHex", + "Hex High Entropy String", bytes, candidate.start(), candidate.end(), @@ -167,6 +170,7 @@ fn detection_from_match(meta: &PatternMeta, bytes: &[u8], start: usize, end: usi let matched = &bytes[start..end]; Detection { rule_id: meta.rule_id, + detect_secrets_type: meta.detect_secrets_type, category: meta.category, byte_offset: start, line_number: line_number_for_offset(bytes, start), @@ -175,9 +179,16 @@ fn detection_from_match(meta: &PatternMeta, bytes: &[u8], start: usize, end: usi } } -fn entropy_detection(rule_id: &'static str, bytes: &[u8], start: usize, end: usize) -> Detection { +fn entropy_detection( + rule_id: &'static str, + detect_secrets_type: &'static str, + bytes: &[u8], + start: usize, + end: usize, +) -> Detection { Detection { rule_id, + detect_secrets_type, category: SecretCategory::HighEntropy, byte_offset: start, line_number: line_number_for_offset(bytes, start), @@ -262,7 +273,7 @@ fn default_pattern_meta() -> Vec { rule_id: "PrivateKeyHeader", detect_secrets_type: "Private Key", category: SecretCategory::PrivateKey, - pattern: r"-----BEGIN (?:RSA|EC|DSA|OPENSSH|PGP|ENCRYPTED) PRIVATE KEY-----", + pattern: r"-----BEGIN (?:(?:RSA|EC|DSA|OPENSSH|ENCRYPTED) PRIVATE KEY|PRIVATE KEY|PGP PRIVATE KEY BLOCK)-----", capture_group: None, }, PatternMeta { @@ -281,6 +292,19 @@ fn range_overlaps(start: usize, end: usize, ranges: &[(usize, usize)]) -> bool { .any(|(range_start, range_end)| start < *range_end && end > *range_start) } +fn base64_candidate_has_boundaries(bytes: &[u8], start: usize, end: usize) -> bool { + let before_ok = start == 0 || !is_base64_candidate_byte(bytes[start - 1]); + let after_ok = end == bytes.len() || !is_base64_candidate_byte(bytes[end]); + before_ok && after_ok +} + +fn is_base64_candidate_byte(byte: u8) -> bool { + matches!( + byte, + b'A'..=b'Z' | b'a'..=b'z' | b'0'..=b'9' | b'+' | b'/' | b'=' + ) +} + fn line_is_comment(bytes: &[u8], offset: usize) -> bool { let line_start = bytes .get(..offset.min(bytes.len())) @@ -292,9 +316,3 @@ fn line_is_comment(bytes: &[u8], offset: usize) -> bool { .find(|byte| !byte.is_ascii_whitespace()) == Some(b'#') } - -fn looks_like_sha256_base64(source: &str, start: usize, end: usize) -> bool { - let candidate = &source[start..end]; - (candidate.len() == 44 && candidate.ends_with('=')) - || (candidate.len() == 43 && source.as_bytes().get(end) == Some(&b'=')) -} diff --git a/crates/clarion-scanner/tests/scanner.rs b/crates/clarion-scanner/tests/scanner.rs index 9341432e..4a6ca9ff 100644 --- a/crates/clarion-scanner/tests/scanner.rs +++ b/crates/clarion-scanner/tests/scanner.rs @@ -1,4 +1,5 @@ use clarion_scanner::{Baseline, BaselineError, Scanner, load_baseline}; +use sha1::{Digest, Sha1}; fn rules_for(input: &str) -> Vec<&'static str> { Scanner::new() @@ -55,7 +56,10 @@ fn named_patterns_detect_expected_credentials() { "JwtToken", ); assert_detects("-----BEGIN RSA PRIVATE KEY-----", "PrivateKeyHeader"); + assert_detects("-----BEGIN DSA PRIVATE KEY-----", "PrivateKeyHeader"); assert_detects("-----BEGIN OPENSSH PRIVATE KEY-----", "PrivateKeyHeader"); + assert_detects("-----BEGIN PRIVATE KEY-----", "PrivateKeyHeader"); + assert_detects("-----BEGIN PGP PRIVATE KEY BLOCK-----", "PrivateKeyHeader"); } #[test] @@ -80,12 +84,46 @@ fn entropy_detection_has_expected_bounds() { "uuid = 123e4567-e89b-12d3-a456-426614174000", "HighEntropyHex", ); - assert_not_detects( + assert_detects( "checksum = 47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", "HighEntropyBase64", ); } +#[test] +fn padded_base64_detection_hashes_the_full_literal_for_baselines() { + let secret = "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU="; + let source = format!("checksum = {secret}\n"); + let detections = Scanner::new().scan_bytes(source.as_bytes()); + let detection = detections + .iter() + .find(|detection| detection.rule_id == "HighEntropyBase64") + .expect("padded base64 detection") + .clone(); + + assert_eq!(detection.matched_len, secret.len()); + assert_eq!(hex20(detection.hashed_secret), sha1_hex(secret.as_bytes())); + + let baseline = Baseline::from_yaml_str(&format!( + r#" +version: "1.0" +results: + "src/checksums.txt": + - type: "Base64 High Entropy String" + hashed_secret: "{}" + line_number: 1 + is_secret: false + justification: "Known non-secret digest checked into the fixture." +"#, + sha1_hex(secret.as_bytes()) + )) + .expect("baseline parses"); + + let result = baseline.suppress(detections, std::path::Path::new("src/checksums.txt")); + assert!(result.allowed.is_empty()); + assert_eq!(result.suppressed.len(), 1); +} + #[test] fn contextual_credentials_detect_assignments_but_not_comments_or_hashes() { assert_detects( @@ -140,12 +178,101 @@ results: )) .expect("baseline parses"); - let result = baseline.suppress(detections, std::path::Path::new("/repo/src/demo.py")); + let result = baseline.suppress(detections, std::path::Path::new("src/demo.py")); assert!(result.allowed.is_empty()); assert_eq!(result.suppressed.len(), 1); assert_eq!(result.fired_entries.len(), 1); } +#[test] +fn baseline_does_not_suppress_when_detector_type_differs() { + let scanner = Scanner::new(); + let detections = scanner.scan_bytes(b"key = 'AKIAIOSFODNN7EXAMPLE'\n"); + let detection = detections + .iter() + .find(|detection| detection.rule_id == "AwsAccessKeyId") + .expect("AWS detection") + .clone(); + let baseline = Baseline::from_yaml_str(&format!( + r#" +version: "1.0" +results: + "src/demo.py": + - type: "GitHub Token" + hashed_secret: "{}" + line_number: 1 + is_secret: false + justification: "Different detector type must not suppress this match." +"#, + hex20(detection.hashed_secret) + )) + .expect("baseline parses"); + + let result = baseline.suppress(detections, std::path::Path::new("src/demo.py")); + assert_eq!(result.allowed.len(), 1); + assert!(result.suppressed.is_empty()); + assert!(result.fired_entries.is_empty()); +} + +#[test] +fn baseline_without_explicit_is_secret_false_does_not_suppress() { + let scanner = Scanner::new(); + let detections = scanner.scan_bytes(b"key = 'AKIAIOSFODNN7EXAMPLE'\n"); + let detection = detections + .iter() + .find(|detection| detection.rule_id == "AwsAccessKeyId") + .expect("AWS detection") + .clone(); + let baseline = Baseline::from_yaml_str(&format!( + r#" +version: "1.0" +results: + "src/demo.py": + - type: "AWS Access Key" + hashed_secret: "{}" + line_number: 1 + justification: "The operator did not explicitly mark this as not secret." +"#, + hex20(detection.hashed_secret) + )) + .expect("baseline parses"); + + let result = baseline.suppress(detections, std::path::Path::new("src/demo.py")); + assert_eq!(result.allowed.len(), 1); + assert!(result.suppressed.is_empty()); + assert!(result.fired_entries.is_empty()); +} + +#[test] +fn baseline_does_not_suppress_same_suffix_in_sibling_directory() { + let scanner = Scanner::new(); + let detections = scanner.scan_bytes(b"key = 'AKIAIOSFODNN7EXAMPLE'\n"); + let detection = detections + .iter() + .find(|detection| detection.rule_id == "AwsAccessKeyId") + .expect("AWS detection") + .clone(); + let baseline = Baseline::from_yaml_str(&format!( + r#" +version: "1.0" +results: + "src/demo.py": + - type: "AWS Access Key" + hashed_secret: "{}" + line_number: 1 + is_secret: false + justification: "Only src/demo.py was reviewed." +"#, + hex20(detection.hashed_secret) + )) + .expect("baseline parses"); + + let result = baseline.suppress(detections, std::path::Path::new("vendor/src/demo.py")); + assert_eq!(result.allowed.len(), 1); + assert!(result.suppressed.is_empty()); + assert!(result.fired_entries.is_empty()); +} + #[test] fn baseline_is_secret_true_does_not_suppress() { let scanner = Scanner::new(); @@ -223,6 +350,48 @@ results: assert_eq!(entries[1].line, 8); } +#[test] +fn baseline_rejects_absolute_result_paths() { + let err = Baseline::from_yaml_str( + r#" +version: "1.0" +results: + "/tmp/secret.py": + - type: "AWS Access Key" + hashed_secret: "0123456789abcdef0123456789abcdef01234567" + line_number: 4 + is_secret: false + justification: "Invalid path must not be accepted." +"#, + ) + .expect_err("absolute baseline path should fail"); + assert!( + err.to_string().contains("repository-relative"), + "unexpected error: {err}" + ); +} + +#[test] +fn baseline_rejects_parent_dir_result_paths() { + let err = Baseline::from_yaml_str( + r#" +version: "1.0" +results: + "../secret.py": + - type: "AWS Access Key" + hashed_secret: "0123456789abcdef0123456789abcdef01234567" + line_number: 4 + is_secret: false + justification: "Invalid path must not be accepted." +"#, + ) + .expect_err("escaping baseline path should fail"); + assert!( + err.to_string().contains("repository-relative"), + "unexpected error: {err}" + ); +} + #[test] fn absent_baseline_file_is_empty() { let dir = tempfile::tempdir().expect("tempdir"); @@ -258,3 +427,16 @@ fn hex20(bytes: [u8; 20]) -> String { } out } + +fn sha1_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut hasher = Sha1::new(); + hasher.update(bytes); + let digest = hasher.finalize(); + let mut out = String::new(); + for byte in digest { + out.push(char::from(HEX[usize::from(byte >> 4)])); + out.push(char::from(HEX[usize::from(byte & 0x0f)])); + } + out +} From 7d0e9194542ec0192eac934d625666a757418f2d Mon Sep 17 00:00:00 2001 From: John Morrissey <544926+tachyon-beep@users.noreply.github.com> Date: Tue, 19 May 2026 03:46:17 +1000 Subject: [PATCH 4/4] fix(wp5): keep secret scan wiring delegated --- crates/clarion-cli/src/analyze.rs | 209 +----------------- crates/clarion-cli/src/main.rs | 1 + crates/clarion-cli/src/run_lifecycle.rs | 20 ++ crates/clarion-cli/src/secret_scan.rs | 62 +++--- crates/clarion-cli/src/secret_scan/anchors.rs | 116 ++++++++++ .../clarion-cli/src/secret_scan/baseline.rs | 43 ++++ crates/clarion-cli/src/secret_scan/files.rs | 75 +++++++ 7 files changed, 292 insertions(+), 234 deletions(-) create mode 100644 crates/clarion-cli/src/run_lifecycle.rs create mode 100644 crates/clarion-cli/src/secret_scan/anchors.rs create mode 100644 crates/clarion-cli/src/secret_scan/baseline.rs create mode 100644 crates/clarion-cli/src/secret_scan/files.rs diff --git a/crates/clarion-cli/src/analyze.rs b/crates/clarion-cli/src/analyze.rs index db47b7d5..1021e2a9 100644 --- a/crates/clarion-cli/src/analyze.rs +++ b/crates/clarion-cli/src/analyze.rs @@ -128,7 +128,8 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti discovery_errors.join("; ") ); tracing::error!(run_id = %run_id, reason = %reason, "failing run: discovery errors"); - begin_run(&writer, &run_id, &analyze_config_json, &started_at).await?; + crate::run_lifecycle::begin_run(&writer, &run_id, &analyze_config_json, &started_at) + .await?; let completed_at = iso8601_now(); writer .send_wait(|ack| WriterCmd::FailRun { @@ -155,7 +156,8 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti } tracing::warn!(run_id = %run_id, "no plugins discovered"); - begin_run(&writer, &run_id, &analyze_config_json, &started_at).await?; + crate::run_lifecycle::begin_run(&writer, &run_id, &analyze_config_json, &started_at) + .await?; let completed_at = iso8601_now(); writer .send_wait(|ack| WriterCmd::CommitRun { @@ -207,14 +209,14 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti let source_files = collect_source_files(&project_root, &wanted_extensions); tracing::info!(file_count = source_files.len(), "source tree walk complete"); - let secret_scan_files = collect_secret_scan_files(&project_root, &source_files); + let secret_scan_files = crate::secret_scan::collect_scan_files(&project_root, &source_files); tracing::info!( file_count = secret_scan_files.len(), "secret scan file walk complete" ); - let secret_scan_outcome = + let mut secret_scan_outcome = crate::secret_scan::pre_ingest(&project_root, &secret_scan_files, &options.secret_scan)?; - begin_run(&writer, &run_id, &analyze_config_json, &started_at).await?; + crate::run_lifecycle::begin_run(&writer, &run_id, &analyze_config_json, &started_at).await?; // ── Per-plugin processing ───────────────────────────────────────────────── // @@ -243,8 +245,6 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti let mut run_outcome: RunOutcome = RunOutcome::Completed; let mut breaker = CrashLoopBreaker::default(); let mut crash_reasons: Vec = Vec::new(); - let mut secret_finding_anchors: BTreeMap = BTreeMap::new(); - 'plugins: for plugin in plugins { let plugin_id = plugin.manifest.plugin.plugin_id.clone(); let plugin_extensions: BTreeSet = plugin @@ -367,7 +367,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti // edge FK references resolve at insert time (B.3 §5). let entity_count = entities.len() as u64; let edge_count = edges.len() as u64; - remember_secret_finding_anchors(&entities, &mut secret_finding_anchors); + secret_scan_outcome.remember_finding_anchors(&entities); let mut insert_err: Option = None; for (id_str, record) in entities { let res = writer @@ -443,30 +443,9 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti } if !matches!(run_outcome, RunOutcome::HardFailed { .. }) - && let Err(e) = ensure_secret_finding_anchors( - &writer, - &project_root, - &started_at, - &secret_scan_outcome, - &mut secret_finding_anchors, - ) - .await - { - tracing::error!(run_id = %run_id, error = %e, "secret finding anchor persistence failed"); - run_outcome = RunOutcome::HardFailed { - reason: format!("secret finding anchor persistence failed: {e:#}"), - }; - } - - if !matches!(run_outcome, RunOutcome::HardFailed { .. }) - && let Err(e) = crate::secret_scan::emit_findings( - &writer, - &run_id, - &started_at, - &secret_scan_outcome, - &secret_finding_anchors, - ) - .await + && let Err(e) = secret_scan_outcome + .persist_findings(&writer, &run_id, &project_root, &started_at) + .await { tracing::error!(run_id = %run_id, error = %e, "secret finding persistence failed"); run_outcome = RunOutcome::HardFailed { @@ -638,24 +617,6 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti Ok(()) } -async fn begin_run( - writer: &Writer, - run_id: &str, - analyze_config_json: &str, - started_at: &str, -) -> Result<()> { - writer - .send_wait(|ack| WriterCmd::BeginRun { - run_id: run_id.to_owned(), - config_json: analyze_config_json.to_owned(), - started_at: started_at.to_owned(), - ack, - }) - .await - .map_err(|e| anyhow::anyhow!("{e}")) - .context("BeginRun") -} - // ── Phase 3 subsystem materialisation ───────────────────────────────────────── #[derive(Debug, Clone)] @@ -1559,108 +1520,6 @@ fn absolute_from_import_submodule_target( .then_some(candidate) } -fn remember_secret_finding_anchors( - entities: &[(String, EntityRecord)], - anchors: &mut BTreeMap, -) { - for (id, record) in entities { - let Some(path) = record.source_file_path.as_deref() else { - continue; - }; - let key = Path::new(path) - .canonicalize() - .unwrap_or_else(|_| PathBuf::from(path)); - if record.kind == "module" || !anchors.contains_key(&key) { - anchors.insert(key, id.clone()); - } - } -} - -async fn ensure_secret_finding_anchors( - writer: &Writer, - project_root: &Path, - started_at: &str, - outcome: &crate::secret_scan::SecretScanOutcome, - anchors: &mut BTreeMap, -) -> Result<()> { - for file in outcome.finding_files() { - let key = canonical_or_original(&file); - if anchors.contains_key(&key) { - continue; - } - let id = secret_finding_anchor_id(project_root, &key); - let relative = display_relative_path(project_root, &key); - let short_name = key - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or(&relative) - .to_owned(); - let mut properties = serde_json::Map::new(); - properties.insert("finding_anchor".to_owned(), serde_json::json!(true)); - if let Some(reason) = outcome.briefing_blocks.get(&key) { - properties.insert( - "briefing_blocked".to_owned(), - serde_json::Value::String(reason.clone()), - ); - } - let record = EntityRecord { - id: id.clone(), - plugin_id: "core".to_owned(), - kind: "file".to_owned(), - name: relative, - short_name, - parent_id: None, - source_file_id: None, - source_file_path: Some(key.display().to_string()), - source_byte_start: None, - source_byte_end: None, - source_line_start: None, - source_line_end: None, - properties_json: serde_json::Value::Object(properties).to_string(), - content_hash: file_content_hash(&key), - summary_json: None, - wardline_json: None, - first_seen_commit: None, - last_seen_commit: None, - created_at: started_at.to_owned(), - updated_at: started_at.to_owned(), - }; - writer - .send_wait(|ack| WriterCmd::InsertEntity { - entity: Box::new(record), - ack, - }) - .await - .map_err(|e| anyhow::anyhow!("{e}")) - .with_context(|| format!("InsertEntity for secret finding anchor {id}"))?; - anchors.insert(key, id); - } - Ok(()) -} - -fn secret_finding_anchor_id(project_root: &Path, file: &Path) -> String { - let relative = display_relative_path(project_root, file); - let digest = blake3::hash(relative.as_bytes()).to_hex().to_string(); - format!("core:file:{digest}") -} - -fn canonical_or_original(path: &Path) -> PathBuf { - path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) -} - -fn display_relative_path(root: &Path, path: &Path) -> String { - path.strip_prefix(root) - .unwrap_or(path) - .display() - .to_string() -} - -fn file_content_hash(path: &Path) -> Option { - fs::read(path) - .ok() - .map(|bytes| blake3::hash(&bytes).to_hex().to_string()) -} - /// Map an `AcceptedEntity` to an `EntityRecord` for the writer-actor. fn map_entity_to_record( entity: &AcceptedEntity, @@ -1956,52 +1815,6 @@ fn collect_source_files(root: &Path, wanted_extensions: &BTreeSet) -> Ve out } -fn collect_secret_scan_files(root: &Path, source_files: &[PathBuf]) -> Vec { - let mut out: BTreeSet = source_files - .iter() - .map(|path| canonical_or_original(path)) - .collect(); - for path in collect_secret_scan_sidecars(root) { - out.insert(canonical_or_original(&path)); - } - out.into_iter().collect() -} - -fn collect_secret_scan_sidecars(root: &Path) -> Vec { - let mut out = Vec::new(); - let mut builder = WalkBuilder::new(root); - builder - .follow_links(false) - .hidden(false) - .ignore(false) - .git_ignore(false) - .git_global(false) - .git_exclude(false) - .parents(false) - .require_git(false) - .filter_entry(|entry| !is_skipped_dir(entry)); - - for entry in builder.build().filter_map(std::result::Result::ok) { - let Some(file_type) = entry.file_type() else { - continue; - }; - let path = entry.into_path(); - if file_type.is_file() && is_secret_scan_sidecar(&path) { - out.push(path); - } - } - out -} - -fn is_secret_scan_sidecar(path: &Path) -> bool { - let file_name = path.file_name().and_then(|name| name.to_str()); - file_name.is_some_and(|name| name == ".env" || name.starts_with(".env.")) - || path - .extension() - .and_then(|extension| extension.to_str()) - .is_some_and(|extension| extension.eq_ignore_ascii_case("env")) -} - fn is_skipped_dir(entry: &DirEntry) -> bool { entry .file_type() diff --git a/crates/clarion-cli/src/main.rs b/crates/clarion-cli/src/main.rs index 71604599..4589d077 100644 --- a/crates/clarion-cli/src/main.rs +++ b/crates/clarion-cli/src/main.rs @@ -3,6 +3,7 @@ mod cli; mod clustering; mod config; mod install; +mod run_lifecycle; mod secret_scan; mod serve; mod stats; diff --git a/crates/clarion-cli/src/run_lifecycle.rs b/crates/clarion-cli/src/run_lifecycle.rs new file mode 100644 index 00000000..6370e2bf --- /dev/null +++ b/crates/clarion-cli/src/run_lifecycle.rs @@ -0,0 +1,20 @@ +use anyhow::{Context, Result}; +use clarion_storage::{Writer, commands::WriterCmd}; + +pub(crate) async fn begin_run( + writer: &Writer, + run_id: &str, + analyze_config_json: &str, + started_at: &str, +) -> Result<()> { + writer + .send_wait(|ack| WriterCmd::BeginRun { + run_id: run_id.to_owned(), + config_json: analyze_config_json.to_owned(), + started_at: started_at.to_owned(), + ack, + }) + .await + .map_err(|e| anyhow::anyhow!("{e}")) + .context("BeginRun") +} diff --git a/crates/clarion-cli/src/secret_scan.rs b/crates/clarion-cli/src/secret_scan.rs index 4dee1f48..6ca0de73 100644 --- a/crates/clarion-cli/src/secret_scan.rs +++ b/crates/clarion-cli/src/secret_scan.rs @@ -13,17 +13,19 @@ use std::{ path::{Path, PathBuf}, }; +mod anchors; +mod baseline; +mod files; mod findings; use anyhow::{Context, Result}; -use clarion_scanner::{Baseline, BaselineError, Detection, Scanner, SuppressionResult}; -pub(crate) use findings::emit_findings; +use clarion_scanner::{Detection, Scanner, SuppressionResult}; +use clarion_storage::{Writer, commands::EntityRecord}; +pub(crate) use files::collect_scan_files; use findings::{PendingFinding, secret_detected_finding}; use serde_json::json; const SECRET_OVERRIDE_ALLOWED: &str = "CLA-SEC-UNREDACTED-SECRETS-ALLOWED"; -const BASELINE_NO_JUSTIFICATION: &str = "CLA-INFRA-SECRET-BASELINE-NO-JUSTIFICATION"; -const BASELINE_MATCH: &str = "CLA-INFRA-SECRET-BASELINE-MATCH"; const OVERRIDE_UNCONFIRMED: &str = "CLA-INFRA-SECRET-OVERRIDE-UNCONFIRMED"; const CONFIRM_TOKEN: &str = "yes-i-understand"; @@ -37,6 +39,7 @@ pub(crate) struct SecretScanOptions { pub(crate) struct SecretScanOutcome { pub(crate) briefing_blocks: BTreeMap, findings: Vec, + finding_anchors: BTreeMap, override_files: Vec, scanned_files: BTreeSet, } @@ -65,6 +68,20 @@ impl SecretScanOutcome { .collect::>() ); } + + pub(crate) fn remember_finding_anchors(&mut self, entities: &[(String, EntityRecord)]) { + anchors::remember_finding_anchors(self, entities); + } + + pub(crate) async fn persist_findings( + &mut self, + writer: &Writer, + run_id: &str, + project_root: &Path, + started_at: &str, + ) -> Result<()> { + anchors::ensure_and_emit_findings(self, writer, run_id, project_root, started_at).await + } } pub(crate) fn pre_ingest( @@ -73,7 +90,7 @@ pub(crate) fn pre_ingest( options: &SecretScanOptions, ) -> Result { let scanner = Scanner::new(); - let (baseline, mut findings) = load_baseline_for_scan(project_root)?; + let (baseline, mut findings) = baseline::load_for_scan(project_root)?; let mut per_file = Vec::new(); let mut all_allowed = Vec::new(); let mut baseline_matches = Vec::new(); @@ -100,7 +117,7 @@ pub(crate) fn pre_ingest( } baseline_matches.extend(fired_entries.into_iter().map(|entry| PendingFinding { file_path: normalize_project_path(project_root, &entry.file_path), - rule_id: BASELINE_MATCH, + rule_id: baseline::baseline_match_rule_id(), kind: "fact", severity: "INFO", confidence: Some(1.0), @@ -162,39 +179,12 @@ pub(crate) fn pre_ingest( Ok(SecretScanOutcome { briefing_blocks, findings, + finding_anchors: BTreeMap::new(), override_files: override_files.into_iter().collect(), scanned_files, }) } -fn load_baseline_for_scan(project_root: &Path) -> Result<(Baseline, Vec)> { - let path = project_root.join(".clarion/secrets-baseline.yaml"); - match clarion_scanner::load_baseline(&path) { - Ok(baseline) => Ok((baseline, Vec::new())), - Err(BaselineError::MissingJustifications { entries }) => Ok(( - Baseline::empty(), - entries - .into_iter() - .map(|entry| PendingFinding { - file_path: normalize_project_path(project_root, &entry.file), - rule_id: BASELINE_NO_JUSTIFICATION, - kind: "defect", - severity: "ERROR", - confidence: Some(1.0), - confidence_basis: Some("baseline_schema"), - message: format!( - "Secret baseline entry missing justification at {}:{}", - entry.file.display(), - entry.line - ), - evidence: json!({"file_path": entry.file, "line_number": entry.line}), - }) - .collect(), - )), - Err(err) => Err(err).context("load secret baseline"), - } -} - fn confirm_override_if_needed( allowed: &[(PathBuf, Detection)], options: &SecretScanOptions, @@ -232,7 +222,7 @@ fn abort_unconfirmed_override() -> ! { std::process::exit(78); } -fn canonical_or_original(path: &Path) -> PathBuf { +pub(super) fn canonical_or_original(path: &Path) -> PathBuf { path.canonicalize().unwrap_or_else(|_| path.to_path_buf()) } @@ -244,7 +234,7 @@ fn project_relative_path(root: &Path, path: &Path) -> PathBuf { path.strip_prefix(root).unwrap_or(path).to_path_buf() } -fn display_relative(root: &Path, path: &Path) -> String { +pub(super) fn display_relative(root: &Path, path: &Path) -> String { path.strip_prefix(root) .unwrap_or(path) .display() diff --git a/crates/clarion-cli/src/secret_scan/anchors.rs b/crates/clarion-cli/src/secret_scan/anchors.rs new file mode 100644 index 00000000..9f358864 --- /dev/null +++ b/crates/clarion-cli/src/secret_scan/anchors.rs @@ -0,0 +1,116 @@ +use std::{fs, path::Path}; + +use anyhow::{Context, Result}; +use clarion_storage::{ + Writer, + commands::{EntityRecord, WriterCmd}, +}; + +use super::{SecretScanOutcome, canonical_or_original, display_relative}; +use crate::secret_scan::findings::emit_findings; + +pub(super) fn remember_finding_anchors( + outcome: &mut SecretScanOutcome, + entities: &[(String, EntityRecord)], +) { + for (id, record) in entities { + let Some(path) = record.source_file_path.as_deref() else { + continue; + }; + let key = canonical_or_original(Path::new(path)); + if record.kind == "module" || !outcome.finding_anchors.contains_key(&key) { + outcome.finding_anchors.insert(key, id.clone()); + } + } +} + +pub(super) async fn ensure_and_emit_findings( + outcome: &mut SecretScanOutcome, + writer: &Writer, + run_id: &str, + project_root: &Path, + started_at: &str, +) -> Result<()> { + ensure_finding_anchors(outcome, writer, project_root, started_at).await?; + emit_findings( + writer, + run_id, + started_at, + outcome, + &outcome.finding_anchors, + ) + .await +} + +async fn ensure_finding_anchors( + outcome: &mut SecretScanOutcome, + writer: &Writer, + project_root: &Path, + started_at: &str, +) -> Result<()> { + for file in outcome.finding_files() { + let key = canonical_or_original(&file); + if outcome.finding_anchors.contains_key(&key) { + continue; + } + let id = secret_finding_anchor_id(project_root, &key); + let relative = display_relative(project_root, &key); + let short_name = key + .file_name() + .and_then(|name| name.to_str()) + .unwrap_or(&relative) + .to_owned(); + let mut properties = serde_json::Map::new(); + properties.insert("finding_anchor".to_owned(), serde_json::json!(true)); + if let Some(reason) = outcome.briefing_blocks.get(&key) { + properties.insert( + "briefing_blocked".to_owned(), + serde_json::Value::String(reason.clone()), + ); + } + let record = EntityRecord { + id: id.clone(), + plugin_id: "core".to_owned(), + kind: "file".to_owned(), + name: relative, + short_name, + parent_id: None, + source_file_id: None, + source_file_path: Some(key.display().to_string()), + source_byte_start: None, + source_byte_end: None, + source_line_start: None, + source_line_end: None, + properties_json: serde_json::Value::Object(properties).to_string(), + content_hash: file_content_hash(&key), + summary_json: None, + wardline_json: None, + first_seen_commit: None, + last_seen_commit: None, + created_at: started_at.to_owned(), + updated_at: started_at.to_owned(), + }; + writer + .send_wait(|ack| WriterCmd::InsertEntity { + entity: Box::new(record), + ack, + }) + .await + .map_err(|e| anyhow::anyhow!("{e}")) + .with_context(|| format!("InsertEntity for secret finding anchor {id}"))?; + outcome.finding_anchors.insert(key, id); + } + Ok(()) +} + +fn secret_finding_anchor_id(project_root: &Path, file: &Path) -> String { + let relative = display_relative(project_root, file); + let digest = blake3::hash(relative.as_bytes()).to_hex().to_string(); + format!("core:file:{digest}") +} + +fn file_content_hash(path: &Path) -> Option { + fs::read(path) + .ok() + .map(|bytes| blake3::hash(&bytes).to_hex().to_string()) +} diff --git a/crates/clarion-cli/src/secret_scan/baseline.rs b/crates/clarion-cli/src/secret_scan/baseline.rs new file mode 100644 index 00000000..793a8c21 --- /dev/null +++ b/crates/clarion-cli/src/secret_scan/baseline.rs @@ -0,0 +1,43 @@ +use std::path::Path; + +use anyhow::{Context, Result}; +use clarion_scanner::{Baseline, BaselineError}; +use serde_json::json; + +use super::normalize_project_path; +use crate::secret_scan::findings::PendingFinding; + +const BASELINE_NO_JUSTIFICATION: &str = "CLA-INFRA-SECRET-BASELINE-NO-JUSTIFICATION"; +const BASELINE_MATCH: &str = "CLA-INFRA-SECRET-BASELINE-MATCH"; + +pub(super) fn load_for_scan(project_root: &Path) -> Result<(Baseline, Vec)> { + let path = project_root.join(".clarion/secrets-baseline.yaml"); + match clarion_scanner::load_baseline(&path) { + Ok(baseline) => Ok((baseline, Vec::new())), + Err(BaselineError::MissingJustifications { entries }) => Ok(( + Baseline::empty(), + entries + .into_iter() + .map(|entry| PendingFinding { + file_path: normalize_project_path(project_root, &entry.file), + rule_id: BASELINE_NO_JUSTIFICATION, + kind: "defect", + severity: "ERROR", + confidence: Some(1.0), + confidence_basis: Some("baseline_schema"), + message: format!( + "Secret baseline entry missing justification at {}:{}", + entry.file.display(), + entry.line + ), + evidence: json!({"file_path": entry.file, "line_number": entry.line}), + }) + .collect(), + )), + Err(err) => Err(err).context("load secret baseline"), + } +} + +pub(super) fn baseline_match_rule_id() -> &'static str { + BASELINE_MATCH +} diff --git a/crates/clarion-cli/src/secret_scan/files.rs b/crates/clarion-cli/src/secret_scan/files.rs new file mode 100644 index 00000000..0f51fec9 --- /dev/null +++ b/crates/clarion-cli/src/secret_scan/files.rs @@ -0,0 +1,75 @@ +use std::{ + collections::BTreeSet, + path::{Path, PathBuf}, +}; + +use ignore::{DirEntry, WalkBuilder}; + +use super::canonical_or_original; + +const SKIP_DIRS: &[&str] = &[ + ".clarion", + ".git", + ".hg", + ".svn", + ".jj", + ".venv", + "__pycache__", + "node_modules", +]; + +pub(crate) fn collect_scan_files(root: &Path, source_files: &[PathBuf]) -> Vec { + let mut out: BTreeSet = source_files + .iter() + .map(|path| canonical_or_original(path)) + .collect(); + for path in collect_secret_scan_sidecars(root) { + out.insert(canonical_or_original(&path)); + } + out.into_iter().collect() +} + +fn collect_secret_scan_sidecars(root: &Path) -> Vec { + let mut out = Vec::new(); + let mut builder = WalkBuilder::new(root); + builder + .follow_links(false) + .hidden(false) + .ignore(false) + .git_ignore(false) + .git_global(false) + .git_exclude(false) + .parents(false) + .require_git(false) + .filter_entry(|entry| !is_skipped_dir(entry)); + + for entry in builder.build().filter_map(std::result::Result::ok) { + let Some(file_type) = entry.file_type() else { + continue; + }; + let path = entry.into_path(); + if file_type.is_file() && is_secret_scan_sidecar(&path) { + out.push(path); + } + } + out +} + +fn is_secret_scan_sidecar(path: &Path) -> bool { + let file_name = path.file_name().and_then(|name| name.to_str()); + file_name.is_some_and(|name| name == ".env" || name.starts_with(".env.")) + || path + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("env")) +} + +fn is_skipped_dir(entry: &DirEntry) -> bool { + entry + .file_type() + .is_some_and(|file_type| file_type.is_dir()) + && entry + .file_name() + .to_str() + .is_some_and(|name| SKIP_DIRS.contains(&name)) +}