WP5 pre-ingest secret scanner

.github/workflows/ci.yml

@@ -128,3 +128,6 @@ jobs:
 
       - name: run walking skeleton
         run: bash tests/e2e/sprint_1_walking_skeleton.sh
+
+      - name: run WP5 secret scanner smoke
+        run: CARGO_BUILD=0 bash tests/e2e/wp5_secret_scan.sh

Cargo.toml

@@ -6,6 +6,7 @@ members = [
     "crates/clarion-cli",
     "crates/clarion-mcp",
     "crates/clarion-plugin-fixture",
+    "crates/clarion-scanner",
 ]
 
 [workspace.package]
@@ -38,9 +39,11 @@ dotenvy = "0.15"
 ignore = "0.4"
 rusqlite = { version = "0.31", features = ["bundled"] }
 reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls-native-roots"] }
+regex = "1"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_norway = "0.9.42"
+sha1 = "0.10"
 sha2 = "0.10"
 thiserror = "1"
 time = { version = "0.3", features = ["formatting", "macros", "parsing"] }

crates/clarion-cli/Cargo.toml

@@ -19,6 +19,7 @@ blake3.workspace = true
 clap.workspace = true
 clarion-core = { path = "../clarion-core", version = "0.1.0-dev" }
 clarion-mcp = { path = "../clarion-mcp", version = "0.1.0-dev" }
+clarion-scanner = { path = "../clarion-scanner", version = "0.1.0-dev" }
 clarion-storage = { path = "../clarion-storage", version = "0.1.0-dev" }
 dotenvy.workspace = true
 ignore.workspace = true

crates/clarion-cli/src/analyze.rs

@@ -41,6 +41,7 @@ const WEAK_MODULARITY_RULE_ID: &str = "CLA-FACT-CLUSTERING-WEAK-MODULARITY";
 #[derive(Debug, Clone, Default)]
 pub(crate) struct AnalyzeOptions {
     pub(crate) config_path: Option<PathBuf>,
+    pub(crate) secret_scan: crate::secret_scan::SecretScanOptions,
 }
 
 /// Run the analyze command against `project_path`.
@@ -93,17 +94,6 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
     let run_id = Uuid::new_v4().to_string();
     let started_at = iso8601_now();
 
-    writer
-        .send_wait(|ack| WriterCmd::BeginRun {
-            run_id: run_id.clone(),
-            config_json: analyze_config_json.clone(),
-            started_at: started_at.clone(),
-            ack,
-        })
-        .await
-        .map_err(|e| anyhow::anyhow!("{e}"))
-        .context("BeginRun")?;
-
     // ── Discover plugins ──────────────────────────────────────────────────────
     let discovery_results = discover();
     let mut plugins: Vec<DiscoveredPlugin> = Vec::new();
@@ -138,6 +128,8 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
                 discovery_errors.join("; ")
             );
             tracing::error!(run_id = %run_id, reason = %reason, "failing run: discovery errors");
+            crate::run_lifecycle::begin_run(&writer, &run_id, &analyze_config_json, &started_at)
+                .await?;
             let completed_at = iso8601_now();
             writer
                 .send_wait(|ack| WriterCmd::FailRun {
@@ -164,6 +156,8 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
         }
 
         tracing::warn!(run_id = %run_id, "no plugins discovered");
+        crate::run_lifecycle::begin_run(&writer, &run_id, &analyze_config_json, &started_at)
+            .await?;
         let completed_at = iso8601_now();
         writer
             .send_wait(|ack| WriterCmd::CommitRun {
@@ -215,6 +209,15 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
     let source_files = collect_source_files(&project_root, &wanted_extensions);
     tracing::info!(file_count = source_files.len(), "source tree walk complete");
 
+    let secret_scan_files = crate::secret_scan::collect_scan_files(&project_root, &source_files);
+    tracing::info!(
+        file_count = secret_scan_files.len(),
+        "secret scan file walk complete"
+    );
+    let mut secret_scan_outcome =
+        crate::secret_scan::pre_ingest(&project_root, &secret_scan_files, &options.secret_scan)?;
+    crate::run_lifecycle::begin_run(&writer, &run_id, &analyze_config_json, &started_at).await?;
+
     // ── Per-plugin processing ─────────────────────────────────────────────────
     //
     // A per-plugin crash (spawn / handshake / analyze_file Err) does NOT tank
@@ -242,7 +245,6 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
     let mut run_outcome: RunOutcome = RunOutcome::Completed;
     let mut breaker = CrashLoopBreaker::default();
     let mut crash_reasons: Vec<String> = Vec::new();
-
     'plugins: for plugin in plugins {
         let plugin_id = plugin.manifest.plugin.plugin_id.clone();
         let plugin_extensions: BTreeSet<String> = plugin
@@ -282,6 +284,8 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
         let pid_clone = plugin_id.clone();
         let exec_clone = plugin.executable.clone();
         let files_clone = plugin_files.clone();
+        let briefing_blocks_clone = secret_scan_outcome.briefing_blocks.clone();
+        let scanned_files_clone = secret_scan_outcome.scanned_files().clone();
 
         // A JoinError here means the blocking task panicked (OOM, stack
         // overflow, internal unwrap, abort — anything that unwinds past the
@@ -299,6 +303,8 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
                     &pid_clone,
                     &exec_clone,
                     &files_clone,
+                    &briefing_blocks_clone,
+                    &scanned_files_clone,
                 )
             })
             .await,
@@ -361,6 +367,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
                 // edge FK references resolve at insert time (B.3 §5).
                 let entity_count = entities.len() as u64;
                 let edge_count = edges.len() as u64;
+                secret_scan_outcome.remember_finding_anchors(&entities);
                 let mut insert_err: Option<anyhow::Error> = None;
                 for (id_str, record) in entities {
                     let res = writer
@@ -435,6 +442,17 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
         }
     }
 
+    if !matches!(run_outcome, RunOutcome::HardFailed { .. })
+        && let Err(e) = secret_scan_outcome
+            .persist_findings(&writer, &run_id, &project_root, &started_at)
+            .await
+    {
+        tracing::error!(run_id = %run_id, error = %e, "secret finding persistence failed");
+        run_outcome = RunOutcome::HardFailed {
+            reason: format!("secret finding persistence failed: {e:#}"),
+        };
+    }
+
     // ── Commit or fail the run ────────────────────────────────────────────────
     //
     // Writer-actor failures set `run_outcome = HardFailed` above (and break).
@@ -497,7 +515,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
 
     match run_outcome {
         RunOutcome::Completed => {
-            let stats_json = serde_json::json!({
+            let mut stats_json = serde_json::json!({
                 "entities_inserted": total_entity_count,
                 "edges_inserted": total_edge_count,
                 "dropped_edges_total": dropped_edges_total,
@@ -513,8 +531,9 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
                 "pyright_index_parse_latency_p95_ms": pyright_index_parse_latency_p95_ms,
                 "extractor_parse_latency_p95_ms": extractor_parse_latency_p95_ms,
                 "clustering": phase3_output.clustering_stats.clone(),
-            })
-            .to_string();
+            });
+            secret_scan_outcome.augment_stats(&mut stats_json);
+            let stats_json = stats_json.to_string();
             writer
                 .send_wait(|ack| WriterCmd::CommitRun {
                     run_id: run_id.clone(),
@@ -532,7 +551,7 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
             // failed, atomically (writer folds the UPDATE into the open tx).
             // The stats JSON carries both fields so operators can see what
             // was persisted alongside the failure reason.
-            let stats_json = serde_json::json!({
+            let mut stats_json = serde_json::json!({
                 "entities_inserted": total_entity_count,
                 "edges_inserted": total_edge_count,
                 "dropped_edges_total": dropped_edges_total,
@@ -549,8 +568,9 @@ pub(crate) async fn run_with_options(project_path: PathBuf, options: AnalyzeOpti
                 "extractor_parse_latency_p95_ms": extractor_parse_latency_p95_ms,
                 "clustering": phase3_output.clustering_stats.clone(),
                 "failure_reason": reason,
-            })
-            .to_string();
+            });
+            secret_scan_outcome.augment_stats(&mut stats_json);
+            let stats_json = stats_json.to_string();
             writer
                 .send_wait(|ack| WriterCmd::CommitRun {
                     run_id: run_id.clone(),
@@ -1170,6 +1190,8 @@ fn run_plugin_blocking(
     plugin_id: &str,
     executable: &Path,
     files: &[PathBuf],
+    briefing_blocks: &BTreeMap<PathBuf, String>,
+    scanned_source_files: &BTreeSet<PathBuf>,
 ) -> Result<BatchResult, PluginRunError> {
     use clarion_core::PluginHost;
 
@@ -1185,6 +1207,8 @@ fn run_plugin_blocking(
                 PluginRunError::new(format!("plugin {plugin_id} spawn/handshake error: {other}"))
             }
         })?;
+    host.set_briefing_blocks(briefing_blocks.clone());
+    host.set_scanned_source_files(scanned_source_files.clone());
 
     let work_result: Result<Collected, String> = (|| {
         let mut collected_entities: Vec<(String, EntityRecord)> = Vec::new();

crates/clarion-cli/src/cli.rs

@@ -32,6 +32,15 @@ pub enum Command {
         /// Path to clarion.yaml (default: project-root/clarion.yaml if present).
         #[arg(long)]
         config: Option<PathBuf>,
+
+        /// Allow analysis of files containing unredacted secrets. Requires a
+        /// confirmation step when detections are present.
+        #[arg(long)]
+        allow_unredacted_secrets: bool,
+
+        /// Non-TTY confirmation token for --allow-unredacted-secrets.
+        #[arg(long, value_name = "TOKEN", requires = "allow_unredacted_secrets")]
+        confirm_allow_unredacted_secrets: Option<String>,
     },
 
     /// Run the MCP stdio server.

crates/clarion-cli/src/main.rs

@@ -3,37 +3,52 @@ mod cli;
 mod clustering;
 mod config;
 mod install;
+mod run_lifecycle;
+mod secret_scan;
 mod serve;
 mod stats;
 
 use anyhow::Result;
 use clap::Parser;
 
 fn main() -> Result<()> {
-    // Load .env from CWD or any ancestor directory, before tracing setup so a
+    let cli = cli::Cli::parse();
+    // Load .env before tracing setup for operator-facing commands so a
     // .env-supplied RUST_LOG is in effect by the time the filter is built.
-    // Existing process env vars win over .env values (dotenvy default), so an
-    // explicit `OPENROUTER_API_KEY=… clarion serve` still beats a checked-in
-    // dev .env. Missing .env is not an error — silently skip.
-    let _ = dotenvy::dotenv();
+    // `analyze` is deliberately excluded: project .env contents are scanned
+    // as source sidecars and must not be imported into plugin subprocess
+    // environments before WP5's pre-ingest secret gate runs.
+    if !matches!(&cli.command, cli::Command::Analyze { .. }) {
+        let _ = dotenvy::dotenv();
+    }
     init_tracing();
-    let cli = cli::Cli::parse();
     match cli.command {
         cli::Command::Install { force, path } => install::run(&path, force),
-        cli::Command::Analyze { path, config } => {
+        cli::Command::Analyze {
+            path,
+            config,
+            allow_unredacted_secrets,
+            confirm_allow_unredacted_secrets,
+        } => {
             let rt = tokio::runtime::Builder::new_multi_thread()
                 .enable_all()
                 .build()?;
-            if let Some(config_path) = config {
-                rt.block_on(analyze::run_with_options(
-                    path,
-                    analyze::AnalyzeOptions {
-                        config_path: Some(config_path),
-                    },
-                ))
-            } else {
-                rt.block_on(analyze::run(path))
+            if config.is_none()
+                && !allow_unredacted_secrets
+                && confirm_allow_unredacted_secrets.is_none()
+            {
+                return rt.block_on(analyze::run(path));
             }
+            rt.block_on(analyze::run_with_options(
+                path,
+                analyze::AnalyzeOptions {
+                    config_path: config,
+                    secret_scan: secret_scan::SecretScanOptions {
+                        allow_unredacted_secrets,
+                        confirm_allow_unredacted_secrets,
+                    },
+                },
+            ))
         }
         cli::Command::Serve { path, config } => serve::run(&path, config.as_deref()),
     }

crates/clarion-cli/src/run_lifecycle.rs

@@ -0,0 +1,20 @@
+use anyhow::{Context, Result};
+use clarion_storage::{Writer, commands::WriterCmd};
+
+pub(crate) async fn begin_run(
+    writer: &Writer,
+    run_id: &str,
+    analyze_config_json: &str,
+    started_at: &str,
+) -> Result<()> {
+    writer
+        .send_wait(|ack| WriterCmd::BeginRun {
+            run_id: run_id.to_owned(),
+            config_json: analyze_config_json.to_owned(),
+            started_at: started_at.to_owned(),
+            ack,
+        })
+        .await
+        .map_err(|e| anyhow::anyhow!("{e}"))
+        .context("BeginRun")
+}