diff --git a/crates/soth-cli/src/cli_config.rs b/crates/soth-cli/src/cli_config.rs index a2d35c9..d70c921 100644 --- a/crates/soth-cli/src/cli_config.rs +++ b/crates/soth-cli/src/cli_config.rs @@ -1127,10 +1127,91 @@ pub fn sync_client_device_id( Ok(device_id) } +/// Force-generate a fresh client device id, overwriting any persisted one +/// in both the config tags and the on-disk `device_id` file. Used by +/// `soth enroll --new-device-id` as the escape hatch when the cloud has +/// rejected the stale device_id (e.g. after an org migration): reusing the +/// old id would just reproduce the same 403. +pub fn regenerate_client_device_id(config: &mut SothConfig) -> Result { + let device_id = format!("device-{}", Uuid::new_v4()); + config + .cloud + .tags + .insert("device_id".to_string(), device_id.clone()); + write_client_device_id(&device_id)?; + Ok(device_id) +} + pub fn resolved_db_path(config: &SothConfig) -> PathBuf { expand_tilde(Path::new(config.proxy.db_path.as_str())) } +#[cfg(test)] +mod device_id_tests { + use super::{regenerate_client_device_id, SothConfig}; + use std::env; + + /// Run `f` with HOME (and the Windows/soth-specific home vars) pointed at + /// a throwaway tempdir so device_id file writes stay sandboxed. + fn with_temp_home(f: impl FnOnce() -> T) -> T { + let _guard = crate::commands::proxy::lock_test_env(); + let temp = tempfile::tempdir().expect("tempdir"); + let old_home = env::var_os("HOME"); + let old_userprofile = env::var_os("USERPROFILE"); + let old_soth_home = env::var_os("SOTH_HOME_DIR"); + unsafe { + env::set_var("HOME", temp.path()); + env::set_var("USERPROFILE", temp.path()); + env::set_var("SOTH_HOME_DIR", temp.path().join(".soth")); + } + let result = f(); + match old_home { + Some(v) => unsafe { env::set_var("HOME", v) }, + None => unsafe { env::remove_var("HOME") }, + } + match old_userprofile { + Some(v) => unsafe { env::set_var("USERPROFILE", v) }, + None => unsafe { env::remove_var("USERPROFILE") }, + } + match old_soth_home { + Some(v) => unsafe { env::set_var("SOTH_HOME_DIR", v) }, + None => unsafe { env::remove_var("SOTH_HOME_DIR") }, + } + result + } + + #[test] + fn regenerate_replaces_stale_device_id_tag() { + with_temp_home(|| { + let mut config = SothConfig::default(); + config + .cloud + .tags + .insert("device_id".to_string(), "device-stale".to_string()); + + let fresh = regenerate_client_device_id(&mut config).expect("regenerate"); + + assert!(fresh.starts_with("device-"), "id has device- prefix"); + assert_ne!(fresh, "device-stale", "stale id was replaced"); + assert_eq!( + config.cloud.tags.get("device_id").map(String::as_str), + Some(fresh.as_str()), + "config tag reflects the fresh id" + ); + }); + } + + #[test] + fn regenerate_yields_distinct_ids() { + with_temp_home(|| { + let mut config = SothConfig::default(); + let first = regenerate_client_device_id(&mut config).expect("first"); + let second = regenerate_client_device_id(&mut config).expect("second"); + assert_ne!(first, second, "each regenerate produces a fresh id"); + }); + } +} + #[cfg(test)] mod code_extension_config_tests { use super::{ diff --git a/crates/soth-cli/src/command_graph.rs b/crates/soth-cli/src/command_graph.rs index c6fd0e7..f4f6cd6 100644 --- a/crates/soth-cli/src/command_graph.rs +++ b/crates/soth-cli/src/command_graph.rs @@ -18,9 +18,29 @@ pub struct GlobalOptions { pub verbose: bool, } +/// GETTING STARTED block appended to `soth --help`. Puts the one-shot +/// `soth up` bootstrap front and centre so first-time users don't have to +/// reverse-engineer the golden path from the flat subcommand list. +const AFTER_HELP: &str = "\ +GETTING STARTED: + Standalone (local-only): + soth setup-ca Generate + trust the local MITM CA (one time) + soth up One-shot bootstrap: init -> CA -> start -> enable proxy + soth status Confirm the daemon is healthy + + Cloud-managed (team policy + telemetry): + soth setup-ca Generate + trust the local MITM CA (one time) + soth enroll Exchange an invite token for cloud credentials + soth up Bootstrap and start under cloud policy + soth status Confirm the daemon + cloud heartbeat are healthy + + Already enrolled? `soth up --token ` folds enrollment into bootstrap. + Tear down with `soth down` (stop daemon + disable system proxy)."; + #[derive(Parser)] #[command(name = "soth")] #[command(author, version, about, long_about = None)] +#[command(after_help = AFTER_HELP)] pub struct Cli { #[command(flatten)] pub global: GlobalOptions, @@ -31,18 +51,18 @@ pub struct Cli { #[derive(Subcommand)] pub enum Commands { + /// One-shot bootstrap (start here): init -> CA -> start -> enable proxy + Up(UpArgs), + + /// stop + off (tear down: stop daemon and disable system proxy) + Down, + /// Start the proxy daemon Start(StartArgs), /// Stop the proxy daemon Stop, - /// One-shot bootstrap: init -> CA -> start -> on - Up(UpArgs), - - /// stop + off - Down, - /// Enable system proxy On(OnArgs), @@ -836,6 +856,7 @@ async fn run_up_command(args: UpArgs, global_config: Option) -> anyhow: from_stdin: false, non_interactive: true, machine_name: args.machine_name, + new_device_id: false, }, effective_config.clone(), ) diff --git a/crates/soth-cli/src/commands/enroll.rs b/crates/soth-cli/src/commands/enroll.rs index cb9ec40..5b96447 100644 --- a/crates/soth-cli/src/commands/enroll.rs +++ b/crates/soth-cli/src/commands/enroll.rs @@ -46,6 +46,13 @@ pub struct EnrollArgs { /// Optional machine name override sent during enrollment #[arg(long)] pub machine_name: Option, + + /// Regenerate the client device id before enrolling. Use this when a + /// prior enrollment left a stale device_id that the cloud now rejects + /// (heartbeat 403 org/identity mismatch): the default enroll reuses the + /// persisted device_id, which reproduces the same rejection. + #[arg(long)] + pub new_device_id: bool, } pub async fn run(args: EnrollArgs, global_config: Option) -> Result<()> { @@ -84,7 +91,17 @@ pub async fn run(args: EnrollArgs, global_config: Option) -> Result<()> .machine_name .clone() .unwrap_or_else(default_machine_name); - let enrollment_device_id = cli_config::sync_client_device_id(&mut config, None)?; + // `--new-device-id` forces a brand-new identity so the exchange (and the + // heartbeats that follow) register a fresh device rather than reusing a + // stale, cloud-rejected one. Without it, sync_client_device_id reuses the + // persisted id. + let enrollment_device_id = if args.new_device_id { + let regenerated = cli_config::regenerate_client_device_id(&mut config)?; + println!("Regenerated client device ID: {regenerated}"); + regenerated + } else { + cli_config::sync_client_device_id(&mut config, None)? + }; let enrollment_proxy_public_key = proxy_public_key_base64(enrollment_device_id.as_str()); let response_json = exchange_enroll_token( diff --git a/crates/soth-cli/src/commands/init.rs b/crates/soth-cli/src/commands/init.rs index d5a93d2..48a2dcc 100644 --- a/crates/soth-cli/src/commands/init.rs +++ b/crates/soth-cli/src/commands/init.rs @@ -25,8 +25,8 @@ pub async fn run(output: PathBuf) -> Result<()> { println!(); println!("Next steps:"); println!(" 1. soth setup-ca"); - println!(" 2. soth start"); - println!(" 3. soth on"); + println!(" 2. soth enroll (cloud-managed only; skip for standalone)"); + println!(" 3. soth up (or: soth start && soth on)"); println!(" 4. soth status"); Ok(()) } diff --git a/crates/soth-cli/src/commands/proxy/doctor.rs b/crates/soth-cli/src/commands/proxy/doctor.rs index 9154826..02ef152 100644 --- a/crates/soth-cli/src/commands/proxy/doctor.rs +++ b/crates/soth-cli/src/commands/proxy/doctor.rs @@ -19,10 +19,29 @@ struct DoctorReport { daemon: DaemonDiagnostics, system_proxy: SystemProxyDiagnostics, ca: CaDiagnostics, + cloud: CloudDiagnostics, loopback_bindings: LoopbackBindings, findings: Vec, } +#[derive(Debug, Serialize)] +struct CloudDiagnostics { + /// Whether cloud sync is enabled in config. When false the whole + /// section is informational (standalone mode heartbeats nothing). + enabled: bool, + /// The last heartbeat rejection, if the most recent heartbeat was + /// rejected by the cloud (e.g. a 403 org/identity mismatch). + #[serde(skip_serializing_if = "Option::is_none")] + heartbeat_rejection: Option, +} + +#[derive(Debug, Serialize)] +struct CloudRejection { + status: u16, + message: String, + rejected_at_unix_secs: u64, +} + #[derive(Debug, Serialize)] struct DaemonDiagnostics { pid_file: PathDetails, @@ -353,6 +372,33 @@ pub async fn run(config_path: Option, json: bool) -> Result<()> { println!("Parse error: {err}"); } + println!(); + println!("CLOUD"); + println!("----------------------------------------"); + println!( + "Sync enabled: {}", + if report.cloud.enabled { + "yes" + } else { + "no (standalone)" + } + ); + match report.cloud.heartbeat_rejection.as_ref() { + Some(rejection) => { + println!( + "Heartbeat: REJECTED (HTTP {}) — {}", + rejection.status, rejection.message + ); + println!( + "Fix: re-enroll this device — `soth enroll --new-device-id `" + ); + } + None if report.cloud.enabled => { + println!("Heartbeat: no rejection recorded"); + } + None => {} + } + println!(); println!("LOOPBACK"); println!("----------------------------------------"); @@ -560,11 +606,32 @@ fn build_report(config_path: Option) -> DoctorReport { details, }; + // Cloud/auth: surface the last heartbeat rejection so an org/identity + // mismatch (403) is diagnosable here, not just a silent daemon warn. + // Only meaningful when cloud sync is enabled; best-effort read. + let heartbeat_rejection = if config.cloud.enabled { + soth_sync::heartbeat_rejection::read() + .ok() + .flatten() + .map(|r| CloudRejection { + status: r.status, + message: r.message, + rejected_at_unix_secs: r.rejected_at, + }) + } else { + None + }; + let cloud = CloudDiagnostics { + enabled: config.cloud.enabled, + heartbeat_rejection, + }; + let mut report = DoctorReport { managed_runtime, daemon, system_proxy, ca, + cloud, loopback_bindings, findings: Vec::new(), }; @@ -647,6 +714,21 @@ fn compute_findings(report: &DoctorReport) -> Vec { } } + if let Some(rejection) = report.cloud.heartbeat_rejection.as_ref() { + findings.push(DoctorFinding { + level: "error".to_string(), + code: "cloud_heartbeat_rejected".to_string(), + message: format!( + "Cloud rejected the last heartbeat (HTTP {}): {}", + rejection.status, rejection.message + ), + remediation: + "Re-enroll this device with a fresh identity: `soth enroll --new-device-id ` \ + (a 403 usually means the persisted device_id belongs to a different org)." + .to_string(), + }); + } + if report.system_proxy.state_file.exists && !report.system_proxy.owner_file.exists { findings.push(DoctorFinding { level: "warn".to_string(), @@ -1148,6 +1230,10 @@ mod tests { os_trust_status: None, os_trust_detail: None, }, + cloud: CloudDiagnostics { + enabled: false, + heartbeat_rejection: None, + }, loopback_bindings: LoopbackBindings { expected_port: 8080, local_listener_open, @@ -1194,4 +1280,30 @@ mod tests { assert!(!findings.iter().any(|f| f.code == "listener_not_open")); assert!(!findings.iter().any(|f| f.code == "dangling_system_proxy")); } + + #[test] + fn heartbeat_rejection_produces_reenroll_finding() { + let mut report = report_for_findings(true, true); + report.cloud.enabled = true; + report.cloud.heartbeat_rejection = Some(CloudRejection { + status: 403, + message: "org mismatch".to_string(), + rejected_at_unix_secs: 1_700_000_000, + }); + let findings = compute_findings(&report); + let finding = findings + .iter() + .find(|f| f.code == "cloud_heartbeat_rejected") + .expect("cloud_heartbeat_rejected finding"); + assert_eq!(finding.level, "error"); + assert!(finding.message.contains("403")); + assert!(finding.remediation.contains("--new-device-id")); + } + + #[test] + fn no_heartbeat_rejection_produces_no_cloud_finding() { + let report = report_for_findings(true, true); + let findings = compute_findings(&report); + assert!(!findings.iter().any(|f| f.code == "cloud_heartbeat_rejected")); + } } diff --git a/crates/soth-cli/src/commands/proxy/status.rs b/crates/soth-cli/src/commands/proxy/status.rs index cea6ea9..c63dc83 100644 --- a/crates/soth-cli/src/commands/proxy/status.rs +++ b/crates/soth-cli/src/commands/proxy/status.rs @@ -49,6 +49,18 @@ struct SyncStatusJson { failed: u64, #[serde(skip_serializing)] endpoint: String, + /// Last heartbeat rejection (e.g. a 403 org/identity mismatch), if the + /// most recent heartbeat was rejected by the cloud. Cleared on the next + /// accepted heartbeat. + #[serde(skip_serializing_if = "Option::is_none")] + heartbeat_rejection: Option, +} + +#[derive(Debug, Serialize)] +struct HeartbeatRejectionJson { + status: u16, + message: String, + rejected_secs_ago: Option, } #[derive(Debug, Serialize)] @@ -99,7 +111,8 @@ pub async fn run(config_path: Option, json: bool) -> Result { && proxy.ca_valid_until.is_some() && ca_trusted && !runtime_degraded - && (!config.cloud.enabled || sync.failed == 0); + && (!config.cloud.enabled + || (sync.failed == 0 && sync.heartbeat_rejection.is_none())); let payload = StatusJson { proxy, @@ -237,6 +250,19 @@ fn render_human(status: &StatusJson) { status.sync.queued, status.sync.failed ); println!("Endpoint: {}", status.sync.endpoint); + if let Some(rejection) = status.sync.heartbeat_rejection.as_ref() { + let ago = rejection + .rejected_secs_ago + .map(format_ago) + .unwrap_or_else(|| "recently".to_string()); + println!( + "Heartbeat: REJECTED (HTTP {}, {}) — {}", + rejection.status, ago, rejection.message + ); + println!( + " Fix: re-enroll this device — `soth enroll --new-device-id `" + ); + } println!(); println!("LAST 24 HOURS"); println!("----------------------------------------"); @@ -265,6 +291,9 @@ fn render_human(status: &StatusJson) { if proxy_runtime_degraded(&status.proxy) { reasons.push("bundle runtime degraded"); } + if status.sync.heartbeat_rejection.is_some() { + reasons.push("cloud heartbeat rejected (re-enroll)"); + } if reasons.is_empty() { style::warning("degraded"); } else { @@ -467,6 +496,23 @@ fn collect_sync_status( let last_sync = read_sync_state(conn, "last_sync_timestamp")?.and_then(|value| parse_sync_age(&value, now)); + // Surface the last heartbeat rejection (e.g. a 403 org mismatch) so a + // "Last heartbeat: never" line isn't the only signal. Only meaningful + // when cloud sync is enabled; best-effort read (a corrupt sidecar just + // yields None rather than failing the whole status command). + let heartbeat_rejection = if config.cloud.enabled { + soth_sync::heartbeat_rejection::read() + .ok() + .flatten() + .map(|r| HeartbeatRejectionJson { + status: r.status, + message: summarize_status_line(&r.message, 200), + rejected_secs_ago: seconds_since_epoch(r.rejected_at, now), + }) + } else { + None + }; + Ok(SyncStatusJson { last_heartbeat_secs: last_sync, queued, @@ -474,9 +520,16 @@ fn collect_sync_status( // Sync/heartbeat targets the ingest endpoint; report the actual URL // the runtime is using so `soth status` reflects the live wire. endpoint: config.cloud.resolved_ingest_endpoint(), + heartbeat_rejection, }) } +/// Whole seconds between a Unix-epoch timestamp and `now`, clamped at 0. +fn seconds_since_epoch(epoch_secs: u64, now: DateTime) -> Option { + let then = Utc.timestamp_opt(epoch_secs as i64, 0).single()?; + Some((now - then).num_seconds().max(0)) +} + fn collect_last_24h(conn: &rusqlite::Connection, now: DateTime) -> Result { if !table_exists(conn, "intercept_records")? { return Ok(Last24hJson { diff --git a/crates/soth-cli/src/style.rs b/crates/soth-cli/src/style.rs index 3787edf..8ce32f5 100644 --- a/crates/soth-cli/src/style.rs +++ b/crates/soth-cli/src/style.rs @@ -1,5 +1,6 @@ //! Minimal CLI styling utilities for the current production command surface. +use crate::logging::use_ansi_colors; use owo_colors::OwoColorize; pub const CHECK: &str = "\u{2713}"; @@ -10,25 +11,49 @@ pub const ARROW_RIGHT: &str = "\u{2192}"; const CIRCLE_FILLED: &str = "\u{25CF}"; pub fn success(msg: &str) { - println!("{} {}", CHECK.green(), msg); + if use_ansi_colors() { + println!("{} {}", CHECK.green(), msg); + } else { + println!("{CHECK} {msg}"); + } } pub fn warning(msg: &str) { - println!("{} {}", WARNING.yellow(), msg); + if use_ansi_colors() { + println!("{} {}", WARNING.yellow(), msg); + } else { + println!("{WARNING} {msg}"); + } } pub fn info(msg: &str) { - println!("{} {}", CIRCLE_FILLED.cyan(), msg); + if use_ansi_colors() { + println!("{} {}", CIRCLE_FILLED.cyan(), msg); + } else { + println!("{CIRCLE_FILLED} {msg}"); + } } pub fn kv(key: &str, value: &str) { - println!(" {}: {}", key.dimmed(), value); + if use_ansi_colors() { + println!(" {}: {}", key.dimmed(), value); + } else { + println!(" {key}: {value}"); + } } pub fn success_prefix() -> String { - CHECK.green().to_string() + if use_ansi_colors() { + CHECK.green().to_string() + } else { + CHECK.to_string() + } } pub fn highlight(text: &str) -> String { - text.cyan().bold().to_string() + if use_ansi_colors() { + text.cyan().bold().to_string() + } else { + text.to_string() + } } diff --git a/crates/soth-sync/src/heartbeat.rs b/crates/soth-sync/src/heartbeat.rs index 5f8c56f..f9a29c2 100644 --- a/crates/soth-sync/src/heartbeat.rs +++ b/crates/soth-sync/src/heartbeat.rs @@ -1,4 +1,5 @@ use crate::api_types::{HeartbeatRequest, HeartbeatResponse}; +use crate::heartbeat_rejection::{self, HeartbeatRejection}; use crate::http_client::SothHttpClient; use anyhow::Context; use tracing::warn; @@ -34,15 +35,35 @@ impl HeartbeatSender { Ok(body) => body, Err(error) => format!(""), }; + let message = truncate_heartbeat_error_body(body.as_str()); warn!( status = status.as_u16(), endpoint = %url, - body = %truncate_heartbeat_error_body(body.as_str()), + body = %message, "heartbeat push rejected by server" ); + // Persist the rejection so `soth status` / `soth doctor` can + // explain *why* heartbeats stopped (e.g. a 403 org/identity + // mismatch) and point at the re-enroll escape hatch. Best-effort: + // a disk failure must not abort the heartbeat loop. + let entry = HeartbeatRejection { + rejected_at: heartbeat_rejection::now_epoch_secs(), + status: status.as_u16(), + message, + endpoint: url, + }; + if let Err(error) = heartbeat_rejection::write(&entry) { + warn!(error = %error, "failed to persist heartbeat rejection sidecar"); + } return Ok(None); } + // Accepted — clear any stale rejection sidecar from a prior failure so + // recovered devices don't keep alarming `soth status`. Best-effort. + if let Err(error) = heartbeat_rejection::clear() { + warn!(error = %error, "failed to clear heartbeat rejection sidecar"); + } + let decoded = response .json::() .await diff --git a/crates/soth-sync/src/heartbeat_rejection.rs b/crates/soth-sync/src/heartbeat_rejection.rs new file mode 100644 index 0000000..1365580 --- /dev/null +++ b/crates/soth-sync/src/heartbeat_rejection.rs @@ -0,0 +1,99 @@ +//! Persists the last heartbeat rejection so out-of-process readers +//! (`soth status`, `soth doctor`) can surface *why* the daemon stopped +//! heartbeating instead of just showing "Last heartbeat: never". +//! +//! Lives at `~/.soth/run/heartbeat_rejection.json`. Written whenever the +//! cloud responds to a heartbeat with a non-2xx status (most commonly a +//! 403 org/identity mismatch after an org migration), and cleared on the +//! next accepted heartbeat. Best-effort throughout — a disk failure here +//! must never abort the heartbeat loop. + +use anyhow::{anyhow, Context, Result}; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; +use std::time::{SystemTime, UNIX_EPOCH}; + +/// On-disk shape for `~/.soth/run/heartbeat_rejection.json`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HeartbeatRejection { + /// Unix epoch seconds when the rejection was observed. + pub rejected_at: u64, + /// HTTP status the cloud returned (e.g. 403). + pub status: u16, + /// Truncated response body / server message, for operator context. + pub message: String, + /// The endpoint URL the heartbeat targeted. + pub endpoint: String, +} + +/// Resolve `~/.soth/run/heartbeat_rejection.json`. +pub fn default_path() -> Result { + let home = dirs::home_dir().ok_or_else(|| anyhow!("could not resolve home directory"))?; + Ok(home + .join(".soth") + .join("run") + .join("heartbeat_rejection.json")) +} + +pub fn write(entry: &HeartbeatRejection) -> Result { + let path = default_path()?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .with_context(|| format!("creating {}", parent.display()))?; + } + let body = serde_json::to_vec_pretty(entry).context("serializing heartbeat_rejection.json")?; + std::fs::write(&path, body).with_context(|| format!("writing {}", path.display()))?; + Ok(path) +} + +pub fn read() -> Result> { + let path = default_path()?; + match std::fs::read(&path) { + Ok(body) => { + let parsed: HeartbeatRejection = serde_json::from_slice(&body) + .with_context(|| format!("parsing {}", path.display()))?; + Ok(Some(parsed)) + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(None), + Err(e) => Err(anyhow::Error::from(e).context(format!("reading {}", path.display()))), + } +} + +/// Best-effort delete. Called on the next accepted heartbeat so a stale +/// rejection doesn't keep alarming `soth status` after recovery. +pub fn clear() -> Result<()> { + let path = default_path()?; + match std::fs::remove_file(&path) { + Ok(()) => Ok(()), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(e) => Err(anyhow::Error::from(e).context(format!("removing {}", path.display()))), + } +} + +pub fn now_epoch_secs() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn roundtrip_serde_preserves_fields() { + let entry = HeartbeatRejection { + rejected_at: 1_700_000_000, + status: 403, + message: "org mismatch".into(), + endpoint: "https://ingest.example.com/v1/edge/heartbeat".into(), + }; + let body = serde_json::to_vec(&entry).unwrap(); + let back: HeartbeatRejection = serde_json::from_slice(&body).unwrap(); + assert_eq!(back.status, 403); + assert_eq!(back.rejected_at, 1_700_000_000); + assert_eq!(back.message, "org mismatch"); + assert_eq!(back.endpoint, "https://ingest.example.com/v1/edge/heartbeat"); + } +} diff --git a/crates/soth-sync/src/lib.rs b/crates/soth-sync/src/lib.rs index 62a5c7e..45403da 100644 --- a/crates/soth-sync/src/lib.rs +++ b/crates/soth-sync/src/lib.rs @@ -23,6 +23,7 @@ pub mod config_puller; pub mod db; pub mod exchange; pub mod heartbeat; +pub mod heartbeat_rejection; pub mod http_client; pub mod metadata_pusher; pub mod registry_puller;