diff --git a/crates/cli/src/commands/doctor.rs b/crates/cli/src/commands/doctor.rs index 70bde5fe..364c868f 100644 --- a/crates/cli/src/commands/doctor.rs +++ b/crates/cli/src/commands/doctor.rs @@ -7,7 +7,7 @@ use platform::{ TrustDomainState, }; use serde::Serialize; -use state::{Database, JobStatus, PvPaths, RuntimeObservedStatus, StateError}; +use state::{Database, JobDiagnosticSubject, PvPaths, RuntimeObservedStatus, StateError}; use crate::args::DoctorArgs; use crate::environment::Environment; @@ -138,7 +138,7 @@ struct DoctorCheck { name: &'static str, message: String, detail: Option, - repair: Option<&'static str>, + repair: Option, #[serde(skip_serializing_if = "Option::is_none")] routing: Option, } @@ -161,18 +161,18 @@ impl DoctorCheck { name, message: message.into(), detail: None, - repair, + repair: repair.map(str::to_owned), routing: None, } } - fn fail(name: &'static str, message: impl Into, repair: &'static str) -> Self { + fn fail(name: &'static str, message: impl Into, repair: impl Into) -> Self { Self { status: CheckStatus::Fail, name, message: message.into(), detail: None, - repair: Some(repair), + repair: Some(repair.into()), routing: None, } } @@ -482,33 +482,32 @@ fn recent_jobs_check(database: Option<&Database>) -> Result>(); + let failed = database.unresolved_job_failures()?; if failed.is_empty() { return Ok(DoctorCheck::pass( "Recent jobs", - "no failed jobs in recent history", + "no unresolved failed jobs", )); } + let repair = repair_for_job_subject(&failed[0].subject); Ok(DoctorCheck::fail( "Recent jobs", - format!("{} failed job(s) in recent history", failed.len()), - "pv setup", + format!("{} unresolved failed job(s)", failed.len()), + repair, ) .with_detail( failed .into_iter() - .map(|job| { + .map(|failure| { + let job = failure.job; format!( - "{} {} {}: {}", + "{} {} {} at {}: {}", job.id, job.kind, job.scope, + job.finished_at.as_deref().unwrap_or(&job.started_at), job.error.unwrap_or_else(|| "failed".to_string()) ) }) @@ -517,6 +516,28 @@ fn recent_jobs_check(database: Option<&Database>) -> Result String { + match subject { + JobDiagnosticSubject::UpdateAssessment => "pv update".to_owned(), + JobDiagnosticSubject::Resource { name, track: _ } if name == "composer" => { + "pv composer:install".to_owned() + } + JobDiagnosticSubject::Resource { name, track } + if matches!( + name.as_str(), + "mailpit" | "mysql" | "postgres" | "redis" | "rustfs" + ) => + { + format!("pv {name}:install {track}") + } + JobDiagnosticSubject::SystemReconciliation + | JobDiagnosticSubject::GatewayRuntime + | JobDiagnosticSubject::Project { .. } + | JobDiagnosticSubject::Resource { .. } + | JobDiagnosticSubject::Other { .. } => "pv daemon:restart".to_owned(), + } +} + fn runtime_states_check(database: Option<&Database>) -> Result { let Some(database) = database else { return Ok(DoctorCheck::warn( diff --git a/crates/cli/src/commands/ports.rs b/crates/cli/src/commands/ports.rs index 809b9de7..253633ff 100644 --- a/crates/cli/src/commands/ports.rs +++ b/crates/cli/src/commands/ports.rs @@ -4,7 +4,10 @@ use std::process::ExitCode; use camino::{Utf8Path, Utf8PathBuf}; use platform::{PfConfReference, PfFileState, PfRedirectConfig}; -use state::{Database, GatewayPort, GatewayPortAssignments, PortOwner, PvPaths, StateError}; +use state::{ + Database, GatewayPort, GatewayPortAssignments, PortOwner, PvPaths, RuntimeObservedStatus, + RuntimeSubject, StateError, +}; use crate::args::PortsStatusArgs; use crate::environment::Environment; @@ -159,6 +162,12 @@ pub(crate) fn install( if active_config.as_ref() == Some(&config) { output.line("System pf redirect config already matches PV")?; + refresh_gateway_observation_after_pf_repair( + environment, + &paths, + &config, + &mut database, + )?; return Ok(ExitCode::SUCCESS); } @@ -183,11 +192,54 @@ pub(crate) fn install( had_http_assignment, had_https_assignment, )?; + refresh_gateway_observation_after_pf_repair(environment, &paths, &config, &mut database)?; output.line("Installed system pf redirect config")?; Ok(ExitCode::SUCCESS) } +fn refresh_gateway_observation_after_pf_repair( + environment: &impl Environment, + paths: &PvPaths, + config: &PfRedirectConfig, + database: &mut Database, +) -> Result<(), ExecuteError> { + if environment + .probe_gateway_redirects(config, &paths.ca_certificate()) + .is_ok() + { + database.record_runtime_observed_snapshot( + RuntimeSubject::Gateway, + RuntimeObservedStatus::Running, + Some("Gateway identity verified through ports 80 and 443 after PF repair"), + )?; + + return Ok(()); + } + + let pf_derived_observation = database + .runtime_observed_states()? + .into_iter() + .any(|state| { + state.subject == RuntimeSubject::Gateway + && state.status == RuntimeObservedStatus::Degraded + && state + .message + .as_deref() + .is_some_and(|message| message.starts_with("Low-port routing is ")) + }); + if pf_derived_observation { + database.record_runtime_observed_snapshot( + RuntimeSubject::Gateway, + RuntimeObservedStatus::Pending, + Some("Low-port routing repaired; Gateway readiness is pending reconciliation"), + )?; + } + let _request_result = environment.request_system_reconciliation(paths); + + Ok(()) +} + fn ensure_active_gateway_ports( environment: &impl Environment, config: &PfRedirectConfig, diff --git a/crates/cli/src/commands/status.rs b/crates/cli/src/commands/status.rs index 4f7d55b9..5c54e5fd 100644 --- a/crates/cli/src/commands/status.rs +++ b/crates/cli/src/commands/status.rs @@ -8,7 +8,7 @@ use platform::{ }; use serde::Serialize; use state::{ - Database, JobRecord, JobStatus, ManagedResourceDesiredState, ManagedResourceTrackRecord, + Database, JobRecord, ManagedResourceDesiredState, ManagedResourceTrackRecord, ProjectEnvObservedStatus, ProjectRecord, PvPaths, RuntimeObservedStateRecord, RuntimeObservedStatus, RuntimeSubject, StateError, }; @@ -83,10 +83,9 @@ impl StatusSnapshot { }; let recent_errors = match &database { Some(database) => database - .recent_jobs()? + .unresolved_job_failures()? .into_iter() - .filter(|job| job.status == JobStatus::Failed) - .map(JobStatusSummary::from_job) + .map(|failure| JobStatusSummary::from_job(failure.job)) .collect::>(), None => Vec::new(), }; @@ -176,10 +175,11 @@ impl StatusSnapshot { } else { for job in &self.recent_errors { output.line(&format!( - " {} {} {} failed: {}", + " {} {} {} failed at {}: {}", job.id, job.kind, job.scope, + job.finished_at.as_deref().unwrap_or(&job.started_at), job.error.as_deref().unwrap_or("-"), ))?; } diff --git a/crates/cli/src/environment.rs b/crates/cli/src/environment.rs index 774a5049..e6e2eafa 100644 --- a/crates/cli/src/environment.rs +++ b/crates/cli/src/environment.rs @@ -141,6 +141,10 @@ pub trait Environment { Err("Gateway identity probing is unavailable in this environment".to_owned()) } + fn request_system_reconciliation(&self, _paths: &state::PvPaths) -> Result<(), String> { + Err("daemon reconciliation requests are unavailable in this environment".to_owned()) + } + fn remove_pf_redirects( &self, system_anchor_path: &Utf8Path, @@ -258,6 +262,12 @@ impl Environment for ProcessEnvironment { .map_err(|error| error.to_string()) } + fn request_system_reconciliation(&self, paths: &state::PvPaths) -> Result<(), String> { + daemon::submit_job_blocking(paths.clone(), "reconcile", "system") + .map(|_job| ()) + .map_err(|error| error.to_string()) + } + fn exec(&self, program: &Path, args: &[String]) -> io::Result { self.exec_with_env(program, args, &[]) } diff --git a/crates/cli/tests/doctor.rs b/crates/cli/tests/doctor.rs index b7e249de..de831338 100644 --- a/crates/cli/tests/doctor.rs +++ b/crates/cli/tests/doctor.rs @@ -15,7 +15,7 @@ use platform::{ ActivePfRedirectInspection, KeychainCertificate, KeychainTrustResult, LaunchAgentConfig, PfConfReference, PfRedirectConfig, ResolverConfig, }; -use state::{Database, PvPaths, RuntimeObservedStatus, RuntimeSubject}; +use state::{Database, JobDiagnosticSubject, PvPaths, RuntimeObservedStatus, RuntimeSubject}; #[derive(Debug)] struct TestEnvironment { @@ -191,6 +191,49 @@ fn doctor_fails_with_repair_commands() -> anyhow::Result<()> { Ok(()) } +#[test] +fn doctor_tracks_failure_repair_and_identical_recurrence() -> anyhow::Result<()> { + let tempdir = tempdir()?; + let home = tempdir.path().join("home"); + let paths = PvPaths::for_home(home.clone()); + let environment = TestEnvironment::new(&home); + seed_required_checks(&paths, &environment, true)?; + let mut database = Database::open(&paths)?; + let failure = database.start_job("reconcile", "project:project_1")?; + database.fail_job(&failure.id, "Gateway failed to start")?; + + let failed = run_doctor_with_health(&paths, &environment)?; + let repair = database.start_job("reconcile", "project:project_1")?; + database.complete_job_with_coverage( + &repair.id, + "Project reconciled", + &[ + JobDiagnosticSubject::Project { + id: "project_1".to_owned(), + }, + JobDiagnosticSubject::GatewayRuntime, + ], + )?; + let healthy = run_doctor_with_health(&paths, &environment)?; + let recurrence = database.start_job("reconcile", "project:project_1")?; + database.fail_job(&recurrence.id, "Gateway failed to start")?; + let recurring = run_doctor_with_health(&paths, &environment)?; + + assert_eq!(failed.exit_code, ExitCode::FAILURE); + assert_eq!(healthy.exit_code, ExitCode::SUCCESS); + assert_eq!(recurring.exit_code, ExitCode::FAILURE); + assert!(failed.stderr.is_empty()); + assert!(healthy.stderr.is_empty()); + assert!(recurring.stderr.is_empty()); + assert_doctor_snapshot( + "doctor_tracks_failure_repair_and_identical_recurrence", + tempdir.path(), + (failed, healthy, recurring), + ); + + Ok(()) +} + #[test] fn doctor_fails_when_daemon_socket_is_stale() -> anyhow::Result<()> { let tempdir = tempdir()?; @@ -345,6 +388,20 @@ fn run_pv(args: &[&str], environment: &impl Environment) -> anyhow::Result anyhow::Result { + if state::fs::path_exists(&paths.daemon_socket()) { + state::fs::delete_file(&paths.daemon_socket())?; + } + let health_server = spawn_health_server(&paths.daemon_socket())?; + let output = run_pv(&["doctor"], environment)?; + join_health_server(health_server)?; + + Ok(output) +} + fn seed_required_checks( paths: &PvPaths, environment: &TestEnvironment, diff --git a/crates/cli/tests/ports.rs b/crates/cli/tests/ports.rs index 324b8dde..a874084d 100644 --- a/crates/cli/tests/ports.rs +++ b/crates/cli/tests/ports.rs @@ -10,7 +10,7 @@ use camino_tempfile::tempdir; use cli::{Environment, run_with_environment}; use insta::assert_debug_snapshot; use platform::{ActivePfRedirectInspection, PfConfReference, PfRedirectConfig}; -use state::{Database, PortOwner, PvPaths, StateError}; +use state::{Database, PortOwner, PvPaths, RuntimeObservedStatus, RuntimeSubject, StateError}; #[derive(Debug)] struct TestEnvironment { @@ -24,6 +24,8 @@ struct TestEnvironment { active_pf_read_fails_when_unloaded: bool, unprivileged_pf_inspection_fails: bool, gateway_probe_succeeds: bool, + accepts_reconciliation_requests: bool, + reconciliation_requests: RefCell, operations: RefCell>, } @@ -45,6 +47,8 @@ impl TestEnvironment { active_pf_read_fails_when_unloaded: false, unprivileged_pf_inspection_fails: false, gateway_probe_succeeds: false, + accepts_reconciliation_requests: false, + reconciliation_requests: RefCell::new(0), operations: RefCell::new(Vec::new()), } } @@ -68,6 +72,11 @@ impl TestEnvironment { self.gateway_probe_succeeds = true; self } + + fn with_reconciliation_requests_succeeding(mut self) -> Self { + self.accepts_reconciliation_requests = true; + self + } } impl Environment for TestEnvironment { @@ -189,6 +198,15 @@ impl Environment for TestEnvironment { } } + fn request_system_reconciliation(&self, _paths: &PvPaths) -> Result<(), String> { + if !self.accepts_reconciliation_requests { + return Err("daemon unavailable".to_owned()); + } + *self.reconciliation_requests.borrow_mut() += 1; + + Ok(()) + } + fn remove_pf_redirects( &self, system_anchor_path: &Utf8Path, @@ -258,6 +276,122 @@ fn ports_install_writes_prepared_and_system_pf_artifacts() -> anyhow::Result<()> Ok(()) } +#[test] +fn ports_install_records_healthy_gateway_after_public_identity_probe() -> anyhow::Result<()> { + let tempdir = tempdir()?; + let home = tempdir.path().join("home"); + let current_dir = tempdir.path().join("work"); + let system_anchor_path = tempdir.path().join("etc/pf.anchors/com.prvious.pv"); + let system_pf_conf_path = tempdir.path().join("etc/pf.conf"); + let environment = TestEnvironment::new( + &home, + ¤t_dir, + &system_anchor_path, + &system_pf_conf_path, + ) + .with_gateway_probe_succeeding(); + let paths = pv_paths(&home); + + let output = run_pv(&["ports:install"], &environment)?; + let gateway = Database::open(&paths)? + .runtime_observed_states()? + .into_iter() + .find(|state| state.subject == RuntimeSubject::Gateway) + .ok_or_else(|| anyhow::anyhow!("missing Gateway observation"))?; + + assert_eq!(output.exit_code, ExitCode::SUCCESS); + assert_eq!(gateway.status, RuntimeObservedStatus::Running); + assert_eq!( + gateway.message.as_deref(), + Some("Gateway identity verified through ports 80 and 443 after PF repair") + ); + assert_eq!(*environment.reconciliation_requests.borrow(), 0); + + Ok(()) +} + +#[test] +fn ports_install_invalidates_pf_degradation_and_requests_reconciliation() -> anyhow::Result<()> { + let tempdir = tempdir()?; + let home = tempdir.path().join("home"); + let current_dir = tempdir.path().join("work"); + let system_anchor_path = tempdir.path().join("etc/pf.anchors/com.prvious.pv"); + let system_pf_conf_path = tempdir.path().join("etc/pf.conf"); + let environment = TestEnvironment::new( + &home, + ¤t_dir, + &system_anchor_path, + &system_pf_conf_path, + ) + .with_reconciliation_requests_succeeding(); + let paths = pv_paths(&home); + let mut database = Database::open(&paths)?; + database.record_runtime_observed_snapshot( + RuntimeSubject::Gateway, + RuntimeObservedStatus::Degraded, + Some("Low-port routing is inactive; run `pv ports:install` to restore ports 80 and 443"), + )?; + drop(database); + + let output = run_pv(&["ports:install"], &environment)?; + let gateway = Database::open(&paths)? + .runtime_observed_states()? + .into_iter() + .find(|state| state.subject == RuntimeSubject::Gateway) + .ok_or_else(|| anyhow::anyhow!("missing Gateway observation"))?; + + assert_eq!(output.exit_code, ExitCode::SUCCESS); + assert_eq!(gateway.status, RuntimeObservedStatus::Pending); + assert_eq!( + gateway.message.as_deref(), + Some("Low-port routing repaired; Gateway readiness is pending reconciliation") + ); + assert_eq!(*environment.reconciliation_requests.borrow(), 1); + + Ok(()) +} + +#[test] +fn ports_install_preserves_unrelated_gateway_failure_observation() -> anyhow::Result<()> { + let tempdir = tempdir()?; + let home = tempdir.path().join("home"); + let current_dir = tempdir.path().join("work"); + let system_anchor_path = tempdir.path().join("etc/pf.anchors/com.prvious.pv"); + let system_pf_conf_path = tempdir.path().join("etc/pf.conf"); + let environment = TestEnvironment::new( + &home, + ¤t_dir, + &system_anchor_path, + &system_pf_conf_path, + ) + .with_reconciliation_requests_succeeding(); + let paths = pv_paths(&home); + let mut database = Database::open(&paths)?; + database.record_runtime_observed_snapshot( + RuntimeSubject::Gateway, + RuntimeObservedStatus::Degraded, + Some("Gateway config validation failed"), + )?; + drop(database); + + let output = run_pv(&["ports:install"], &environment)?; + let gateway = Database::open(&paths)? + .runtime_observed_states()? + .into_iter() + .find(|state| state.subject == RuntimeSubject::Gateway) + .ok_or_else(|| anyhow::anyhow!("missing Gateway observation"))?; + + assert_eq!(output.exit_code, ExitCode::SUCCESS); + assert_eq!(gateway.status, RuntimeObservedStatus::Degraded); + assert_eq!( + gateway.message.as_deref(), + Some("Gateway config validation failed") + ); + assert_eq!(*environment.reconciliation_requests.borrow(), 1); + + Ok(()) +} + #[test] fn ports_install_refuses_non_pv_owned_system_anchor() -> anyhow::Result<()> { let tempdir = tempdir()?; diff --git a/crates/cli/tests/snapshots/doctor__doctor_fails_when_active_pf_redirects_are_missing.snap b/crates/cli/tests/snapshots/doctor__doctor_fails_when_active_pf_redirects_are_missing.snap index 28c8d325..598076ae 100644 --- a/crates/cli/tests/snapshots/doctor__doctor_fails_when_active_pf_redirects_are_missing.snap +++ b/crates/cli/tests/snapshots/doctor__doctor_fails_when_active_pf_redirects_are_missing.snap @@ -8,6 +8,6 @@ RunOutput { 1, ), ), - stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 9 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[pass] DNS config: system resolver uses port 35353\n[fail] Port redirect config: low-port redirects are inactive\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP -, HTTPS -; observed: \n repair: `pv ports:install`\n[pass] Local CA trust: system trust matches fingerprint \n[pass] Recent jobs: no failed jobs in recent history\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 9 passed, 0 warning(s), 1 failed\n", + stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 10 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[pass] DNS config: system resolver uses port 35353\n[fail] Port redirect config: low-port redirects are inactive\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP -, HTTPS -; observed: \n repair: `pv ports:install`\n[pass] Local CA trust: system trust matches fingerprint \n[pass] Recent jobs: no unresolved failed jobs\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 9 passed, 0 warning(s), 1 failed\n", stderr: "", } diff --git a/crates/cli/tests/snapshots/doctor__doctor_fails_when_daemon_socket_is_stale.snap b/crates/cli/tests/snapshots/doctor__doctor_fails_when_daemon_socket_is_stale.snap index fc85c0cf..e5c53654 100644 --- a/crates/cli/tests/snapshots/doctor__doctor_fails_when_daemon_socket_is_stale.snap +++ b/crates/cli/tests/snapshots/doctor__doctor_fails_when_daemon_socket_is_stale.snap @@ -8,6 +8,6 @@ RunOutput { 1, ), ), - stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 9 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[fail] Daemon socket: daemon socket is present but daemon did not answer health check\n path: /home/.pv/run/pv.sock; error: I/O error: Socket operation on non-socket (os error 38)\n repair: `pv daemon:restart`\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[pass] Recent jobs: no failed jobs in recent history\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 9 passed, 0 warning(s), 1 failed\n", + stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 10 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[fail] Daemon socket: daemon socket is present but daemon did not answer health check\n path: /home/.pv/run/pv.sock; error: I/O error: Socket operation on non-socket (os error 38)\n repair: `pv daemon:restart`\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[pass] Recent jobs: no unresolved failed jobs\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 9 passed, 0 warning(s), 1 failed\n", stderr: "", } diff --git a/crates/cli/tests/snapshots/doctor__doctor_fails_when_system_ca_trust_is_missing.snap b/crates/cli/tests/snapshots/doctor__doctor_fails_when_system_ca_trust_is_missing.snap index f95ae423..8e0991f0 100644 --- a/crates/cli/tests/snapshots/doctor__doctor_fails_when_system_ca_trust_is_missing.snap +++ b/crates/cli/tests/snapshots/doctor__doctor_fails_when_system_ca_trust_is_missing.snap @@ -8,6 +8,6 @@ RunOutput { 1, ), ), - stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 9 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[fail] Local CA trust: local CA is not trusted in the System keychain\n fingerprint: \n repair: `pv ca:trust`\n[pass] Recent jobs: no failed jobs in recent history\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 9 passed, 0 warning(s), 1 failed\n", + stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 10 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[fail] Local CA trust: local CA is not trusted in the System keychain\n fingerprint: \n repair: `pv ca:trust`\n[pass] Recent jobs: no unresolved failed jobs\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 9 passed, 0 warning(s), 1 failed\n", stderr: "", } diff --git a/crates/cli/tests/snapshots/doctor__doctor_fails_when_system_resolver_is_missing.snap b/crates/cli/tests/snapshots/doctor__doctor_fails_when_system_resolver_is_missing.snap index 6a38c5eb..98281e2b 100644 --- a/crates/cli/tests/snapshots/doctor__doctor_fails_when_system_resolver_is_missing.snap +++ b/crates/cli/tests/snapshots/doctor__doctor_fails_when_system_resolver_is_missing.snap @@ -8,6 +8,6 @@ RunOutput { 1, ), ), - stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 9 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[fail] DNS config: system resolver config is missing\n path: /home/etc/resolver/test\n repair: `pv dns:install`\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[pass] Recent jobs: no failed jobs in recent history\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 9 passed, 0 warning(s), 1 failed\n", + stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 10 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[fail] DNS config: system resolver config is missing\n path: /home/etc/resolver/test\n repair: `pv dns:install`\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[pass] Recent jobs: no unresolved failed jobs\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 9 passed, 0 warning(s), 1 failed\n", stderr: "", } diff --git a/crates/cli/tests/snapshots/doctor__doctor_fails_with_repair_commands.snap b/crates/cli/tests/snapshots/doctor__doctor_fails_with_repair_commands.snap index 09e98fa2..ec1eec8a 100644 --- a/crates/cli/tests/snapshots/doctor__doctor_fails_with_repair_commands.snap +++ b/crates/cli/tests/snapshots/doctor__doctor_fails_with_repair_commands.snap @@ -8,6 +8,6 @@ RunOutput { 1, ), ), - stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 9 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[fail] Daemon socket: daemon socket is missing\n path: /home/.pv/run/pv.sock\n repair: `pv daemon:restart`\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[fail] Recent jobs: 1 failed job(s) in recent history\n reconcile system: Gateway failed to start\n repair: `pv setup`\n[fail] Runtime states: 1 degraded or failed runtime observation(s)\n gateway Gateway failed to start\n repair: `pv daemon:restart`\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 7 passed, 0 warning(s), 3 failed\n", + stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 10 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[fail] Daemon socket: daemon socket is missing\n path: /home/.pv/run/pv.sock\n repair: `pv daemon:restart`\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[fail] Recent jobs: 1 unresolved failed job(s)\n reconcile system at : Gateway failed to start\n repair: `pv daemon:restart`\n[fail] Runtime states: 1 degraded or failed runtime observation(s)\n gateway Gateway failed to start\n repair: `pv daemon:restart`\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 7 passed, 0 warning(s), 3 failed\n", stderr: "", } diff --git a/crates/cli/tests/snapshots/doctor__doctor_passes_when_required_checks_pass.snap b/crates/cli/tests/snapshots/doctor__doctor_passes_when_required_checks_pass.snap index 8b722370..6c250d5a 100644 --- a/crates/cli/tests/snapshots/doctor__doctor_passes_when_required_checks_pass.snap +++ b/crates/cli/tests/snapshots/doctor__doctor_passes_when_required_checks_pass.snap @@ -9,7 +9,7 @@ expression: snapshot 0, ), ), - stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 9 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[pass] Recent jobs: no failed jobs in recent history\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 10 passed, 0 warning(s), 0 failed\n", + stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 10 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[pass] Recent jobs: no unresolved failed jobs\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 10 passed, 0 warning(s), 0 failed\n", stderr: "", }, RunOutput { @@ -18,7 +18,7 @@ expression: snapshot 0, ), ), - stdout: "{\"checks\":[{\"status\":\"pass\",\"name\":\"State layout\",\"message\":\"9 PV-owned directories have user-only permissions\",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Database\",\"message\":\"read-only open succeeded; 9 migrations applied\",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Daemon LaunchAgent\",\"message\":\"PV-owned LaunchAgent is installed\",\"detail\":\"path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\",\"repair\":null},{\"status\":\"pass\",\"name\":\"Daemon socket\",\"message\":\"daemon answered health check\",\"detail\":\"path: /home/.pv/run/pv.sock\",\"repair\":null},{\"status\":\"pass\",\"name\":\"DNS config\",\"message\":\"system resolver uses port 35353\",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Port redirect config\",\"message\":\"low-port routing is active\",\"detail\":\"evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \",\"repair\":null,\"routing\":{\"state\":\"active\",\"evidence\":\"pfctl\",\"expected_http_port\":48080,\"expected_https_port\":48443,\"active_http_port\":48080,\"active_https_port\":48443,\"observed_at\":\"\"}},{\"status\":\"pass\",\"name\":\"Local CA trust\",\"message\":\"system trust matches fingerprint \",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Recent jobs\",\"message\":\"no failed jobs in recent history\",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Runtime states\",\"message\":\"no degraded or failed runtime observations\",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Artifact manifest cache\",\"message\":\"cached manifest is present\",\"detail\":\"path: /home/.pv/downloads/manifest.json\",\"repair\":null}]}\n", + stdout: "{\"checks\":[{\"status\":\"pass\",\"name\":\"State layout\",\"message\":\"9 PV-owned directories have user-only permissions\",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Database\",\"message\":\"read-only open succeeded; 10 migrations applied\",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Daemon LaunchAgent\",\"message\":\"PV-owned LaunchAgent is installed\",\"detail\":\"path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\",\"repair\":null},{\"status\":\"pass\",\"name\":\"Daemon socket\",\"message\":\"daemon answered health check\",\"detail\":\"path: /home/.pv/run/pv.sock\",\"repair\":null},{\"status\":\"pass\",\"name\":\"DNS config\",\"message\":\"system resolver uses port 35353\",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Port redirect config\",\"message\":\"low-port routing is active\",\"detail\":\"evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \",\"repair\":null,\"routing\":{\"state\":\"active\",\"evidence\":\"pfctl\",\"expected_http_port\":48080,\"expected_https_port\":48443,\"active_http_port\":48080,\"active_https_port\":48443,\"observed_at\":\"\"}},{\"status\":\"pass\",\"name\":\"Local CA trust\",\"message\":\"system trust matches fingerprint \",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Recent jobs\",\"message\":\"no unresolved failed jobs\",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Runtime states\",\"message\":\"no degraded or failed runtime observations\",\"detail\":null,\"repair\":null},{\"status\":\"pass\",\"name\":\"Artifact manifest cache\",\"message\":\"cached manifest is present\",\"detail\":\"path: /home/.pv/downloads/manifest.json\",\"repair\":null}]}\n", stderr: "", }, ) diff --git a/crates/cli/tests/snapshots/doctor__doctor_tracks_failure_repair_and_identical_recurrence.snap b/crates/cli/tests/snapshots/doctor__doctor_tracks_failure_repair_and_identical_recurrence.snap new file mode 100644 index 00000000..c7cbdca4 --- /dev/null +++ b/crates/cli/tests/snapshots/doctor__doctor_tracks_failure_repair_and_identical_recurrence.snap @@ -0,0 +1,33 @@ +--- +source: crates/cli/tests/doctor.rs +expression: snapshot +--- +( + RunOutput { + exit_code: ExitCode( + unix_exit_status( + 1, + ), + ), + stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 10 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[fail] Recent jobs: 1 unresolved failed job(s)\n reconcile project:project_1 at : Gateway failed to start\n repair: `pv daemon:restart`\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 9 passed, 0 warning(s), 1 failed\n", + stderr: "", + }, + RunOutput { + exit_code: ExitCode( + unix_exit_status( + 0, + ), + ), + stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 10 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[pass] Recent jobs: no unresolved failed jobs\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 10 passed, 0 warning(s), 0 failed\n", + stderr: "", + }, + RunOutput { + exit_code: ExitCode( + unix_exit_status( + 1, + ), + ), + stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 10 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[fail] Recent jobs: 1 unresolved failed job(s)\n reconcile project:project_1 at : Gateway failed to start\n repair: `pv daemon:restart`\n[pass] Runtime states: no degraded or failed runtime observations\n[pass] Artifact manifest cache: cached manifest is present\n path: /home/.pv/downloads/manifest.json\nSummary: 9 passed, 0 warning(s), 1 failed\n", + stderr: "", + }, +) diff --git a/crates/cli/tests/snapshots/doctor__doctor_warnings_do_not_fail.snap b/crates/cli/tests/snapshots/doctor__doctor_warnings_do_not_fail.snap index 2b1fbd36..93ecb062 100644 --- a/crates/cli/tests/snapshots/doctor__doctor_warnings_do_not_fail.snap +++ b/crates/cli/tests/snapshots/doctor__doctor_warnings_do_not_fail.snap @@ -8,6 +8,6 @@ RunOutput { 0, ), ), - stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 9 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[pass] Recent jobs: no failed jobs in recent history\n[pass] Runtime states: no degraded or failed runtime observations\n[warn] Artifact manifest cache: cached artifact manifest is missing\n path: /home/.pv/downloads/manifest.json\n repair: `pv setup`\nSummary: 9 passed, 1 warning(s), 0 failed\n", + stdout: "PV doctor\n[pass] State layout: 9 PV-owned directories have user-only permissions\n[pass] Database: read-only open succeeded; 10 migrations applied\n[pass] Daemon LaunchAgent: PV-owned LaunchAgent is installed\n path: /home/Library/LaunchAgents/com.prvious.pv.daemon.plist\n[pass] Daemon socket: daemon answered health check\n path: /home/.pv/run/pv.sock\n[pass] DNS config: system resolver uses port 35353\n[pass] Port redirect config: low-port routing is active\n evidence: pfctl; expected: HTTP 48080, HTTPS 48443; active: HTTP 48080, HTTPS 48443; observed: \n[pass] Local CA trust: system trust matches fingerprint \n[pass] Recent jobs: no unresolved failed jobs\n[pass] Runtime states: no degraded or failed runtime observations\n[warn] Artifact manifest cache: cached artifact manifest is missing\n path: /home/.pv/downloads/manifest.json\n repair: `pv setup`\nSummary: 9 passed, 1 warning(s), 0 failed\n", stderr: "", } diff --git a/crates/cli/tests/snapshots/status__status_reports_failed_jobs_as_failure.snap b/crates/cli/tests/snapshots/status__status_reports_failed_jobs_as_failure.snap deleted file mode 100644 index f9138984..00000000 --- a/crates/cli/tests/snapshots/status__status_reports_failed_jobs_as_failure.snap +++ /dev/null @@ -1,13 +0,0 @@ ---- -source: crates/cli/tests/status.rs -expression: snapshot ---- -RunOutput { - exit_code: ExitCode( - unix_exit_status( - 1, - ), - ), - stdout: "PV status\nOverall: failed\nDaemon: disabled\n LaunchAgent: missing\n Socket: missing\nIntegrations:\n DNS: missing\n Ports: inactive\n repair: `pv ports:install`\n CA: missing\nLogs: /home/.pv/logs\nManaged Resources:\n none\nProjects:\n none\nRecent errors:\n job_000001 reconcile project:acme failed: Gateway failed to start\n", - stderr: "", -} diff --git a/crates/cli/tests/snapshots/status__status_reports_runtime_and_resource_states.snap b/crates/cli/tests/snapshots/status__status_reports_runtime_and_resource_states.snap index 47cc0a25..f7d1c1c3 100644 --- a/crates/cli/tests/snapshots/status__status_reports_runtime_and_resource_states.snap +++ b/crates/cli/tests/snapshots/status__status_reports_runtime_and_resource_states.snap @@ -8,6 +8,6 @@ RunOutput { 1, ), ), - stdout: "PV status\nOverall: failed\nDaemon: disabled\n LaunchAgent: missing\n Socket: missing\nIntegrations:\n DNS: missing\n Ports: inactive\n repair: `pv ports:install`\n CA: missing\nLogs: /home/.pv/logs\nManaged Resources:\n mailpit 1 running projects=0 version=1.20.0-pv1\n mysql 8.0 running projects=0 version=8.0.36-pv1\n php 8.4 not-running projects=0 version=8.4.8-pv1\n postgres 16 running projects=0 version=16.4-pv1\n redis 7 running projects=0 version=7.2.5-pv1\n rustfs 1 running projects=0 version=1.0.0-pv1\nRuntimes:\n worker:8.4 running PHP worker is ready\nProjects:\n none\nRecent errors:\n job_000001 reconcile resource:redis:7 failed: Redis failed readiness\n", + stdout: "PV status\nOverall: failed\nDaemon: disabled\n LaunchAgent: missing\n Socket: missing\nIntegrations:\n DNS: missing\n Ports: inactive\n repair: `pv ports:install`\n CA: missing\nLogs: /home/.pv/logs\nManaged Resources:\n mailpit 1 running projects=0 version=1.20.0-pv1\n mysql 8.0 running projects=0 version=8.0.36-pv1\n php 8.4 not-running projects=0 version=8.4.8-pv1\n postgres 16 running projects=0 version=16.4-pv1\n redis 7 running projects=0 version=7.2.5-pv1\n rustfs 1 running projects=0 version=1.0.0-pv1\nRuntimes:\n worker:8.4 running PHP worker is ready\nProjects:\n none\nRecent errors:\n job_000001 reconcile resource:redis:7 failed at : Redis failed readiness\n", stderr: "", } diff --git a/crates/cli/tests/snapshots/status__status_tracks_failure_repair_and_identical_recurrence.snap b/crates/cli/tests/snapshots/status__status_tracks_failure_repair_and_identical_recurrence.snap new file mode 100644 index 00000000..edebbdd0 --- /dev/null +++ b/crates/cli/tests/snapshots/status__status_tracks_failure_repair_and_identical_recurrence.snap @@ -0,0 +1,33 @@ +--- +source: crates/cli/tests/status.rs +expression: snapshot +--- +( + RunOutput { + exit_code: ExitCode( + unix_exit_status( + 1, + ), + ), + stdout: "PV status\nOverall: failed\nDaemon: disabled\n LaunchAgent: missing\n Socket: missing\nIntegrations:\n DNS: missing\n Ports: inactive\n repair: `pv ports:install`\n CA: missing\nLogs: /home/.pv/logs\nManaged Resources:\n none\nProjects:\n none\nRecent errors:\n job_000001 reconcile project:acme failed at : Gateway failed to start\n", + stderr: "", + }, + RunOutput { + exit_code: ExitCode( + unix_exit_status( + 0, + ), + ), + stdout: "PV status\nOverall: ok\nDaemon: disabled\n LaunchAgent: missing\n Socket: missing\nIntegrations:\n DNS: missing\n Ports: inactive\n repair: `pv ports:install`\n CA: missing\nLogs: /home/.pv/logs\nManaged Resources:\n none\nProjects:\n none\nRecent errors:\n none\n", + stderr: "", + }, + RunOutput { + exit_code: ExitCode( + unix_exit_status( + 1, + ), + ), + stdout: "PV status\nOverall: failed\nDaemon: disabled\n LaunchAgent: missing\n Socket: missing\nIntegrations:\n DNS: missing\n Ports: inactive\n repair: `pv ports:install`\n CA: missing\nLogs: /home/.pv/logs\nManaged Resources:\n none\nProjects:\n none\nRecent errors:\n job_000003 reconcile project:acme failed at : Gateway failed to start\n", + stderr: "", + }, +) diff --git a/crates/cli/tests/status.rs b/crates/cli/tests/status.rs index b417ef12..03900df7 100644 --- a/crates/cli/tests/status.rs +++ b/crates/cli/tests/status.rs @@ -14,8 +14,9 @@ use platform::{ }; use platform::{KeychainTrustResult, LaunchAgentConfig}; use state::{ - Database, LinkProjectInput, ManagedResourceTrackInstallInput, ProjectEnvObservedStatus, - ProjectEnvObservedWarningInput, PvPaths, RuntimeObservedStatus, RuntimeSubject, + Database, JobDiagnosticSubject, LinkProjectInput, ManagedResourceTrackInstallInput, + ProjectEnvObservedStatus, ProjectEnvObservedWarningInput, PvPaths, RuntimeObservedStatus, + RuntimeSubject, }; #[derive(Debug)] @@ -163,7 +164,7 @@ fn status_reports_current_launch_agent_with_stale_socket_as_down() -> anyhow::Re } #[test] -fn status_reports_failed_jobs_as_failure() -> anyhow::Result<()> { +fn status_tracks_failure_repair_and_identical_recurrence() -> anyhow::Result<()> { let tempdir = tempdir()?; let home = tempdir.path().join("home"); let paths = PvPaths::for_home(home.clone()); @@ -172,14 +173,30 @@ fn status_reports_failed_jobs_as_failure() -> anyhow::Result<()> { let job = database.start_job("reconcile", "project:acme")?; database.fail_job(&job.id, "Gateway failed to start")?; - let output = run_pv(&["status"], &environment)?; + let failed = run_pv(&["status"], &environment)?; - assert_eq!(output.exit_code, ExitCode::FAILURE); - assert!(output.stderr.is_empty()); + let repair = database.start_job("reconcile", "system")?; + database.complete_job_with_coverage( + &repair.id, + "System reconciled", + &[JobDiagnosticSubject::SystemReconciliation], + )?; + let healthy = run_pv(&["status"], &environment)?; + + let recurrence = database.start_job("reconcile", "project:acme")?; + database.fail_job(&recurrence.id, "Gateway failed to start")?; + let recurring = run_pv(&["status"], &environment)?; + + assert_eq!(failed.exit_code, ExitCode::FAILURE); + assert_eq!(healthy.exit_code, ExitCode::SUCCESS); + assert_eq!(recurring.exit_code, ExitCode::FAILURE); + assert!(failed.stderr.is_empty()); + assert!(healthy.stderr.is_empty()); + assert!(recurring.stderr.is_empty()); assert_status_snapshot( - "status_reports_failed_jobs_as_failure", + "status_tracks_failure_repair_and_identical_recurrence", tempdir.path(), - output, + (failed, healthy, recurring), ); Ok(()) diff --git a/crates/daemon/src/jobs.rs b/crates/daemon/src/jobs.rs index 86b61d91..f70f483e 100644 --- a/crates/daemon/src/jobs.rs +++ b/crates/daemon/src/jobs.rs @@ -12,7 +12,9 @@ use crate::project_env::reconcile_project_env_with_runtime_catalog_and_progress; use crate::reconciliation::{EnqueueResult, ReconciliationQueue, ReconciliationScope}; use crate::structured_log; use protocol::{DaemonEvent, DaemonResponse, DaemonTransport, write_line}; -use state::{Database, JobStatus, ManagedResourceDesiredState, ProjectRecord, PvPaths, StateError}; +use state::{ + Database, JobDiagnosticSubject, ManagedResourceDesiredState, ProjectRecord, PvPaths, StateError, +}; use tokio::io::AsyncWrite; use tokio::sync::mpsc::{Receiver, Sender, channel}; use tokio::time::{Duration, Instant, MissedTickBehavior, interval_at, timeout}; @@ -32,6 +34,11 @@ enum ForegroundJobEvent { }, } +struct CompletedUpdateJob { + summary: String, + reconciled: bool, +} + #[derive(Debug)] struct StreamedJobCompletion { result: Result, @@ -719,10 +726,14 @@ async fn complete_update_job_with_progress( let result = complete_update_job_inner(paths, runtime_catalog, progress).await; match &result { - Ok(summary) => { + Ok(completed) => { let mut database = Database::open(paths)?; - database.complete_job(job_id, summary)?; - structured_log::job_completed(paths, job_id, "update", "system", summary); + let mut coverage = vec![JobDiagnosticSubject::UpdateAssessment]; + if completed.reconciled { + coverage.push(JobDiagnosticSubject::SystemReconciliation); + } + database.complete_job_with_coverage(job_id, &completed.summary, &coverage)?; + structured_log::job_completed(paths, job_id, "update", "system", &completed.summary); } Err(error) => { let error_message = error.to_string(); @@ -732,14 +743,14 @@ async fn complete_update_job_with_progress( } } - result + result.map(|completed| completed.summary) } async fn complete_update_job_inner( paths: &PvPaths, runtime_catalog: Option<&ManagedResourceRuntimeCatalog>, progress: DaemonDownloadProgress, -) -> Result { +) -> Result { let report = if runtime_catalog.is_none() { let update_paths = paths.clone(); let update_progress = progress.clone(); @@ -759,7 +770,10 @@ async fn complete_update_job_inner( ) }?; if report.updated_count == 0 { - return Ok(unchanged_update_summary(&report)); + return Ok(CompletedUpdateJob { + summary: unchanged_update_summary(&report), + reconciled: false, + }); } let project_report = @@ -768,10 +782,13 @@ async fn complete_update_job_inner( let gateway_summary = reconcile_gateway_runtimes(paths).await?; let reconciliation_summary = system_reconciliation_summary(&project_report, &gateway_summary); - Ok(format!( - "updated {} artifact(s); reconciled: {reconciliation_summary}", - report.updated_count - )) + Ok(CompletedUpdateJob { + summary: format!( + "updated {} artifact(s); reconciled: {reconciliation_summary}", + report.updated_count + ), + reconciled: true, + }) } fn unchanged_update_summary(report: &ManagedResourceUpdateReport) -> String { @@ -795,8 +812,17 @@ async fn complete_managed_resource_reconciliation_with_progress( let summary = managed_resource_reconciliation_summary(name.as_str(), track.as_str(), &project_report); let mut database = Database::open(paths)?; - - database.complete_job(job_id, &summary)?; + let mut coverage = vec![JobDiagnosticSubject::Resource { + name: name.as_str().to_owned(), + track: track.as_str().to_owned(), + }]; + coverage.extend( + database + .projects()? + .into_iter() + .map(|project| JobDiagnosticSubject::Project { id: project.id }), + ); + database.complete_job_with_coverage(job_id, &summary, &coverage)?; Ok(summary) } @@ -879,8 +905,11 @@ async fn complete_gateway_reconciliation( ) -> Result { let summary = reconcile_gateway_runtimes(paths).await?; let mut database = Database::open(paths)?; - - database.complete_job(job_id, &summary)?; + database.complete_job_with_coverage( + job_id, + &summary, + &[JobDiagnosticSubject::GatewayRuntime], + )?; Ok(summary) } @@ -897,8 +926,11 @@ async fn complete_system_reconciliation_with_progress( let gateway_summary = reconcile_gateway_runtimes(paths).await?; let summary = system_reconciliation_summary(&project_report, &gateway_summary); let mut database = Database::open(paths)?; - - database.complete_job(job_id, &summary)?; + database.complete_job_with_coverage( + job_id, + &summary, + &[JobDiagnosticSubject::SystemReconciliation], + )?; Ok(summary) } @@ -924,8 +956,16 @@ async fn complete_project_reconciliation_with_progress( format!("{}; {gateway_summary}", project_env_summary.as_str()) }; let mut database = Database::open(paths)?; - - database.complete_job(job_id, &summary)?; + database.complete_job_with_coverage( + job_id, + &summary, + &[ + JobDiagnosticSubject::Project { + id: id.as_str().to_owned(), + }, + JobDiagnosticSubject::GatewayRuntime, + ], + )?; Ok(summary) } @@ -1203,12 +1243,14 @@ pub(crate) fn record_background_reconciliation_error( ) -> Result<(), DaemonError> { let error_message = error.to_string(); let mut database = Database::open(paths)?; - let already_recorded = database.recent_jobs()?.into_iter().any(|job| { - job.kind == "reconcile" - && job.scope == scope - && job.status == JobStatus::Failed - && job.error.as_deref() == Some(error_message.as_str()) - }); + let already_recorded = database + .unresolved_job_failures()? + .into_iter() + .any(|failure| { + failure.job.kind == "reconcile" + && failure.job.scope == scope + && failure.job.error.as_deref() == Some(error_message.as_str()) + }); if already_recorded { return Ok(()); @@ -2209,6 +2251,40 @@ mod tests { Ok(()) } + #[test] + fn background_error_deduplication_resets_after_successful_coverage() -> anyhow::Result<()> { + let tempdir = tempdir()?; + let paths = PvPaths::for_home(tempdir.path().join("home")); + let error = crate::DaemonError::Io(io::Error::other("background task failed")); + + record_background_reconciliation_error(&paths, "project:project_1", &error)?; + record_background_reconciliation_error(&paths, "project:project_1", &error)?; + let mut database = Database::open(&paths)?; + assert_eq!(database.recent_jobs()?.len(), 1); + let success = database.start_job("reconcile", "project:project_1")?; + database.complete_job_with_coverage( + &success.id, + "Project reconciled", + &[state::JobDiagnosticSubject::Project { + id: "project_1".to_owned(), + }], + )?; + drop(database); + + record_background_reconciliation_error(&paths, "project:project_1", &error)?; + + let database = Database::open(&paths)?; + let failed = database + .recent_jobs()? + .into_iter() + .filter(|job| job.status == JobStatus::Failed) + .collect::>(); + assert_eq!(failed.len(), 2); + assert_eq!(database.unresolved_job_failures()?.len(), 1); + + Ok(()) + } + #[test] fn background_reconciliation_error_writes_structured_daemon_log() -> anyhow::Result<()> { let tempdir = tempdir()?; diff --git a/crates/state/src/database.rs b/crates/state/src/database.rs index 647cdbc4..a0380e81 100644 --- a/crates/state/src/database.rs +++ b/crates/state/src/database.rs @@ -81,6 +81,107 @@ pub struct JobRecord { pub error: Option, } +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub enum JobDiagnosticSubject { + SystemReconciliation, + GatewayRuntime, + Project { id: String }, + Resource { name: String, track: String }, + UpdateAssessment, + Other { kind: String, scope: String }, +} + +#[derive(Clone, Debug, Eq, PartialEq)] +pub struct UnresolvedJobFailure { + pub job: JobRecord, + pub subject: JobDiagnosticSubject, +} + +impl JobDiagnosticSubject { + fn from_job_identity(kind: &str, scope: &str) -> Self { + if kind == "update" && scope == "system" { + return Self::UpdateAssessment; + } + if kind != "reconcile" { + return Self::Other { + kind: kind.to_owned(), + scope: scope.to_owned(), + }; + } + + let components = scope.split(':').collect::>(); + match components.as_slice() { + ["system"] => Self::SystemReconciliation, + ["project", id] if !id.is_empty() => Self::Project { + id: (*id).to_owned(), + }, + ["resource", name, _track] if matches!(*name, "php" | "frankenphp") => { + Self::GatewayRuntime + } + ["resource", name, track] if !name.is_empty() && !track.is_empty() => Self::Resource { + name: (*name).to_owned(), + track: (*track).to_owned(), + }, + _ => Self::Other { + kind: kind.to_owned(), + scope: scope.to_owned(), + }, + } + } + + fn database_identity(&self) -> (String, String) { + match self { + Self::SystemReconciliation => ("system_reconciliation".to_owned(), String::new()), + Self::GatewayRuntime => ("gateway_runtime".to_owned(), String::new()), + Self::Project { id } => ("project".to_owned(), id.clone()), + Self::Resource { name, track } => ("resource".to_owned(), format!("{name}:{track}")), + Self::UpdateAssessment => ("update_assessment".to_owned(), String::new()), + Self::Other { kind, scope } => (format!("other:{kind}"), scope.clone()), + } + } + + fn from_database(subject_kind: &str, subject_id: &str) -> Self { + match subject_kind { + "system_reconciliation" => Self::SystemReconciliation, + "gateway_runtime" => Self::GatewayRuntime, + "project" => Self::Project { + id: subject_id.to_owned(), + }, + "resource" => subject_id.split_once(':').map_or_else( + || Self::Other { + kind: subject_kind.to_owned(), + scope: subject_id.to_owned(), + }, + |(name, track)| Self::Resource { + name: name.to_owned(), + track: track.to_owned(), + }, + ), + "update_assessment" => Self::UpdateAssessment, + _ => Self::Other { + kind: subject_kind + .strip_prefix("other:") + .unwrap_or(subject_kind) + .to_owned(), + scope: subject_id.to_owned(), + }, + } + } + + fn covers(&self, failure: &Self) -> bool { + match self { + Self::SystemReconciliation => matches!( + failure, + Self::SystemReconciliation + | Self::GatewayRuntime + | Self::Project { .. } + | Self::Resource { .. } + ), + _ => self == failure, + } + } +} + #[derive(Clone, Debug, Eq, PartialEq)] pub struct PortRequest { owner: PortOwner, @@ -406,6 +507,13 @@ struct JobRecordRow { error: Option, } +#[derive(Clone, Debug, Eq, PartialEq)] +struct JobDiagnosticOutcome { + job_id: String, + subject: JobDiagnosticSubject, + outcome: String, +} + struct PortAssignmentRow { owner_kind: String, owner_id: String, @@ -557,6 +665,15 @@ impl Database { } pub fn complete_job(&mut self, id: &str, summary: &str) -> Result<(), StateError> { + self.complete_job_with_coverage(id, summary, &[]) + } + + pub fn complete_job_with_coverage( + &mut self, + id: &str, + summary: &str, + coverage: &[JobDiagnosticSubject], + ) -> Result<(), StateError> { let finished_at = timestamp()?; let updated = self.transaction(|transaction| { @@ -565,6 +682,9 @@ impl Database { params![JobStatus::Succeeded.as_str(), finished_at, summary, id], )?; if updated > 0 { + for subject in coverage { + insert_job_diagnostic_outcome(transaction, id, subject, "success")?; + } prune_old_jobs(transaction)?; } @@ -581,11 +701,20 @@ impl Database { let finished_at = timestamp()?; let updated = self.transaction(|transaction| { + let identity = transaction + .query_row( + "SELECT kind, scope FROM jobs WHERE id = ?1", + params![id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)), + ) + .optional()?; let updated = transaction.execute( "UPDATE jobs SET status = ?1, finished_at = ?2, error = ?3 WHERE id = ?4", params![JobStatus::Failed.as_str(), finished_at, error, id], )?; - if updated > 0 { + if let Some((kind, scope)) = identity { + let subject = JobDiagnosticSubject::from_job_identity(&kind, &scope); + insert_job_diagnostic_outcome(transaction, id, &subject, "failure")?; prune_old_jobs(transaction)?; } @@ -602,6 +731,22 @@ impl Database { let finished_at = timestamp()?; let updated = self.transaction(|transaction| { + let running_jobs = { + let mut statement = transaction + .prepare("SELECT id, kind, scope FROM jobs WHERE status = ?1 ORDER BY id")?; + let rows = statement.query_map(params![JobStatus::Running.as_str()], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + )) + })?; + let mut jobs = Vec::new(); + for row in rows { + jobs.push(row?); + } + jobs + }; let updated = transaction.execute( "UPDATE jobs SET status = ?1, finished_at = ?2, error = ?3 WHERE status = ?4", params![ @@ -612,6 +757,10 @@ impl Database { ], )?; if updated > 0 { + for (id, kind, scope) in running_jobs { + let subject = JobDiagnosticSubject::from_job_identity(&kind, &scope); + insert_job_diagnostic_outcome(transaction, &id, &subject, "failure")?; + } prune_old_jobs(transaction)?; } @@ -656,6 +805,140 @@ impl Database { Ok(jobs) } + pub fn unresolved_job_failures(&self) -> Result, StateError> { + let jobs = self.recent_jobs()?; + let outcomes = self.job_diagnostic_outcomes()?; + let runtime_states = self.runtime_observed_states()?; + let jobs_by_id = jobs + .iter() + .map(|job| (job.id.as_str(), job)) + .collect::>(); + let mut newer_successes = Vec::new(); + let mut visited_failures = BTreeSet::new(); + let mut recorded_failure_job_ids = BTreeSet::new(); + let mut unresolved = Vec::new(); + + for outcome in outcomes { + if outcome.outcome == "success" { + newer_successes.push(outcome.subject); + continue; + } + if outcome.outcome != "failure" { + continue; + } + recorded_failure_job_ids.insert(outcome.job_id.clone()); + let Some(job) = jobs_by_id.get(outcome.job_id.as_str()) else { + continue; + }; + if !visited_failures.insert(outcome.subject.clone()) + || newer_successes + .iter() + .any(|coverage| coverage.covers(&outcome.subject)) + || self.has_newer_healthy_observation(job, &outcome.subject, &runtime_states)? + { + continue; + } + unresolved.push(UnresolvedJobFailure { + job: (*job).clone(), + subject: outcome.subject, + }); + } + drop(jobs_by_id); + + for job in jobs { + if job.status != JobStatus::Failed || recorded_failure_job_ids.contains(&job.id) { + continue; + } + let subject = JobDiagnosticSubject::from_job_identity(&job.kind, &job.scope); + if !visited_failures.insert(subject.clone()) + || newer_successes + .iter() + .any(|coverage| coverage.covers(&subject)) + || self.has_newer_healthy_observation(&job, &subject, &runtime_states)? + { + continue; + } + unresolved.push(UnresolvedJobFailure { job, subject }); + } + + Ok(unresolved) + } + + fn job_diagnostic_outcomes(&self) -> Result, StateError> { + let mut statement = self.connection.prepare( + "SELECT job_id, subject_kind, subject_id, outcome + FROM job_diagnostic_outcomes + ORDER BY sequence DESC", + )?; + let rows = statement.query_map([], |row| { + let job_id = row.get::<_, String>(0)?; + let subject_kind = row.get::<_, String>(1)?; + let subject_id = row.get::<_, String>(2)?; + let outcome = row.get::<_, String>(3)?; + + Ok((job_id, subject_kind, subject_id, outcome)) + })?; + let mut outcomes = Vec::new(); + for row in rows { + let (job_id, subject_kind, subject_id, outcome) = row?; + let subject = JobDiagnosticSubject::from_database(&subject_kind, &subject_id); + outcomes.push(JobDiagnosticOutcome { + job_id, + subject, + outcome, + }); + } + + Ok(outcomes) + } + + fn has_newer_healthy_observation( + &self, + job: &JobRecord, + subject: &JobDiagnosticSubject, + runtime_states: &[RuntimeObservedStateRecord], + ) -> Result { + let failure_at = job.finished_at.as_deref().unwrap_or(&job.started_at); + let observed_at = match subject { + JobDiagnosticSubject::GatewayRuntime => runtime_states.iter().find_map(|state| { + (state.subject == RuntimeSubject::Gateway + && state.status == RuntimeObservedStatus::Running) + .then(|| state.observed_at.clone()) + }), + JobDiagnosticSubject::Project { id } => self + .project_env_observed_state(id)? + .filter(|state| { + matches!( + state.status, + ProjectEnvObservedStatus::Rendered | ProjectEnvObservedStatus::Warning + ) + }) + .map(|state| state.observed_at), + JobDiagnosticSubject::Resource { name, track } => { + runtime_states.iter().find_map(|state| { + let matches_subject = matches!( + &state.subject, + RuntimeSubject::Resource { + name: actual_name, + track: actual_track, + } if actual_name == name && actual_track == track + ); + (matches_subject + && matches!( + state.status, + RuntimeObservedStatus::Running | RuntimeObservedStatus::Stopped + )) + .then(|| state.observed_at.clone()) + }) + } + JobDiagnosticSubject::SystemReconciliation + | JobDiagnosticSubject::UpdateAssessment + | JobDiagnosticSubject::Other { .. } => None, + }; + + Ok(observed_at.is_some_and(|observed_at| observed_at.as_str() > failure_at)) + } + pub fn link_project( &mut self, input: LinkProjectInput, @@ -4226,6 +4509,26 @@ fn prune_old_jobs(transaction: &Transaction<'_>) -> rusqlite::Result<()> { Ok(()) } +fn insert_job_diagnostic_outcome( + transaction: &Transaction<'_>, + job_id: &str, + subject: &JobDiagnosticSubject, + outcome: &str, +) -> rusqlite::Result<()> { + let (subject_kind, subject_id) = subject.database_identity(); + transaction.execute( + "INSERT OR IGNORE INTO job_diagnostic_outcomes ( + job_id, + subject_kind, + subject_id, + outcome + ) VALUES (?1, ?2, ?3, ?4)", + params![job_id, subject_kind, subject_id, outcome], + )?; + + Ok(()) +} + fn timestamp() -> Result { let format = time::macros::format_description!("[year]-[month]-[day]T[hour]:[minute]:[second]Z"); diff --git a/crates/state/src/lib.rs b/crates/state/src/lib.rs index 5365c2b0..455c4a16 100644 --- a/crates/state/src/lib.rs +++ b/crates/state/src/lib.rs @@ -11,16 +11,17 @@ pub use app_release::{AppReleaseInstall, AppReleaseLayout}; pub use database::{ DNS_PREFERRED_PORT, Database, DatabaseInspection, EnvContextValues, GATEWAY_HTTP_PREFERRED_PORT, GATEWAY_HTTPS_PREFERRED_PORT, GatewayPort, GatewayPortAssignments, - JobRecord, JobStatus, LinkProjectInput, LinkProjectResult, LinkProjectStatus, - ManagedResourceDesiredState, ManagedResourceTrackInstallInput, ManagedResourceTrackRecord, - ManagedResourceTrackRemovalInput, PortAssignment, PortOwner, PortRequest, ProjectConfigWatch, - ProjectEnvAllocationContext, ProjectEnvObservedStateRecord, ProjectEnvObservedStatus, - ProjectEnvObservedWarningInput, ProjectEnvObservedWarningRecord, ProjectEnvResourceContext, - ProjectEnvStateContext, ProjectManagedResourceInput, ProjectManagedResourceRecord, ProjectMode, - ProjectPhpRuntimeInput, ProjectPhpRuntimeRecord, ProjectReconciliationStateInput, - ProjectRecord, RUNTIME_PORT_FALLBACK_END, RUNTIME_PORT_FALLBACK_START, ResourceAllocationInput, - ResourceAllocationRecord, ResourceAllocationStatus, RuntimeObservedStateRecord, - RuntimeObservedStatus, RuntimeSubject, php_runtime_key, + JobDiagnosticSubject, JobRecord, JobStatus, LinkProjectInput, LinkProjectResult, + LinkProjectStatus, ManagedResourceDesiredState, ManagedResourceTrackInstallInput, + ManagedResourceTrackRecord, ManagedResourceTrackRemovalInput, PortAssignment, PortOwner, + PortRequest, ProjectConfigWatch, ProjectEnvAllocationContext, ProjectEnvObservedStateRecord, + ProjectEnvObservedStatus, ProjectEnvObservedWarningInput, ProjectEnvObservedWarningRecord, + ProjectEnvResourceContext, ProjectEnvStateContext, ProjectManagedResourceInput, + ProjectManagedResourceRecord, ProjectMode, ProjectPhpRuntimeInput, ProjectPhpRuntimeRecord, + ProjectReconciliationStateInput, ProjectRecord, RUNTIME_PORT_FALLBACK_END, + RUNTIME_PORT_FALLBACK_START, ResourceAllocationInput, ResourceAllocationRecord, + ResourceAllocationStatus, RuntimeObservedStateRecord, RuntimeObservedStatus, RuntimeSubject, + UnresolvedJobFailure, php_runtime_key, }; pub use error::{StateCapability, StateError}; pub use paths::{PathSummaryEntry, PvPaths}; diff --git a/crates/state/src/migrations.rs b/crates/state/src/migrations.rs index 350a411c..b5aedba1 100644 --- a/crates/state/src/migrations.rs +++ b/crates/state/src/migrations.rs @@ -17,6 +17,7 @@ const RESOURCE_PORT_ROLES_SQL: &str = include_str!("sql/007_resource_port_roles. const PROJECT_PHP_RUNTIME_EXTENSIONS_SQL: &str = include_str!("sql/008_project_php_runtime_extensions.sql"); const PROJECT_MODE_AND_SLUG_SQL: &str = include_str!("sql/009_project_mode_and_slug.sql"); +const JOB_DIAGNOSTIC_OUTCOMES_SQL: &str = include_str!("sql/010_job_diagnostic_outcomes.sql"); pub(crate) const DEFAULT_MIGRATIONS: &[Migration] = &[ Migration::new(1, "core_state_schema", CORE_SCHEMA_SQL), @@ -44,6 +45,7 @@ pub(crate) const DEFAULT_MIGRATIONS: &[Migration] = &[ PROJECT_PHP_RUNTIME_EXTENSIONS_SQL, ), Migration::new(9, "project_mode_and_slug", PROJECT_MODE_AND_SLUG_SQL), + Migration::new(10, "job_diagnostic_outcomes", JOB_DIAGNOSTIC_OUTCOMES_SQL), ]; #[derive(Copy, Clone, Debug, Eq, PartialEq)] diff --git a/crates/state/src/sql/010_job_diagnostic_outcomes.sql b/crates/state/src/sql/010_job_diagnostic_outcomes.sql new file mode 100644 index 00000000..fc021c61 --- /dev/null +++ b/crates/state/src/sql/010_job_diagnostic_outcomes.sql @@ -0,0 +1,11 @@ +CREATE TABLE job_diagnostic_outcomes ( + sequence INTEGER PRIMARY KEY, + job_id TEXT NOT NULL REFERENCES jobs(id) ON DELETE CASCADE, + subject_kind TEXT NOT NULL, + subject_id TEXT NOT NULL, + outcome TEXT NOT NULL CHECK (outcome IN ('failure', 'success')), + UNIQUE (job_id, subject_kind, subject_id, outcome) +); + +CREATE INDEX job_diagnostic_outcomes_subject +ON job_diagnostic_outcomes(subject_kind, subject_id, outcome); diff --git a/crates/state/tests/snapshots/state_foundation__database_runs_migrations_and_exposes_core_schema.snap b/crates/state/tests/snapshots/state_foundation__database_runs_migrations_and_exposes_core_schema.snap index 253a0740..9762ed17 100644 --- a/crates/state/tests/snapshots/state_foundation__database_runs_migrations_and_exposes_core_schema.snap +++ b/crates/state/tests/snapshots/state_foundation__database_runs_migrations_and_exposes_core_schema.snap @@ -16,9 +16,11 @@ DatabaseInspection { "007:resource_port_roles", "008:project_php_runtime_extensions", "009:project_mode_and_slug", + "010:job_diagnostic_outcomes", ], tables: [ "global_php_default_track", + "job_diagnostic_outcomes", "jobs", "managed_resource_tracks", "observed_states", diff --git a/crates/state/tests/state_foundation.rs b/crates/state/tests/state_foundation.rs index 7c921e37..6bb887dd 100644 --- a/crates/state/tests/state_foundation.rs +++ b/crates/state/tests/state_foundation.rs @@ -9,11 +9,12 @@ use rusqlite::{Connection, params}; use state::testing::Migration; use state::{ AppReleaseLayout, Database, EnvContextValues, GATEWAY_HTTP_PREFERRED_PORT, - GATEWAY_HTTPS_PREFERRED_PORT, GatewayPort, JobStatus, ManagedResourceDesiredState, - ManagedResourceTrackInstallInput, ManagedResourceTrackRemovalInput, PortOwner, PortRequest, - ProjectEnvObservedStatus, ProjectEnvObservedWarningInput, ProjectManagedResourceInput, - ProjectMode, ProjectRecord, PvPaths, RUNTIME_PORT_FALLBACK_END, RUNTIME_PORT_FALLBACK_START, - ResourceAllocationInput, RuntimeObservedStatus, RuntimeSubject, StateError, UpdateLock, + GATEWAY_HTTPS_PREFERRED_PORT, GatewayPort, JobDiagnosticSubject, JobStatus, + ManagedResourceDesiredState, ManagedResourceTrackInstallInput, + ManagedResourceTrackRemovalInput, PortOwner, PortRequest, ProjectEnvObservedStatus, + ProjectEnvObservedWarningInput, ProjectManagedResourceInput, ProjectMode, ProjectRecord, + PvPaths, RUNTIME_PORT_FALLBACK_END, RUNTIME_PORT_FALLBACK_START, ResourceAllocationInput, + RuntimeObservedStatus, RuntimeSubject, StateError, UpdateLock, }; #[test] @@ -3292,6 +3293,155 @@ fn job_records_expose_typed_statuses() -> Result<()> { Ok(()) } +#[test] +fn successful_coverage_resolves_failure_without_rewriting_history() -> Result<()> { + let tempdir = tempdir()?; + let paths = PvPaths::for_home(tempdir.path().join("home")); + let mut database = Database::open(&paths)?; + let first_failure = database.start_job("reconcile", "project:project_1")?; + database.fail_job(&first_failure.id, "Gateway failed")?; + let repair = database.start_job("reconcile", "project:project_1")?; + database.complete_job_with_coverage( + &repair.id, + "Project reconciled", + &[ + JobDiagnosticSubject::Project { + id: "project_1".to_owned(), + }, + JobDiagnosticSubject::GatewayRuntime, + ], + )?; + + assert!(database.unresolved_job_failures()?.is_empty()); + assert_eq!(database.recent_jobs()?.len(), 2); + assert!( + database + .recent_jobs()? + .iter() + .any(|job| job.id == first_failure.id && job.status == JobStatus::Failed) + ); + + let recurring_failure = database.start_job("reconcile", "project:project_1")?; + database.fail_job(&recurring_failure.id, "Gateway failed")?; + let unresolved = database.unresolved_job_failures()?; + + assert_eq!(unresolved.len(), 1); + assert_eq!(unresolved[0].job.id, recurring_failure.id); + assert_eq!( + unresolved[0].subject, + JobDiagnosticSubject::Project { + id: "project_1".to_owned() + } + ); + + Ok(()) +} + +#[test] +fn update_assessment_coverage_does_not_hide_gateway_failure() -> Result<()> { + let tempdir = tempdir()?; + let paths = PvPaths::for_home(tempdir.path().join("home")); + let mut database = Database::open(&paths)?; + let failure = database.start_job("reconcile", "resource:php:8.4")?; + database.fail_job(&failure.id, "Gateway failed")?; + let update = database.start_job("update", "system")?; + database.complete_job_with_coverage( + &update.id, + "current", + &[JobDiagnosticSubject::UpdateAssessment], + )?; + + let unresolved = database.unresolved_job_failures()?; + + assert_eq!(unresolved.len(), 1); + assert_eq!(unresolved[0].job.id, failure.id); + assert_eq!(unresolved[0].subject, JobDiagnosticSubject::GatewayRuntime); + + Ok(()) +} + +#[test] +fn supersession_uses_outcome_order_for_overlapping_jobs() -> Result<()> { + let tempdir = tempdir()?; + let paths = PvPaths::for_home(tempdir.path().join("home")); + let mut database = Database::open(&paths)?; + let earlier_started_failure = database.start_job("reconcile", "project:project_1")?; + let later_started_success = database.start_job("reconcile", "project:project_1")?; + database.complete_job_with_coverage( + &later_started_success.id, + "Project reconciled", + &[JobDiagnosticSubject::Project { + id: "project_1".to_owned(), + }], + )?; + database.fail_job(&earlier_started_failure.id, "late failure")?; + + let unresolved = database.unresolved_job_failures()?; + + assert_eq!(unresolved.len(), 1); + assert_eq!(unresolved[0].job.id, earlier_started_failure.id); + + Ok(()) +} + +#[test] +fn system_reconciliation_coverage_resolves_component_failures() -> Result<()> { + let tempdir = tempdir()?; + let paths = PvPaths::for_home(tempdir.path().join("home")); + let mut database = Database::open(&paths)?; + let failure = database.start_job("reconcile", "resource:mysql:8.4")?; + database.fail_job(&failure.id, "MySQL failed")?; + let repair = database.start_job("reconcile", "system")?; + database.complete_job_with_coverage( + &repair.id, + "System reconciled", + &[JobDiagnosticSubject::SystemReconciliation], + )?; + + assert!(database.unresolved_job_failures()?.is_empty()); + + Ok(()) +} + +#[test] +fn matching_healthy_observation_resolves_only_its_job_subject() -> Result<()> { + let tempdir = tempdir()?; + let paths = PvPaths::for_home(tempdir.path().join("home")); + let mut database = Database::open(&paths)?; + let gateway_failure = database.start_job("reconcile", "resource:php:8.4")?; + database.fail_job(&gateway_failure.id, "Gateway failed")?; + let system_failure = database.start_job("reconcile", "system")?; + database.fail_job(&system_failure.id, "System config failed")?; + state::testing::transaction(&mut database, |transaction| { + transaction.execute( + "UPDATE jobs SET finished_at = ?1 WHERE id IN (?2, ?3)", + params![ + "2026-01-01T00:00:00Z", + gateway_failure.id.as_str(), + system_failure.id.as_str() + ], + )?; + + Ok(()) + })?; + database.record_runtime_observed_snapshot( + RuntimeSubject::Gateway, + RuntimeObservedStatus::Running, + Some("Gateway ready"), + )?; + + let unresolved = database.unresolved_job_failures()?; + + assert_eq!(unresolved.len(), 1); + assert_eq!(unresolved[0].job.id, system_failure.id); + assert_eq!( + unresolved[0].subject, + JobDiagnosticSubject::SystemReconciliation + ); + + Ok(()) +} + #[test] fn completing_unknown_job_returns_typed_error() -> Result<()> { let tempdir = tempdir()?;