diff --git a/Cargo.lock b/Cargo.lock index 057fde39..211cd8e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -407,6 +407,21 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + [[package]] name = "crypto-common" version = "0.1.6" @@ -543,6 +558,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" +[[package]] +name = "endian-type" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" + [[package]] name = "equivalent" version = "1.0.2" @@ -1302,6 +1323,36 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38d1115007560874e373613744c6fba374c17688327a71c1476d1a5954cc857b" +[[package]] +name = "metrics" +version = "0.24.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89550ee9f79e88fef3119de263694973a8adb26c21d75322164fb8c493039fe2" +dependencies = [ + "portable-atomic", + "rapidhash", +] + +[[package]] +name = "metrics-util" +version = "0.19.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8496cc523d1f94c1385dd8f0f0c2c480b2b8aeccb5b7e4485ad6365523ae376" +dependencies = [ + "aho-corasick", + "crossbeam-epoch", + "crossbeam-utils", + "hashbrown 0.15.4", + "indexmap", + "metrics", + "ordered-float", + "quanta", + "radix_trie", + "rand", + "rand_xoshiro", + "sketches-ddsketch", +] + [[package]] name = "miette" version = "7.6.0" @@ -1419,6 +1470,8 @@ dependencies = [ "itoa", "lru", "md-5", + "metrics", + "metrics-util", "miette", "minijinja", "mockable", @@ -1464,6 +1517,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8b7b69b0eafaa88ec8dc9fe7c3860af0a147517e5207cfbd0ecd21cd7cde18" +[[package]] +name = "nibble_vec" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77a5d83df9f36fe23f0c3648c6bbb8b0298bb5f1939c8f2704431371f4b84d43" +dependencies = [ + "smallvec 1.15.1", +] + [[package]] name = "ninja_env" version = "0.1.0" @@ -1532,6 +1594,15 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "ordered-float" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" +dependencies = [ + "num-traits", +] + [[package]] name = "ortho_config" version = "0.8.0" @@ -1828,6 +1899,21 @@ dependencies = [ "unarray", ] +[[package]] +name = "quanta" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7" +dependencies = [ + "crossbeam-utils", + "libc", + "once_cell", + "raw-cpuid", + "wasi 0.11.1+wasi-snapshot-preview1", + "web-sys", + "winapi", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -1849,6 +1935,16 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "radix_trie" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c069c179fcdc6a2fe24d8d18305cf085fdbd4f922c041943e203685d6a1c58fd" +dependencies = [ + "endian-type", + "nibble_vec", +] + [[package]] name = "rand" version = "0.9.4" @@ -1887,6 +1983,33 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rand_xoshiro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41" +dependencies = [ + "rand_core", +] + +[[package]] +name = "rapidhash" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5e48930979c155e2f33aa36ab3119b5ee81332beb6482199a8ecd6029b80b59" +dependencies = [ + "rustversion", +] + +[[package]] +name = "raw-cpuid" +version = "11.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "498cd0dc59d73224351ee52a95fee0f1a617a2eae0e7d9d720cc622c73a54186" +dependencies = [ + "bitflags", +] + [[package]] name = "redox_syscall" version = "0.5.17" @@ -2447,6 +2570,12 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "sketches-ddsketch" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c6f73aeb92d671e0cc4dca167e59b2deb6387c375391bc99ee743f326994a2b" + [[package]] name = "slab" version = "0.4.10" @@ -3173,6 +3302,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "web-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "web-time" version = "1.1.0" diff --git a/Cargo.toml b/Cargo.toml index 13dbe4be..2881d326 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,6 +57,7 @@ wait-timeout = "0.2" url = "^2.5.0" ortho_config = { version = "0.8.0", features = ["serde_json"] } sys-locale = "0.3.2" +metrics = "0.24" [build-dependencies] clap = { version = "4.5.0", features = ["derive"] } @@ -161,6 +162,7 @@ strip-ansi-escapes = "0.2" toml = "0.8" serde_yaml = "0.9" proptest = "1.11.0" +metrics-util = { version = "0.19", features = ["debugging"] } # Target-specific dev-deps [target.'cfg(unix)'.dev-dependencies] diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 858e5efb..067891f7 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -626,6 +626,38 @@ Versioning and compatibility rules: present because the same layer object also participates in full config merging. +### Configuration-load observability + +Startup configuration loading is instrumented through the +[`metrics`](https://docs.rs/metrics) façade so operators can detect failure +trends and startup-latency regressions in production. The instrumentation +lives in `src/main.rs` around `resolve_configuration`, which spans +diagnostic-mode resolution (`cli::resolve_merged_diag_json`) through the full +layer merge (`cli::merge_with_config`). + +Because `metrics` is a façade, the macros are no-ops unless the operator +installs a recorder (for example a Prometheus or OpenTelemetry exporter) at +process start. Netsuke does not bundle a recorder; it only emits the +measurements. + +Instruments emitted by `record_config_load_metrics`: + +- `netsuke_config_load_total` — a counter incremented once per startup + configuration-load attempt. It carries a single label `outcome` with values + `success` or `failure`, where `failure` corresponds to a `merge_with_config` + error. Use it to compute the configuration-load failure rate. +- `netsuke_config_load_duration_seconds` — a histogram recording the + wall-clock duration of the configuration-load phase in seconds (one sample + per startup). Suggested operator bucket boundaries: `0.001, 0.005, 0.01, + 0.05, 0.1, 0.5, 1.0` seconds; configuration loading is expected to complete + in single-digit milliseconds, so buckets above one second exist only to + catch pathological filesystem or environment stalls. + +Naming convention: metric names use the `netsuke_` prefix and a `snake_case` +unit suffix (`_total` for counters, `_seconds` for duration histograms), +matching Prometheus conventions. Label values are bounded constant strings +(`success`/`failure`) to keep cardinality fixed. + ## BDD command helpers and environment handling The BDD step module `tests/bdd/steps/manifest_command.rs` provides three diff --git a/src/main.rs b/src/main.rs index 3da50cc6..fcea27de 100644 --- a/src/main.rs +++ b/src/main.rs @@ -10,14 +10,25 @@ use netsuke::{ cli, cli_localization, diagnostic_json, locale_resolution, localization, manifest, output_mode, output_prefs, runner, }; -use ortho_config::Localizer; +use ortho_config::{Localizer, OrthoError}; use std::ffi::OsString; use std::io::{self, Write}; use std::process::ExitCode; use std::sync::Arc; +use std::time::{Duration, Instant}; use tracing::Level; use tracing_subscriber::fmt; +/// Counter recording the outcome of each configuration-load attempt. +/// +/// Labelled by `outcome` (`success` or `failure`) so operators can track the +/// startup configuration-load failure rate. +const CONFIG_LOAD_TOTAL: &str = "netsuke_config_load_total"; + +/// Histogram recording the wall-clock duration of the configuration-load +/// phase (diagnostic-mode resolution through layer merge) in seconds. +const CONFIG_LOAD_DURATION_SECONDS: &str = "netsuke_config_load_duration_seconds"; + #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum DiagMode { Human, @@ -53,11 +64,10 @@ fn run_with_args( Ok(parsed) => parsed, Err(code) => return code, }; - let mode = DiagMode::from_json_enabled(cli::resolve_merged_diag_json(&parsed_cli, &matches)); - - let merged_cli = match merge_cli_or_exit(&parsed_cli, &matches, mode) { - Ok(merged) => merged, - Err(code) => return code, + let (mode, merge_result) = resolve_configuration(&parsed_cli, &matches); + let merged_cli = match merge_result { + Ok(merged) => merged.with_default_command(), + Err(err) => return handle_config_load_error(&err, mode), }; let runtime_mode = DiagMode::from_json_enabled(merged_cli.diag_json); configure_runtime(&merged_cli, system_locale, runtime_mode); @@ -115,24 +125,42 @@ fn parse_cli_or_exit( } } -fn merge_cli_or_exit( +/// Resolve the diagnostic mode and merge configuration layers, recording +/// startup-latency and outcome metrics for the combined config-load phase. +/// +/// The returned [`DiagMode`] is needed to render any merge failure, so it is +/// produced even when the merge fails. The histogram spans diagnostic-mode +/// resolution through the layer merge, and the counter is labelled by the +/// merge outcome. +fn resolve_configuration( parsed_cli: &cli::Cli, matches: &ArgMatches, - mode: DiagMode, -) -> Result { - match cli::merge_with_config(parsed_cli, matches) { - Ok(merged) => Ok(merged.with_default_command()), - Err(err) => { - if mode.is_json() { - Err(diagnostic_json::emit_or_fallback( - diagnostic_json::render_error_json(err.as_ref()), - )) - } else { - init_tracing(Level::ERROR); - tracing::error!(error = %err, "configuration load failed"); - Err(ExitCode::FAILURE) - } - } +) -> (DiagMode, Result>) { + let start = Instant::now(); + let mode = DiagMode::from_json_enabled(cli::resolve_merged_diag_json(parsed_cli, matches)); + let merged = cli::merge_with_config(parsed_cli, matches); + record_config_load_metrics(start.elapsed(), merged.is_ok()); + (mode, merged) +} + +/// Emit the configuration-load metrics for one startup attempt. +/// +/// Recording goes through the `metrics` façade, so it is a no-op unless the +/// operator has installed a recorder. +fn record_config_load_metrics(elapsed: Duration, succeeded: bool) { + let outcome = if succeeded { "success" } else { "failure" }; + metrics::histogram!(CONFIG_LOAD_DURATION_SECONDS).record(elapsed.as_secs_f64()); + metrics::counter!(CONFIG_LOAD_TOTAL, "outcome" => outcome).increment(1); +} + +/// Render a configuration-load failure to the user and map it to an exit code. +fn handle_config_load_error(err: &Arc, mode: DiagMode) -> ExitCode { + if mode.is_json() { + diagnostic_json::emit_or_fallback(diagnostic_json::render_error_json(err.as_ref())) + } else { + init_tracing(Level::ERROR); + tracing::error!(error = %err, "configuration load failed"); + ExitCode::FAILURE } } @@ -192,3 +220,79 @@ fn render_runtime_error_json(err: &anyhow::Error) -> serde_json::Result } diagnostic_json::render_error_json(err.as_ref()) } + +#[cfg(test)] +mod tests { + use super::*; + use metrics::Label; + use metrics_util::debugging::{DebugValue, DebuggingRecorder}; + + /// Capture the metrics emitted while `body` runs against a local recorder. + fn captured_metrics(body: impl FnOnce()) -> Vec<(String, Vec