diff --git a/config/README.md b/config/README.md index d8aafd17a..d8ab47e4b 100644 --- a/config/README.md +++ b/config/README.md @@ -90,6 +90,7 @@ If you want to contribute your own configuration, please | log-level-dependencies | Logging level for dependencies. Possible values: "off", "error", "warn", "info", "debug", "trace". | "warn" | | RUSTIC_LOG_LEVEL_DEPENDENCIES | --log-level-dependencies | | log-file | Path to the log file. | No log file | "/log/rustic.log" | RUSTIC_LOG_FILE | --log-file | | no-progress | If true, disables progress indicators. | false | | RUSTIC_NO_PROGRESS | --no-progress | +| json-progress | If true, writes progress as newline-delimited JSON. | false | | RUSTIC_JSON_PROGRESS | --json-progress | | progress-interval | The interval at which progress indicators are shown. | "100ms" | "1m" | RUSTIC_PROGRESS_INTERVAL | --progress-interval | | group-by | Group snapshots by any combination of host,label,paths,tags e.g. for "latest" | "host,label,paths" | | RUSTIC_GROUP_BY | --group-by, -g | | check-index | If true, check the index and read pack headers if index information is missing. | false | | RUSTIC_CHECK_INDEX | --check-index | diff --git a/config/full.toml b/config/full.toml index bcf92969c..9ea35a74a 100644 --- a/config/full.toml +++ b/config/full.toml @@ -17,6 +17,7 @@ log-level-dryrun = "info" # any of "off", "error", "warn", "info", "debug", "tra log-level-dependencies = "warn" # any of "off", "error", "warn", "info", "debug", "trace"; default: "warn" log-file = "/path/to/rustic.log" # Default: not set no-progress = false +json-progress = false progress-interval = "100ms" group-by = "host,label,paths" check-index = false diff --git a/src/commands/backup.rs b/src/commands/backup.rs index 8b2e927dc..bde5f5ffa 100644 --- a/src/commands/backup.rs +++ b/src/commands/backup.rs @@ -27,7 +27,8 @@ use serde_with::serde_as; use rustic_core::{ BackupOptions, CommandInput, ConfigOptions, KeyOptions, LocalSourceFilterOptions, - LocalSourceSaveOptions, ParentOptions, PathList, SnapshotOptions, repofile::SnapshotFile, + LocalSourceSaveOptions, ParentOptions, PathList, SnapshotOptions, + repofile::{SnapshotFile, SnapshotId, SnapshotSummary}, }; /// `backup` subcommand @@ -377,7 +378,9 @@ impl BackupCmd { Ok(repo.backup(&backup_opts, &source, self.snap_opts.to_snapshot()?)?) })?; - if self.json { + if config.global.progress_options.json_progress { + write_json_progress_summary(&snap)?; + } else if self.json { let mut stdout = std::io::stdout(); serde_json::to_writer_pretty(&mut stdout, &snap)?; } else if self.long { @@ -431,6 +434,56 @@ impl BackupCmd { } } +#[derive(Serialize)] +struct JsonProgressSummary<'a> { + message_type: &'static str, + files_new: u64, + files_changed: u64, + files_unmodified: u64, + dirs_new: u64, + dirs_changed: u64, + dirs_unmodified: u64, + data_blobs: u64, + tree_blobs: u64, + data_added: u64, + data_added_packed: u64, + total_files_processed: u64, + total_bytes_processed: u64, + total_duration: f64, + #[serde(skip_serializing_if = "Option::is_none")] + snapshot_id: Option<&'a SnapshotId>, +} + +impl<'a> JsonProgressSummary<'a> { + fn new(summary: &'a SnapshotSummary, snapshot_id: &'a SnapshotId) -> Self { + Self { + message_type: "summary", + files_new: summary.files_new, + files_changed: summary.files_changed, + files_unmodified: summary.files_unmodified, + dirs_new: summary.dirs_new, + dirs_changed: summary.dirs_changed, + dirs_unmodified: summary.dirs_unmodified, + data_blobs: summary.data_blobs, + tree_blobs: summary.tree_blobs, + data_added: summary.data_added, + data_added_packed: summary.data_added_packed, + total_files_processed: summary.total_files_processed, + total_bytes_processed: summary.total_bytes_processed, + total_duration: summary.total_duration, + snapshot_id: (*snapshot_id != SnapshotId::default()).then_some(snapshot_id), + } + } +} + +fn write_json_progress_summary(snap: &SnapshotFile) -> Result<()> { + let summary = snap.summary.as_ref().unwrap(); + let mut stdout = std::io::stdout(); + serde_json::to_writer(&mut stdout, &JsonProgressSummary::new(summary, &snap.id))?; + println!(); + Ok(()) +} + #[cfg(not(any(feature = "prometheus", feature = "opentelemetry")))] fn publish_metrics( snap: &SnapshotFile, diff --git a/src/config/progress_options.rs b/src/config/progress_options.rs index 7c9f1d23f..03b427fc9 100644 --- a/src/config/progress_options.rs +++ b/src/config/progress_options.rs @@ -1,6 +1,6 @@ //! Progress Bar Config -use std::{fmt::Write, time::Duration}; +use std::{fmt::Write, io::Write as _, time::Duration}; use std::io::IsTerminal; use std::sync::{Arc, Mutex, OnceLock}; @@ -49,6 +49,16 @@ pub struct ProgressOptions { #[merge(strategy=conflate::bool::overwrite_false)] pub no_progress: bool, + /// Write progress as newline-delimited JSON + #[clap( + long, + global = true, + env = "RUSTIC_JSON_PROGRESS", + conflicts_with = "no_progress" + )] + #[merge(strategy=conflate::bool::overwrite_false)] + pub json_progress: bool, + /// Interval to update progress bars (default: 100ms) #[clap( long, @@ -89,6 +99,15 @@ impl ProgressOptions { return Progress::hidden(); } + let interval = self.log_interval(); + if self.json_progress { + return if interval > Duration::ZERO && matches!(kind, ProgressType::Bytes) { + Progress::new(JsonProgress::new(prefix, interval, kind)) + } else { + Progress::hidden() + }; + } + if std::io::stderr().is_terminal() { Progress::new(InteractiveProgress::new( prefix, @@ -96,7 +115,6 @@ impl ProgressOptions { self.interactive_interval(), )) } else { - let interval = self.log_interval(); if interval > Duration::ZERO { Progress::new(NonInteractiveProgress::new(prefix, interval, kind)) } else { @@ -200,6 +218,28 @@ struct NonInteractiveState { last_log: Instant, } +impl NonInteractiveState { + fn progress_text(&self, kind: ProgressType) -> String { + let format_value = |value| match kind { + ProgressType::Bytes => ByteSize(value).to_string(), + ProgressType::Counter | ProgressType::Spinner => value.to_string(), + }; + + self.length.map_or_else( + || format_value(self.position), + |len| format!("{} / {}", format_value(self.position), format_value(len)), + ) + } + + fn should_log(&self, interval: Duration) -> bool { + self.last_log.elapsed() >= interval + } + + fn mark_logged(&mut self) { + self.last_log = Instant::now(); + } +} + /// Periodic logger for non-interactive environments (i.e. systemd) /// Implemented thread-safe and decouples logging logic from indicatif #[derive(Clone, Debug)] @@ -234,17 +274,7 @@ impl NonInteractiveProgress { } fn log_progress(&self, state: &NonInteractiveState) { - let progress = state.length.map_or_else( - || self.format_value(state.position), - |len| { - format!( - "{} / {}", - self.format_value(state.position), - self.format_value(len) - ) - }, - ); - info!("{}: {}", state.prefix, progress); + info!("{}: {}", state.prefix, state.progress_text(self.kind)); } } @@ -269,9 +299,9 @@ impl RusticProgress for NonInteractiveProgress { if let Ok(mut state) = self.state.lock() { state.position += inc; - if state.last_log.elapsed() >= self.interval { + if state.should_log(self.interval) { self.log_progress(&state); - state.last_log = Instant::now(); + state.mark_logged(); } } } @@ -289,3 +319,112 @@ impl RusticProgress for NonInteractiveProgress { ); } } + +// ================ JSON ================ + +/// Periodic JSON lines progress for machine-readable consumers +#[derive(Clone, Debug)] +pub struct JsonProgress { + state: Arc>, + start: Instant, + interval: Duration, + kind: ProgressType, +} + +#[derive(Serialize)] +struct JsonProgressStatus { + message_type: &'static str, + seconds_elapsed: u64, + #[serde(skip_serializing_if = "Option::is_none")] + seconds_remaining: Option, + #[serde(skip_serializing_if = "Option::is_none")] + percent_done: Option, + #[serde(skip_serializing_if = "Option::is_none")] + total_bytes: Option, + #[serde(skip_serializing_if = "Option::is_none")] + bytes_done: Option, +} + +impl JsonProgress { + fn new(prefix: &str, interval: Duration, kind: ProgressType) -> Self { + let now = Instant::now(); + Self { + state: Arc::new(Mutex::new(NonInteractiveState { + prefix: prefix.to_string(), + position: 0, + length: None, + last_log: now, + })), + start: now, + interval, + kind, + } + } + + fn log_progress(&self, state: &NonInteractiveState) { + let is_bytes = matches!(self.kind, ProgressType::Bytes); + let elapsed = self.start.elapsed().as_secs(); + let percent_done = state + .length + .filter(|len| *len > 0) + .map(|len| (state.position as f64 / len as f64).min(1.0)); + let seconds_remaining = match (state.position, state.length) { + (position, Some(len)) if position > 0 && len > position => { + Some(elapsed.saturating_mul(len - position) / position) + } + _ => None, + }; + + let status = JsonProgressStatus { + message_type: "status", + seconds_elapsed: elapsed, + seconds_remaining, + percent_done, + total_bytes: is_bytes.then_some(state.length).flatten(), + bytes_done: is_bytes.then_some(state.position), + }; + + let mut stdout = std::io::stdout().lock(); + _ = serde_json::to_writer(&mut stdout, &status); + _ = writeln!(stdout); + } +} + +impl RusticProgress for JsonProgress { + fn is_hidden(&self) -> bool { + false + } + + fn set_length(&self, len: u64) { + if let Ok(mut state) = self.state.lock() { + state.length = Some(len); + self.log_progress(&state); + state.mark_logged(); + } + } + + fn set_title(&self, title: &str) { + if let Ok(mut state) = self.state.lock() { + state.prefix = title.to_string(); + } + } + + fn inc(&self, inc: u64) { + if let Ok(mut state) = self.state.lock() { + state.position += inc; + + if state.should_log(self.interval) { + self.log_progress(&state); + state.mark_logged(); + } + } + } + + fn finish(&self) { + let Ok(state) = self.state.lock() else { + return; + }; + + self.log_progress(&state); + } +} diff --git a/src/snapshots/rustic_rs__config__tests__default_config_display_passes.snap b/src/snapshots/rustic_rs__config__tests__default_config_display_passes.snap index af9a83102..a110300ab 100644 --- a/src/snapshots/rustic_rs__config__tests__default_config_display_passes.snap +++ b/src/snapshots/rustic_rs__config__tests__default_config_display_passes.snap @@ -9,6 +9,7 @@ dry-run = false dry-run-warmup = false check-index = false no-progress = false +json-progress = false show-time-offset = false [global.hooks] diff --git a/src/snapshots/rustic_rs__config__tests__default_config_passes.snap b/src/snapshots/rustic_rs__config__tests__default_config_passes.snap index 6edadab8d..5ce6b9bc6 100644 --- a/src/snapshots/rustic_rs__config__tests__default_config_passes.snap +++ b/src/snapshots/rustic_rs__config__tests__default_config_passes.snap @@ -19,6 +19,7 @@ RusticConfig { }, progress_options: ProgressOptions { no_progress: false, + json_progress: false, progress_interval: None, }, hooks: Hooks { diff --git a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-2.snap b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-2.snap index ab750cd3f..2f44f627f 100644 --- a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-2.snap +++ b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-2.snap @@ -9,6 +9,7 @@ dry-run = false dry-run-warmup = false check-index = false no-progress = false +json-progress = false show-time-offset = false [global.hooks] diff --git a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap index 9cb5b678f..a83536c47 100644 --- a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap +++ b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes-3.snap @@ -19,6 +19,7 @@ RusticConfig { }, progress_options: ProgressOptions { no_progress: false, + json_progress: false, progress_interval: None, }, hooks: Hooks { diff --git a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes.snap b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes.snap index 1a3d6bcb8..230704c10 100644 --- a/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes.snap +++ b/src/snapshots/rustic_rs__config__tests__global_env_roundtrip_passes.snap @@ -9,6 +9,7 @@ dry-run = false dry-run-warmup = false check-index = false no-progress = false +json-progress = false show-time-offset = false [global.hooks] diff --git a/tests/snapshots/show_config__show_config_passes.snap b/tests/snapshots/show_config__show_config_passes.snap index 22808bd00..5281284e8 100644 --- a/tests/snapshots/show_config__show_config_passes.snap +++ b/tests/snapshots/show_config__show_config_passes.snap @@ -9,6 +9,7 @@ dry-run = false dry-run-warmup = false check-index = false no-progress = false +json-progress = false show-time-offset = false [global.hooks]