diff --git a/src/sonic-ctrmgrd-rs/Cargo.lock b/src/sonic-ctrmgrd-rs/Cargo.lock index 3faf27b8d24..4543428e23f 100644 --- a/src/sonic-ctrmgrd-rs/Cargo.lock +++ b/src/sonic-ctrmgrd-rs/Cargo.lock @@ -421,6 +421,7 @@ dependencies = [ "mockall", "serial_test", "sonic-rs-common", + "swss-common", "syslog-tracing", "thiserror", "tokio", diff --git a/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/Cargo.toml b/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/Cargo.toml index 6a8d7dcb071..ae4d84b0dd7 100644 --- a/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/Cargo.toml +++ b/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/Cargo.toml @@ -23,6 +23,8 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } syslog-tracing = { workspace = true } sonic-rs-common = { path = "../../../sonic-rs-common" } +swss-common = { path = "../../../sonic-swss-common/crates/swss-common" } + [dev-dependencies] serial_test = "1.0.0" diff --git a/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/src/lib.rs b/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/src/lib.rs index ec205c4ece2..b0afdc66a90 100644 --- a/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/src/lib.rs +++ b/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/src/lib.rs @@ -19,6 +19,8 @@ pub enum Error { Join(#[from] tokio::task::JoinError), #[error("Device info error")] DeviceInfo(#[from] sonic_rs_common::device_info::DeviceInfoError), + #[error("SWSS common error")] + SwssError(#[from] swss_common::Exception), #[error("Syslog initialization error: {0}")] Syslog(String), } @@ -86,7 +88,8 @@ pub async fn wait_for_container( info!("No longer waiting on container '{}'", container_name); if dependent_services.contains(&container_name) { - let warm_restart = device_info::is_warm_restart_enabled(&container_name)?; + let (service_name, namespace) = parse_container_name(&container_name); + let warm_restart = device_info::is_warm_restart_enabled_in_namespace(service_name, &namespace)?; let fast_reboot = device_info::is_fast_reboot_enabled()?; if warm_restart || fast_reboot { @@ -99,6 +102,18 @@ pub async fn wait_for_container( } } +fn parse_container_name(container_name: &str) -> (&str, String) { + match container_name.find(|c: char| c.is_digit(10)) { + Some(index) => { + let (service_name, namespace) = container_name.split_at(index); + (service_name, format!("asic{}", namespace)) + } + None => { + (container_name, "".to_string()) + } + } +} + pub async fn run_main( docker_client: Arc, service: Option>, diff --git a/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/src/main.rs b/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/src/main.rs index ae16dea9b00..25e406e0266 100644 --- a/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/src/main.rs +++ b/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/src/main.rs @@ -33,6 +33,10 @@ struct Cli { #[tokio::main] async fn main() -> Result<(), Error> { + const DEFAULT_DATABASE_GLOBAL_CONFIG_PATH: &'static str = "/var/run/redis/sonic-db/database_global.json"; + if std::path::Path::new(DEFAULT_DATABASE_GLOBAL_CONFIG_PATH).exists() { + swss_common::sonic_db_config_initialize_global(DEFAULT_DATABASE_GLOBAL_CONFIG_PATH, true)?; + } let identity = CString::new("docker-wait-any-rs") .map_err(|e| Error::Syslog(format!("invalid identity string: {}", e)))?; let syslog = syslog_tracing::Syslog::new( diff --git a/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/tests/docker_wait_any_test.rs b/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/tests/docker_wait_any_test.rs index cf955cfdbec..4a3919d4a59 100644 --- a/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/tests/docker_wait_any_test.rs +++ b/src/sonic-ctrmgrd-rs/crates/docker-wait-any-rs/tests/docker_wait_any_test.rs @@ -151,9 +151,71 @@ async fn test_teamd_exits_warm_restart_main_will_not_exit() { // Mock warm restart enabled using injectorpp - keep injector alive for entire test let mut injector = InjectorPP::new(); injector - .when_called(func!(sonic_rs_common::device_info::is_warm_restart_enabled, fn(&str) -> Result)) + .when_called(func!(sonic_rs_common::device_info::is_warm_restart_enabled_in_namespace, fn(&str, &str) -> Result)) .will_execute(fake!( - func_type: fn(container_name: &str) -> Result, + func_type: fn(container_name: &str, namespace: &str) -> Result, + returns: Ok(true) + )); + injector + .when_called(func!(sonic_rs_common::device_info::is_fast_reboot_enabled, fn() -> Result)) + .will_execute(fake!( + func_type: fn() -> Result, + returns: Ok(false) + )); + + // Test run_main - should timeout because warm restart is enabled and main will not exit + let result = timeout( + Duration::from_secs(3), + run_main(Arc::new(mock_docker), Some(services), Some(dependents)) + ) + .await; + + // Should timeout because warm restart is enabled and dependent service teamd exits but main continues + assert!(result.is_err(), "run_main should timeout when warm restart is enabled and teamd exits"); +} + +#[tokio::test] +#[serial] +async fn test_teamd_exits_warm_restart_main_will_not_exit_mutli_asic() { + + let services = vec!["swss1".to_string()]; + let dependents = vec!["syncd1".to_string(), "teamd1".to_string()]; + + let mut mock_docker = MockDockerApi::new(); + + // teamd exits (simulates container exit, warm restart will cause it to be called again) + mock_docker + .expect_wait_container() + .with(eq("teamd1".to_string()), always()) + .returning(|_, _| { + Box::pin(futures_util::stream::iter(vec![Ok( + bollard::models::ContainerWaitResponse { + status_code: 0, + error: None, + } + )]).then(|item| async move { + // Add context switch to allow timeout to work + tokio::task::yield_now().await; + item + })) + }); + + mock_docker + .expect_wait_container() + .with(eq("swss1".to_string()), always()) + .returning(|_, _| Box::pin(stream::pending())); + + mock_docker + .expect_wait_container() + .with(eq("syncd1".to_string()), always()) + .returning(|_, _| Box::pin(stream::pending())); + + // Mock warm restart enabled using injectorpp - keep injector alive for entire test + let mut injector = InjectorPP::new(); + injector + .when_called(func!(sonic_rs_common::device_info::is_warm_restart_enabled_in_namespace, fn(&str, &str) -> Result)) + .will_execute(fake!( + func_type: fn(container_name: &str, namespace: &str) -> Result, returns: Ok(true) )); injector diff --git a/src/sonic-rs-common/src/device_info.rs b/src/sonic-rs-common/src/device_info.rs index 6af83383c63..d858c6707fc 100644 --- a/src/sonic-rs-common/src/device_info.rs +++ b/src/sonic-rs-common/src/device_info.rs @@ -24,6 +24,11 @@ impl StateDBConnector { let db = DbConnector::new_named("STATE_DB", true, 0)?; Ok(StateDBConnector { db }) } + + pub fn new_with_namespace(namespace: &str) -> std::result::Result { + let db = DbConnector::new_keyed("STATE_DB", false, 0, "", namespace)?; + Ok(StateDBConnector { db }) + } } impl StateDBTrait for StateDBConnector { @@ -69,6 +74,12 @@ pub fn is_warm_restart_enabled(container_name: &str) -> Result { is_warm_restart_enabled_with_db(container_name, &state_db) } +pub fn is_warm_restart_enabled_in_namespace(container_name: &str, namespace: &str) -> Result { + let state_db = StateDBConnector::new_with_namespace(namespace) + .map_err(|e| DeviceInfoError::SwSS(e))?; + is_warm_restart_enabled_with_db(container_name, &state_db) +} + pub fn is_fast_reboot_enabled() -> Result { let state_db = StateDBConnector::new() .map_err(|e| DeviceInfoError::SwSS(e))?; diff --git a/src/sonic-rs-common/tests/device_info_test.rs b/src/sonic-rs-common/tests/device_info_test.rs index 43dcafebbd1..00b32db5144 100644 --- a/src/sonic-rs-common/tests/device_info_test.rs +++ b/src/sonic-rs-common/tests/device_info_test.rs @@ -161,6 +161,21 @@ mod test_device_info { } } + // Integration test for public API (will use real database connection, may fail in test env) + #[test] + fn test_is_warm_restart_enabled_in_namespace_public_api() { + let result = is_warm_restart_enabled_in_namespace("swss", "asic0"); + // Test passes regardless of result since database may not be available in test environment + match result { + Ok(_) => { + // Function succeeded - database was available + } + Err(_) => { + // Function failed - expected in test environment without database + } + } + } + // Integration test for public API (will use real database connection, may fail in test env) #[test] fn test_is_fast_reboot_enabled_public_api() {