From 427a48ca7a837a1d85c831e6cb535341a8f80ac1 Mon Sep 17 00:00:00 2001 From: Chet Nichols III Date: Tue, 9 Jun 2026 16:41:09 -0700 Subject: [PATCH] feat: preserve the boot interface pair across interface deletion A machine's boot target is the (MAC, EthernetInterface.Id) pair on its primary interface row -- also known as a `MachineBootInterface` -- and the ID half can be impossible to re-derive from a MAC address later; after a DPU flips to NIC mode, the BMC can report the interface ID without its MAC. So anything that deletes the `machine_interfaces` row (e.g. `force-delete --delete-interfaces`) can leave the re-ingested machine with no boot target until it happens to PXE on a leftover UEFI boot entry (if one exists). Soooo... this PR introduces a small `retained_boot_interfaces` table (keyed by MAC) to stash `MachineBootInterface` pairings in between ingestion(s). `machine_interface::delete` records every deleted pair there -- `force-delete`, `interface-delete`, any deletion path -- and the next `machine_interfaces` row for that MAC picks it back up at creation -- recovery lives in `create_with_type`, the one place every new row passes through -- however it's created: plain DHCP, a static `fixed_ip` preallocation, the proactive interface a DPU-mode ingest creates, or predicted-interface promotion (predicted rows gain a `boot_interface_id` column holding the live report's id -- refreshed every exploration -- for a host with unmanaged/no DPUs explored before its first DHCP). And to be clear, this applies to ANY interface -- a DPU in DPU mode, a DPU in NIC mode, an integrated NIC, etc. One more thing worth calling out (if it wasn't obvious): rows in `retained_boot_interfaces` are temporary. As soon as the `MachineBootInterface` is dropped into a `machine_interfaces` row, the record is removed (via `take_by_mac`). How long a record *stays* retained is configurable via a new top-level `retained_boot_interface_window` config; the default is forever -- if the machine eventually comes back, the pair will be waiting. The window is checked when the row is created (i.e. at DHCP time); retained IDs are deliberately never copied into predictions, so there's no copy sitting around that could dodge the window. Setting a window means a MAC reappearing on different hardware months later won't inherit an obsolete Redfish interface ID (e.g. the old `NIC.Slot.X` resource may not even exist there); a too-old record just gets swept on the spot, and the new row starts without a boot ID until exploration fills it in from a live report. Tests added to check `force-delete` boot interface retention, predicted-interface hand-off at promotion (and record removal), a DHCP-derived row recovering a retained ID (and removing the record), a statically preallocated row doing the same, and a pending prediction picking up the ID once a later report resolves it. The new `retained_boot_interface_window` config is documented in `cfg/README.md`. Existing zero-DPU, backfill, and force-delete tests still pass. Signed-off-by: Chet Nichols III --- crates/api-core/src/cfg/README.md | 1 + crates/api-core/src/cfg/file.rs | 32 +- crates/api-core/src/dhcp/discover.rs | 17 +- .../api-core/src/handlers/expected_machine.rs | 53 +- .../src/handlers/expected_power_shelf.rs | 9 +- .../api-core/src/handlers/expected_switch.rs | 16 +- crates/api-core/src/handlers/machine.rs | 3 + .../src/handlers/machine_discovery.rs | 1 + .../src/handlers/machine_interface_address.rs | 2 + crates/api-core/src/setup.rs | 20 +- .../src/test_support/default_config.rs | 1 + .../src/tests/common/api_fixtures/mod.rs | 28 + .../common/api_fixtures/site_explorer.rs | 2 + .../src/tests/dhcp_lease_expiration.rs | 5 + crates/api-core/src/tests/expected_machine.rs | 24 +- .../src/tests/expected_power_shelf.rs | 5 + crates/api-core/src/tests/expected_switch.rs | 1 + crates/api-core/src/tests/finder.rs | 2 +- crates/api-core/src/tests/ip_allocator.rs | 8 + crates/api-core/src/tests/ipxe.rs | 2 +- .../src/tests/machine_admin_force_delete.rs | 62 +++ crates/api-core/src/tests/machine_dhcp.rs | 4 + .../api-core/src/tests/machine_discovery.rs | 2 + .../src/tests/machine_interface_addresses.rs | 1 + .../api-core/src/tests/machine_interfaces.rs | 19 + crates/api-core/src/tests/machine_network.rs | 1 + crates/api-core/src/tests/machine_topology.rs | 1 + crates/api-core/src/tests/network_segment.rs | 1 + .../tests/prevent_duplicate_mac_addresses.rs | 2 + crates/api-core/src/tests/site_explorer.rs | 510 ++++++++++++++++++ .../src/tests/static_address_management.rs | 12 + crates/api-core/src/tests/tpm_ca.rs | 3 + ...0260609232431_retained_boot_interfaces.sql | 21 + crates/api-db/src/lib.rs | 1 + crates/api-db/src/machine_interface.rs | 199 +++++-- .../api-db/src/predicted_machine_interface.rs | 23 +- crates/api-db/src/retained_boot_interface.rs | 125 +++++ .../src/predicted_machine_interface.rs | 5 + crates/site-explorer/src/config.rs | 8 + crates/site-explorer/src/lib.rs | 54 +- crates/site-explorer/src/machine_creator.rs | 26 + crates/site-explorer/tests/reconcile.rs | 5 + crates/site-explorer/tests/site_explorer.rs | 13 + crates/utils/src/config.rs | 10 + 44 files changed, 1264 insertions(+), 76 deletions(-) create mode 100644 crates/api-db/migrations/20260609232431_retained_boot_interfaces.sql create mode 100644 crates/api-db/src/retained_boot_interface.rs diff --git a/crates/api-core/src/cfg/README.md b/crates/api-core/src/cfg/README.md index f2bb5e1e8f..db79de2df6 100644 --- a/crates/api-core/src/cfg/README.md +++ b/crates/api-core/src/cfg/README.md @@ -42,6 +42,7 @@ applicable. | `initial_dpu_agent_upgrade_policy` | `Option` | — | Policy for nico-dpu-agent upgrades. Also settable via `nico-admin-cli`. | | `max_concurrent_machine_updates` | `Option` | — | **Deprecated.** Use `machine_updater` instead. | | `machine_update_run_interval` | `Option` | — | Interval (seconds) at which the machine update manager checks for updates. | +| `retained_boot_interface_window` | `Option` | — (forever) | How long a retained boot interface pair (`retained_boot_interfaces` table) stays applicable after its `machine_interfaces` row was deleted. Unset retains forever; set a window (e.g. `30d`) so a MAC reappearing on different hardware doesn't inherit an obsolete Redfish interface id. | | `site_explorer` | `SiteExplorerConfig` | *(see below)* | SiteExplorer hardware discovery settings (see [SiteExplorerConfig](#siteexplorerconfig)). | | `nvue_enabled` | `bool` | `true` | DPU agent uses NVUE for config instead of writing files directly. | | `vpc_peering_policy` | `Option` | — | Policy for VPC peering based on network virtualization type at creation time. | diff --git a/crates/api-core/src/cfg/file.rs b/crates/api-core/src/cfg/file.rs index 11e3c07beb..790bccb274 100644 --- a/crates/api-core/src/cfg/file.rs +++ b/crates/api-core/src/cfg/file.rs @@ -38,7 +38,7 @@ use carbide_preingestion_manager::PreingestionManagerConfig; use carbide_rack_controller::config::{RackValidationConfig, RmsConfig}; use carbide_site_explorer::config::SiteExplorerConfig; use carbide_state_controller_common::config::StateControllerConfig; -use carbide_utils::config::{as_duration, as_std_duration}; +use carbide_utils::config::{as_duration, as_option_duration, as_std_duration}; use chrono::Duration; use db::host_naming::HostNamingStrategyKind; use duration_str::{deserialize_duration, deserialize_duration_chrono}; @@ -64,6 +64,20 @@ use serde::{Deserialize, Deserializer, Serialize}; pub(crate) const DEFAULT_DPU_NUM_OF_VFS: u32 = 16; pub(crate) const MAX_DPU_NUM_OF_VFS: u32 = 126; +/// Parses an optional duration ("30d", "12h", ...; absent = `None`) into +/// `Option`. Hand-rolled because `duration_str` deprecated +/// its own Option variant -- we do NOT use the deprecated function. +fn deserialize_option_duration_chrono<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + Option::::deserialize(deserializer)? + .map(|value| duration_str::parse_chrono(&value).map_err(serde::de::Error::custom)) + .transpose() +} + /// nico-api configuration file content #[derive(Clone, Debug, Deserialize, Serialize)] pub struct CarbideConfig { @@ -246,6 +260,19 @@ pub struct CarbideConfig { /// The interval at which the machine update manager checks for machine updates in seconds. pub machine_update_run_interval: Option, + /// How long a retained boot interface pair (see the + /// `retained_boot_interfaces` table) stays applicable after its + /// `machine_interfaces` row was deleted. The default (`None`) retains + /// forever: if the machine eventually comes back, the pair is waiting. + /// Set a window (e.g. "30d") to keep a MAC that reappears on different + /// hardware from inheriting an obsolete Redfish interface id. + #[serde( + default, + deserialize_with = "deserialize_option_duration_chrono", + serialize_with = "as_option_duration" + )] + pub retained_boot_interface_window: Option, + /// SiteExplorer related configuration #[serde(default)] pub site_explorer: SiteExplorerConfig, @@ -2690,6 +2717,7 @@ mod tests { assert_eq!( config.site_explorer, SiteExplorerConfig { + retained_boot_interface_window: None, enabled: Arc::new(false.into()), run_interval: std::time::Duration::from_secs(120), concurrent_explorations: 10, @@ -2881,6 +2909,7 @@ mod tests { assert_eq!( config.site_explorer, SiteExplorerConfig { + retained_boot_interface_window: None, enabled: Arc::new(true.into()), run_interval: std::time::Duration::from_secs(100), concurrent_explorations: 30, @@ -3207,6 +3236,7 @@ mod tests { assert_eq!( config.site_explorer, SiteExplorerConfig { + retained_boot_interface_window: None, enabled: Arc::new(false.into()), run_interval: std::time::Duration::from_secs(100), concurrent_explorations: 10, diff --git a/crates/api-core/src/dhcp/discover.rs b/crates/api-core/src/dhcp/discover.rs index 614f475409..179a77ae56 100644 --- a/crates/api-core/src/dhcp/discover.rs +++ b/crates/api-core/src/dhcp/discover.rs @@ -208,6 +208,7 @@ pub async fn discover_dhcp( &mut txn, &expected_interface, relay_ip, + api.runtime_config.retained_boot_interface_window, ) .await?; Some(expected_interface.machine_id) @@ -269,7 +270,10 @@ pub async fn discover_dhcp( // machine interface (and machine interface address) for it, // creating one if needed. db::machine_interface::preallocate_machine_interface( - &mut txn, parsed_mac, fixed_ip, + &mut txn, + parsed_mac, + fixed_ip, + api.runtime_config.retained_boot_interface_window, ) .await?; } @@ -286,7 +290,10 @@ pub async fn discover_dhcp( // InterfaceType::Bmc (and primary=false). Races against // site-explorer's reconciliation pass are handled inside preallocate. db::machine_interface::preallocate_bmc_machine_interface( - &mut txn, parsed_mac, bmc_ip, + &mut txn, + parsed_mac, + bmc_ip, + api.runtime_config.retained_boot_interface_window, ) .await?; } else if let Some(s) = @@ -303,7 +310,10 @@ pub async fn discover_dhcp( // Races against site-explorer's reconciliation pass are handled // inside preallocate. db::machine_interface::preallocate_machine_interface( - &mut txn, parsed_mac, nvos_ip, + &mut txn, + parsed_mac, + nvos_ip, + api.runtime_config.retained_boot_interface_window, ) .await?; } @@ -319,6 +329,7 @@ pub async fn discover_dhcp( std::slice::from_ref(&parsed_relay), host_nic, is_primary_nic, + api.runtime_config.retained_boot_interface_window, ) .await?; diff --git a/crates/api-core/src/handlers/expected_machine.rs b/crates/api-core/src/handlers/expected_machine.rs index b327bbe6b1..1e4225b007 100644 --- a/crates/api-core/src/handlers/expected_machine.rs +++ b/crates/api-core/src/handlers/expected_machine.rs @@ -224,13 +224,25 @@ pub(crate) async fn update( // Update BMC interface if bmc_ip_address is set. if let Some(bmc_ip) = machine.data.bmc_ip_address { - update_preallocated_machine_interface(&mut txn, machine.bmc_mac_address, bmc_ip).await?; + update_preallocated_machine_interface( + &mut txn, + machine.bmc_mac_address, + bmc_ip, + api.runtime_config.retained_boot_interface_window, + ) + .await?; } // Update/create machine interfaces for host NICs with fixed IPs. for nic in &machine.data.host_nics { if let Some(ip) = nic.fixed_ip { - update_preallocated_machine_interface(&mut txn, nic.mac_address, ip).await?; + update_preallocated_machine_interface( + &mut txn, + nic.mac_address, + ip, + api.runtime_config.retained_boot_interface_window, + ) + .await?; } } @@ -426,6 +438,7 @@ async fn update_expected_machine( machine: rpc::ExpectedMachine, id: Uuid, parsed_mac: MacAddress, + retained_window: Option, ) -> Result<(), CarbideError> { let data: ExpectedMachineData = machine.try_into()?; @@ -436,8 +449,13 @@ async fn update_expected_machine( }; if let Some(bmc_ip) = expected_machine.data.bmc_ip_address { - update_preallocated_machine_interface(txn, expected_machine.bmc_mac_address, bmc_ip) - .await?; + update_preallocated_machine_interface( + txn, + expected_machine.bmc_mac_address, + bmc_ip, + retained_window, + ) + .await?; } db::expected_machine::update(txn, &expected_machine).await?; @@ -491,10 +509,13 @@ async fn apply_operation( machine: rpc::ExpectedMachine, id: Uuid, parsed_mac: MacAddress, + retained_window: Option, ) -> Result<(), CarbideError> { match op { BatchOperation::Create => create_expected_machine(txn, machine, id, parsed_mac).await, - BatchOperation::Update => update_expected_machine(txn, machine, id, parsed_mac).await, + BatchOperation::Update => { + update_expected_machine(txn, machine, id, parsed_mac, retained_window).await + } } } @@ -542,7 +563,16 @@ async fn process_batch_operations( } }; - match apply_operation(op, txn.as_pgconn(), machine, id, parsed_mac).await { + match apply_operation( + op, + txn.as_pgconn(), + machine, + id, + parsed_mac, + api.runtime_config.retained_boot_interface_window, + ) + .await + { Ok(_) => match txn.commit().await { Ok(_) => results.push(build_success_result(machine_for_result)), Err(e) => { @@ -574,7 +604,16 @@ async fn process_batch_operations( value: id.to_string(), }); - if let Err(e) = apply_operation(op, txn.as_pgconn(), machine, id, parsed_mac).await { + if let Err(e) = apply_operation( + op, + txn.as_pgconn(), + machine, + id, + parsed_mac, + api.runtime_config.retained_boot_interface_window, + ) + .await + { let _ = txn.rollback().await; return Err(e); } diff --git a/crates/api-core/src/handlers/expected_power_shelf.rs b/crates/api-core/src/handlers/expected_power_shelf.rs index 624a8e28af..c133b20f19 100644 --- a/crates/api-core/src/handlers/expected_power_shelf.rs +++ b/crates/api-core/src/handlers/expected_power_shelf.rs @@ -108,8 +108,13 @@ pub async fn update_expected_power_shelf( })?; if let Some(bmc_ip) = power_shelf.bmc_ip_address { - update_preallocated_machine_interface(&mut txn, power_shelf.bmc_mac_address, bmc_ip) - .await?; + update_preallocated_machine_interface( + &mut txn, + power_shelf.bmc_mac_address, + bmc_ip, + api.runtime_config.retained_boot_interface_window, + ) + .await?; } db_expected_power_shelf::update(&mut txn, &power_shelf) diff --git a/crates/api-core/src/handlers/expected_switch.rs b/crates/api-core/src/handlers/expected_switch.rs index c2c4119e20..46b1318f2c 100644 --- a/crates/api-core/src/handlers/expected_switch.rs +++ b/crates/api-core/src/handlers/expected_switch.rs @@ -126,12 +126,24 @@ pub async fn update_expected_switch( })?; if let Some(bmc_ip) = switch.bmc_ip_address { - update_preallocated_machine_interface(&mut txn, switch.bmc_mac_address, bmc_ip).await?; + update_preallocated_machine_interface( + &mut txn, + switch.bmc_mac_address, + bmc_ip, + api.runtime_config.retained_boot_interface_window, + ) + .await?; } if let Some(nvos_ip) = switch.nvos_ip_address { // Pairing already validated above; nvos_mac_addresses has exactly one entry. let nvos_mac = switch.nvos_mac_addresses[0]; - update_preallocated_machine_interface(&mut txn, nvos_mac, nvos_ip).await?; + update_preallocated_machine_interface( + &mut txn, + nvos_mac, + nvos_ip, + api.runtime_config.retained_boot_interface_window, + ) + .await?; } db_expected_switch::update(&mut txn, &switch) diff --git a/crates/api-core/src/handlers/machine.rs b/crates/api-core/src/handlers/machine.rs index 6c6b8140cf..8384dd01f3 100644 --- a/crates/api-core/src/handlers/machine.rs +++ b/crates/api-core/src/handlers/machine.rs @@ -596,6 +596,9 @@ pub(crate) async fn admin_force_delete_machine( if request.delete_interfaces { for interface in &machine.interfaces { + // The delete retains each row's boot interface pair in + // `retained_boot_interfaces`, so a re-ingested machine + // recovers its boot target before its first DHCP. db::machine_interface::delete(&interface.id, &mut txn).await?; } response.host_interfaces_deleted = true; diff --git a/crates/api-core/src/handlers/machine_discovery.rs b/crates/api-core/src/handlers/machine_discovery.rs index b3f7d6b4ca..7c3e7ebd86 100644 --- a/crates/api-core/src/handlers/machine_discovery.rs +++ b/crates/api-core/src/handlers/machine_discovery.rs @@ -301,6 +301,7 @@ pub(crate) async fn discover_machine( &mut txn, Some(&hardware_info), &machine_id, + api.runtime_config.retained_boot_interface_window, ) .await?; diff --git a/crates/api-core/src/handlers/machine_interface_address.rs b/crates/api-core/src/handlers/machine_interface_address.rs index 5adb82d919..1f3235d066 100644 --- a/crates/api-core/src/handlers/machine_interface_address.rs +++ b/crates/api-core/src/handlers/machine_interface_address.rs @@ -52,6 +52,7 @@ pub async fn update_preallocated_machine_interface( txn: &mut sqlx::PgConnection, bmc_mac_address: MacAddress, bmc_ip: std::net::IpAddr, + retained_window: Option, ) -> Result<(), CarbideError> { let existing = db::machine_interface::find_by_mac_address(&mut *txn, bmc_mac_address).await?; @@ -98,6 +99,7 @@ pub async fn update_preallocated_machine_interface( &bmc_mac_address, true, AddressSelectionStrategy::StaticAddress(bmc_ip), + retained_window, ) .await?; diff --git a/crates/api-core/src/setup.rs b/crates/api-core/src/setup.rs index c0189197f9..8ca0b3e892 100644 --- a/crates/api-core/src/setup.rs +++ b/crates/api-core/src/setup.rs @@ -1357,9 +1357,27 @@ pub async fn initialize_and_start_controllers<'a>( .start(join_set, cancel_token.clone())?; } + let site_explorer_config = { + let mut config = carbide_config.site_explorer.clone(); + // `retained_boot_interface_window` is a single top-level knob + // (retention spans DHCP, deletion, and ingestion -- it isn't a + // site-explorer feature). Site-explorer's copy is `#[serde(skip)]`, + // so it can't be set under `[site_explorer]`; this hand-off is the + // only way the value gets in, sparing a constructor parameter + // through `SiteExplorer::new` and every test fixture. + config.retained_boot_interface_window = carbide_config.retained_boot_interface_window; + if let Some(window) = config.retained_boot_interface_window { + tracing::info!( + window_seconds = window.num_seconds(), + "retained_boot_interface_window configured; retained boot interface \ + records expire instead of waiting forever" + ); + } + config + }; SiteExplorer::new( db_pool.clone(), - carbide_config.site_explorer.clone(), + site_explorer_config, meter.clone(), bmc_explorer.clone(), Arc::new(carbide_config.get_firmware_config()), diff --git a/crates/api-core/src/test_support/default_config.rs b/crates/api-core/src/test_support/default_config.rs index 54c3e801c2..43c9093e35 100644 --- a/crates/api-core/src/test_support/default_config.rs +++ b/crates/api-core/src/test_support/default_config.rs @@ -94,6 +94,7 @@ pub fn get() -> CarbideConfig { initial_dpu_agent_upgrade_policy: None, max_concurrent_machine_updates: None, machine_update_run_interval: Some(1), + retained_boot_interface_window: None, site_explorer: SiteExplorerConfig { enabled: Arc::new(false.into()), run_interval: std::time::Duration::from_secs(0), diff --git a/crates/api-core/src/tests/common/api_fixtures/mod.rs b/crates/api-core/src/tests/common/api_fixtures/mod.rs index 4b43a8d556..fe739d9e55 100644 --- a/crates/api-core/src/tests/common/api_fixtures/mod.rs +++ b/crates/api-core/src/tests/common/api_fixtures/mod.rs @@ -1028,6 +1028,33 @@ pub async fn create_test_env(db_pool: sqlx::PgPool) -> TestEnv { create_test_env_with_overrides(db_pool, Default::default()).await } +/// `create_test_env` with the fixture admin + host-inband site prefixes +/// routable and the host-inband network segment created -- the standard +/// setup for zero-DPU / NicMode ingestion tests. +pub async fn create_test_env_with_host_inband(db_pool: sqlx::PgPool) -> TestEnv { + let env = create_test_env_with_overrides( + db_pool, + TestEnvOverrides { + site_prefixes: Some(vec![ + IpNetwork::new( + network_segment::FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.network(), + network_segment::FIXTURE_ADMIN_NETWORK_SEGMENT_GATEWAY.prefix(), + ) + .unwrap(), + IpNetwork::new( + network_segment::FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.network(), + network_segment::FIXTURE_HOST_INBAND_NETWORK_SEGMENT_GATEWAY.prefix(), + ) + .unwrap(), + ]), + ..Default::default() + }, + ) + .await; + network_segment::create_host_inband_network_segment(&env.api, None).await; + env +} + #[derive(Debug, Default)] pub struct VerifierSimImpl { should_fail_parsing: Arc, @@ -1495,6 +1522,7 @@ pub async fn create_test_env_with_overrides( db_pool.clone(), SiteExplorerConfig { enabled: Arc::new(true.into()), + retained_boot_interface_window: None, // run_interval shouldn't matter, this should not be run(), we only trigger intervals manually. run_interval: Duration::seconds(0).to_std().unwrap(), concurrent_explorations: 100, diff --git a/crates/api-core/src/tests/common/api_fixtures/site_explorer.rs b/crates/api-core/src/tests/common/api_fixtures/site_explorer.rs index 3185dea212..efd99152eb 100644 --- a/crates/api-core/src/tests/common/api_fixtures/site_explorer.rs +++ b/crates/api-core/src/tests/common/api_fixtures/site_explorer.rs @@ -1972,6 +1972,7 @@ pub async fn create_expected_switches( nvos_mac, false, AddressSelectionStrategy::NextAvailableIp, + None, ) .await .map_err(|e| eyre::eyre!("Failed to create NVOS machine interface: {:?}", e)) @@ -1988,6 +1989,7 @@ pub async fn create_expected_switches( &result.bmc_mac_address.clone(), false, AddressSelectionStrategy::NextAvailableIp, + None, ) .await .map_err(|e| eyre::eyre!("Failed to create BMC machine interface: {:?}", e)) diff --git a/crates/api-core/src/tests/dhcp_lease_expiration.rs b/crates/api-core/src/tests/dhcp_lease_expiration.rs index ba85a579a5..76f6444417 100644 --- a/crates/api-core/src/tests/dhcp_lease_expiration.rs +++ b/crates/api-core/src/tests/dhcp_lease_expiration.rs @@ -42,6 +42,7 @@ async fn test_expire_releases_allocation( MacAddress::from_str("aa:bb:cc:dd:ee:01").unwrap(), std::slice::from_ref(&relay), None, + None, ) .await?; let ip = interface.addresses[0]; @@ -213,6 +214,7 @@ async fn test_expire_does_not_delete_static_allocation( &MacAddress::from_str("aa:bb:cc:dd:ee:08").unwrap(), true, AddressSelectionStrategy::StaticAddress(static_ip), + None, ) .await?; txn.commit().await?; @@ -264,6 +266,7 @@ async fn test_static_address_survives_expiration_and_rediscover( &mac, true, AddressSelectionStrategy::StaticAddress(static_ip), + None, ) .await?; txn.commit().await?; @@ -322,6 +325,7 @@ async fn test_expire_with_matching_mac_releases( mac, std::slice::from_ref(&relay), None, + None, ) .await?; let ip = interface.addresses[0]; @@ -435,6 +439,7 @@ async fn test_expire_with_mismatched_mac_is_no_op( mac_b, std::slice::from_ref(&relay), None, + None, ) .await?; let ip = interface.addresses[0]; diff --git a/crates/api-core/src/tests/expected_machine.rs b/crates/api-core/src/tests/expected_machine.rs index 2c4d987430..f561f359ff 100644 --- a/crates/api-core/src/tests/expected_machine.rs +++ b/crates/api-core/src/tests/expected_machine.rs @@ -2254,6 +2254,7 @@ async fn test_add_with_host_nic_fixed_ip_creates_interface( fixed_ip.parse().unwrap(), model::machine_interface::InterfaceType::Data, "expected_machine host NIC", + None, ) .await; @@ -2349,11 +2350,11 @@ async fn test_preallocate_machine_interface_is_idempotent( let ip: std::net::IpAddr = "192.0.2.241".parse().unwrap(); let mut txn = env.db_txn().await; - db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip).await?; + db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip, None).await?; txn.commit().await?; let mut txn = env.db_txn().await; - db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip).await?; + db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip, None).await?; let interfaces = db::machine_interface::find_by_mac_address(txn.as_mut(), mac).await?; txn.commit().await?; @@ -2384,11 +2385,12 @@ async fn test_preallocate_machine_interface_rejects_conflicting_ip( let ip2: std::net::IpAddr = "192.0.2.243".parse().unwrap(); let mut txn = env.db_txn().await; - db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip1).await?; + db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip1, None).await?; txn.commit().await?; let mut txn = env.db_txn().await; - let result = db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip2).await; + let result = + db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip2, None).await; assert!( matches!(result, Err(DatabaseError::InvalidArgument(_))), "preallocating a different IP for the same MAC should be rejected, got {result:?}" @@ -2410,12 +2412,12 @@ async fn test_preallocate_machine_interface_rejects_ip_owned_by_different_mac( let ip: std::net::IpAddr = "192.0.2.248".parse().unwrap(); let mut txn = env.db_txn().await; - db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac_a, ip).await?; + db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac_a, ip, None).await?; txn.commit().await?; let mut txn = env.db_txn().await; let result = - db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac_b, ip).await; + db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac_b, ip, None).await; assert!( matches!(result, Err(DatabaseError::InvalidArgument(_))), "preallocating an IP owned by a different MAC should be rejected, got {result:?}" @@ -2437,14 +2439,14 @@ async fn test_preallocate_machine_interface_recreates_after_deletion( let ip: std::net::IpAddr = "192.0.2.244".parse().unwrap(); let mut txn = env.db_txn().await; - db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip).await?; + db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip, None).await?; let interfaces_before = db::machine_interface::find_by_mac_address(txn.as_mut(), mac).await?; let interface_id = interfaces_before[0].id; db::machine_interface::delete(&interface_id, txn.as_mut()).await?; txn.commit().await?; let mut txn = env.db_txn().await; - db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip).await?; + db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip, None).await?; let interfaces_after = db::machine_interface::find_by_mac_address(txn.as_mut(), mac).await?; txn.commit().await?; @@ -2477,7 +2479,7 @@ async fn test_preallocate_machine_interface_promotes_interface_type( // Initial preallocation lands as InterfaceType::Data. let mut txn = env.db_txn().await; - db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip).await?; + db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, ip, None).await?; let before = db::machine_interface::find_by_mac_address(txn.as_mut(), mac).await?; assert_eq!( before[0].interface_type, @@ -2489,7 +2491,7 @@ async fn test_preallocate_machine_interface_promotes_interface_type( // Re-preallocate the same (MAC, IP) but as the BMC variant. Helper should promote // the existing row's interface_type rather than erroring or creating a duplicate. let mut txn = env.db_txn().await; - db::machine_interface::preallocate_bmc_machine_interface(txn.as_mut(), mac, ip).await?; + db::machine_interface::preallocate_bmc_machine_interface(txn.as_mut(), mac, ip, None).await?; let after = db::machine_interface::find_by_mac_address(txn.as_mut(), mac).await?; txn.commit().await?; @@ -3118,6 +3120,7 @@ async fn test_create_missing_from_preallocates_interfaces( bmc_ip, model::machine_interface::InterfaceType::Bmc, "expected_machine BMC", + None, ) .await; carbide_site_explorer::try_preallocate_one( @@ -3126,6 +3129,7 @@ async fn test_create_missing_from_preallocates_interfaces( host_ip, model::machine_interface::InterfaceType::Data, "expected_machine host NIC", + None, ) .await; diff --git a/crates/api-core/src/tests/expected_power_shelf.rs b/crates/api-core/src/tests/expected_power_shelf.rs index 007b2c1ce2..0cbbaa8551 100644 --- a/crates/api-core/src/tests/expected_power_shelf.rs +++ b/crates/api-core/src/tests/expected_power_shelf.rs @@ -973,6 +973,7 @@ async fn test_add_with_bmc_ip_creates_static_interface( bmc_ip.parse().unwrap(), model::machine_interface::InterfaceType::Bmc, "expected_power_shelf BMC", + None, ) .await; @@ -1072,6 +1073,7 @@ async fn test_add_with_external_bmc_ip_uses_static_assignments( external_ip.parse().unwrap(), model::machine_interface::InterfaceType::Bmc, "expected_power_shelf BMC", + None, ) .await; @@ -1168,6 +1170,7 @@ async fn test_update_with_different_bmc_ip_leaves_interface_alone( original_ip.parse().unwrap(), model::machine_interface::InterfaceType::Bmc, "expected_power_shelf BMC", + None, ) .await; @@ -1220,6 +1223,7 @@ async fn test_update_with_bmc_ip_assigns_to_empty_interface( bmc_mac, std::slice::from_ref(&relay), None, + None, ) .await?; db::machine_interface_address::delete(&mut txn, &iface.id).await?; @@ -1356,6 +1360,7 @@ async fn test_update_without_bmc_ip_does_not_touch_interface( bmc_ip.parse().unwrap(), model::machine_interface::InterfaceType::Bmc, "expected_power_shelf BMC", + None, ) .await; diff --git a/crates/api-core/src/tests/expected_switch.rs b/crates/api-core/src/tests/expected_switch.rs index fa8db2f097..2da3dee876 100644 --- a/crates/api-core/src/tests/expected_switch.rs +++ b/crates/api-core/src/tests/expected_switch.rs @@ -996,6 +996,7 @@ async fn test_add_with_bmc_ip_creates_static_interface( bmc_ip.parse().unwrap(), model::machine_interface::InterfaceType::Bmc, "expected_switch BMC", + None, ) .await; diff --git a/crates/api-core/src/tests/finder.rs b/crates/api-core/src/tests/finder.rs index 38548f41b7..087d31525f 100644 --- a/crates/api-core/src/tests/finder.rs +++ b/crates/api-core/src/tests/finder.rs @@ -402,7 +402,7 @@ async fn test_static_bmc_ip_finder(db_pool: sqlx::PgPool) -> Result<(), eyre::Re let bmc_mac = "AA:BB:CC:DD:EE:99".parse().unwrap(); let mut txn = db_pool.begin().await.unwrap(); - db::machine_interface::preallocate_machine_interface(txn.as_mut(), bmc_mac, static_ip) + db::machine_interface::preallocate_machine_interface(txn.as_mut(), bmc_mac, static_ip, None) .await .expect("preallocate static BMC interface"); txn.commit().await.unwrap(); diff --git a/crates/api-core/src/tests/ip_allocator.rs b/crates/api-core/src/tests/ip_allocator.rs index 27e1c94949..9d3f750d0b 100644 --- a/crates/api-core/src/tests/ip_allocator.rs +++ b/crates/api-core/src/tests/ip_allocator.rs @@ -70,6 +70,7 @@ async fn test_machine_interface_create_with_ipv4_prefix( MacAddress::from_str("ff:ff:ff:ff:ff:ff").as_ref().unwrap(), true, AddressSelectionStrategy::NextAvailableIp, + None, ) .await .unwrap(); @@ -91,6 +92,7 @@ async fn test_machine_interface_create_with_ipv4_prefix( &MacAddress::from_str("ff:ff:ff:ff:ff:fe").unwrap(), false, AddressSelectionStrategy::NextAvailableIp, + None, ) .await .unwrap(); @@ -152,6 +154,7 @@ async fn test_machine_interface_create_falls_through_admin_segments( &MacAddress::from_str("aa:bb:cc:dd:ee:10").unwrap(), true, AddressSelectionStrategy::NextAvailableIp, + None, ) .await?; assert_eq!(first_interface.segment_id, first_segment.id); @@ -167,6 +170,7 @@ async fn test_machine_interface_create_falls_through_admin_segments( &MacAddress::from_str("aa:bb:cc:dd:ee:11").unwrap(), false, AddressSelectionStrategy::NextAvailableIp, + None, ) .await?; @@ -228,6 +232,7 @@ async fn test_machine_interface_create_with_ipv6_prefix( &MacAddress::from_str("aa:bb:cc:dd:ee:01").unwrap(), true, AddressSelectionStrategy::NextAvailableIp, + None, ) .await?; @@ -249,6 +254,7 @@ async fn test_machine_interface_create_with_ipv6_prefix( &MacAddress::from_str("aa:bb:cc:dd:ee:02").unwrap(), false, AddressSelectionStrategy::NextAvailableIp, + None, ) .await?; @@ -315,6 +321,7 @@ async fn test_machine_interface_create_dual_stack( &MacAddress::from_str("aa:bb:cc:00:00:01").unwrap(), true, AddressSelectionStrategy::NextAvailableIp, + None, ) .await?; @@ -350,6 +357,7 @@ async fn test_machine_interface_create_dual_stack( &MacAddress::from_str("aa:bb:cc:00:00:02").unwrap(), false, AddressSelectionStrategy::NextAvailableIp, + None, ) .await?; diff --git a/crates/api-core/src/tests/ipxe.rs b/crates/api-core/src/tests/ipxe.rs index d15f363448..c62a9d8f15 100644 --- a/crates/api-core/src/tests/ipxe.rs +++ b/crates/api-core/src/tests/ipxe.rs @@ -534,7 +534,7 @@ async fn preallocate_external_interface( let ip_addr: std::net::IpAddr = ip.parse().unwrap(); let mut txn = env.pool.begin().await.unwrap(); - db::machine_interface::preallocate_machine_interface(&mut txn, mac_address, ip_addr) + db::machine_interface::preallocate_machine_interface(&mut txn, mac_address, ip_addr, None) .await .unwrap(); txn.commit().await.unwrap(); diff --git a/crates/api-core/src/tests/machine_admin_force_delete.rs b/crates/api-core/src/tests/machine_admin_force_delete.rs index 320e4cc86d..c32e45fdfb 100644 --- a/crates/api-core/src/tests/machine_admin_force_delete.rs +++ b/crates/api-core/src/tests/machine_admin_force_delete.rs @@ -829,3 +829,65 @@ async fn test_admin_force_delete_with_dpf_uses_bmc_mac(pool: sqlx::PgPool) { ); } } + +/// `delete_interfaces` keeps each deleted interface's boot pair alive in +/// `retained_boot_interfaces`: the vendor-named Redfish interface id is the +/// one piece a re-ingested machine can't always rediscover on its own +/// (after a DPU-to-NIC mode flip the BMC can report the id without its +/// MAC), so the pair is recorded at the only moment it's guaranteed +/// complete -- deletion. +#[crate::sqlx_test] +async fn test_admin_force_delete_retains_boot_interface_ids(pool: sqlx::PgPool) { + let env = create_test_env(pool).await; + let (host_machine_id, _dpu_machine_id) = create_managed_host(&env).await.into(); + + // Record a boot interface id on the host's interface row, as + // site-explorer would during exploration. + let mut txn = env.pool.begin().await.unwrap(); + let host_machine = db::machine::find_one( + txn.as_mut(), + &host_machine_id, + MachineSearchConfig::default(), + ) + .await + .unwrap() + .unwrap(); + let boot_mac = host_machine.interfaces[0].mac_address; + db::machine_interface::set_boot_interface_id(boot_mac, "NIC.Slot.5-1", txn.as_mut()) + .await + .unwrap(); + txn.commit().await.unwrap(); + + let response = env + .api + .admin_force_delete_machine(tonic::Request::new(AdminForceDeleteMachineRequest { + host_query: host_machine_id.to_string(), + delete_interfaces: true, + delete_bmc_interfaces: false, + delete_bmc_credentials: false, + allow_delete_with_orphaned_dpf_crds: false, + })) + .await + .unwrap() + .into_inner(); + assert!(response.all_done, "host must be deleted"); + assert!(response.host_interfaces_deleted); + + let mut txn = env.pool.begin().await.unwrap(); + // The interface row is gone... + assert!( + db::machine_interface::find_by_mac_address(txn.as_mut(), boot_mac) + .await + .unwrap() + .is_empty() + ); + // ...and its boot pair survives in retention, ready for the re-ingest. + assert_eq!( + db::retained_boot_interface::find_by_mac(txn.as_mut(), boot_mac, None) + .await + .unwrap() + .as_deref(), + Some("NIC.Slot.5-1"), + ); + txn.rollback().await.unwrap(); +} diff --git a/crates/api-core/src/tests/machine_dhcp.rs b/crates/api-core/src/tests/machine_dhcp.rs index 427a12e6f0..a1b269a0b5 100644 --- a/crates/api-core/src/tests/machine_dhcp.rs +++ b/crates/api-core/src/tests/machine_dhcp.rs @@ -57,6 +57,7 @@ async fn test_machine_dhcp(pool: sqlx::PgPool) -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box> { + let env = common::api_fixtures::create_test_env_with_host_inband(pool.clone()).await; + + let mock_host = ManagedHostConfig { + dpus: vec![], + ..ManagedHostConfig::default() + }; + let inband_mac = *mock_host.non_dpu_macs.first().unwrap(); + api_fixtures::site_explorer::register_expected_machine(&env, &mock_host, None).await; + + MockExploredHost::new(&env, mock_host) + .discover_dhcp_host_bmc(|result, _| { + assert!(result.is_ok()); + Ok(()) + }) + .await? + // Site-explorer runs BEFORE the in-band NIC ever DHCPs, so ingestion + // mints a predicted interface for it. + .insert_site_exploration_results()? + .run_site_explorer_iteration() + .await + .mark_preingestion_complete() + .await? + .run_site_explorer_iteration() + .await + .then(move |mock| { + let pool = mock.test_env.pool.clone(); + async move { + let mut txn = pool.begin().await?; + let predicted = + db::predicted_machine_interface::find_by_mac_address(&mut txn, inband_mac) + .await? + .expect("zero-DPU ingest should have minted a predicted interface"); + // The fixture report names the embedded NIC's Redfish id. + assert_eq!( + predicted.boot_interface_id.as_deref(), + Some("NIC.Embedded.1-1-1"), + "predicted interface should hold the report-derived boot interface id" + ); + Ok(()) + } + }) + .await? + // The in-band NIC's first DHCP promotes the prediction into a + // machine_interfaces row. + .discover_dhcp_host_primary_iface(|result, _| { + let response = result.unwrap().into_inner(); + assert!(response.machine_id.is_some()); + Ok(()) + }) + .await? + .then(move |mock| { + let pool = mock.test_env.pool.clone(); + async move { + let mut txn = pool.begin().await?; + let interfaces = + db::machine_interface::find_by_mac_address(txn.as_mut(), inband_mac).await?; + assert_eq!(interfaces.len(), 1); + assert_eq!( + interfaces[0].boot_interface_id.as_deref(), + Some("NIC.Embedded.1-1-1"), + "promotion should land the predicted boot interface id on the promoted row" + ); + assert!( + db::predicted_machine_interface::find_by_mac_address(&mut txn, inband_mac) + .await? + .is_none(), + "the prediction should be consumed by promotion" + ); + Ok(()) + } + }) + .await?; + + Ok(()) +} + +/// When a retained boot interface id AND a prediction with a live-report +/// id both exist for a MAC, DHCP promotion lands the LIVE id on the +/// promoted row -- the prediction is refreshed every exploration cycle, +/// while the retained id predates the deletion that recorded it. The +/// retention record is consumed either way. +#[sqlx_test] +async fn test_predicted_live_boot_interface_id_outranks_retained_at_promotion( + pool: PgPool, +) -> Result<(), Box> { + let env = common::api_fixtures::create_test_env_with_host_inband(pool.clone()).await; + + let mock_host = ManagedHostConfig { + dpus: vec![], + ..ManagedHostConfig::default() + }; + let inband_mac = *mock_host.non_dpu_macs.first().unwrap(); + api_fixtures::site_explorer::register_expected_machine(&env, &mock_host, None).await; + + // A prior row for this MAC was deleted with its boot pair retained -- + // the id names a slot the NIC occupied before the migration. + let mut txn = env.pool.begin().await?; + db::retained_boot_interface::upsert(txn.as_mut(), inband_mac, "NIC.Old.9-9-9").await?; + txn.commit().await?; + + MockExploredHost::new(&env, mock_host) + .discover_dhcp_host_bmc(|result, _| { + assert!(result.is_ok()); + Ok(()) + }) + .await? + // Ingestion mints a predicted interface holding the CURRENT id + // from the live report. + .insert_site_exploration_results()? + .run_site_explorer_iteration() + .await + .mark_preingestion_complete() + .await? + .run_site_explorer_iteration() + .await + .then(move |mock| { + let pool = mock.test_env.pool.clone(); + async move { + let mut txn = pool.begin().await?; + let predicted = + db::predicted_machine_interface::find_by_mac_address(&mut txn, inband_mac) + .await? + .expect("zero-DPU ingest should have minted a predicted interface"); + assert_eq!( + predicted.boot_interface_id.as_deref(), + Some("NIC.Embedded.1-1-1"), + "the prediction holds the live-report id, never the retained one" + ); + Ok(()) + } + }) + .await? + // The in-band NIC's first DHCP promotes the prediction. Creation + // recovers the retained id onto the brand-new row first -- the + // prediction's live id must still win. + .discover_dhcp_host_primary_iface(|result, _| { + assert!(result.is_ok()); + Ok(()) + }) + .await? + .then(move |mock| { + let pool = mock.test_env.pool.clone(); + async move { + let mut txn = pool.begin().await?; + let interfaces = + db::machine_interface::find_by_mac_address(txn.as_mut(), inband_mac).await?; + assert_eq!(interfaces.len(), 1); + assert_eq!( + interfaces[0].boot_interface_id.as_deref(), + Some("NIC.Embedded.1-1-1"), + "the live predicted id outranks the retained id on the promoted row" + ); + assert!( + db::retained_boot_interface::find_by_mac(txn.as_mut(), inband_mac, None) + .await? + .is_none(), + "the retention record is consumed by promotion regardless" + ); + Ok(()) + } + }) + .await?; + + Ok(()) +} + +/// If a static preallocation creates the machine_interfaces row while a +/// prediction is still pending (an ExpectedMachine `fixed_ip` recorded in +/// between), the prediction's live-report id still outranks the retained +/// id the preallocated row recovered. +#[sqlx_test] +async fn test_predicted_live_boot_interface_id_outranks_preallocated_retained_row_at_promotion( + pool: PgPool, +) -> Result<(), Box> { + let env = common::api_fixtures::create_test_env_with_host_inband(pool.clone()).await; + + let mock_host = ManagedHostConfig { + dpus: vec![], + ..ManagedHostConfig::default() + }; + let inband_mac = *mock_host.non_dpu_macs.first().unwrap(); + api_fixtures::site_explorer::register_expected_machine(&env, &mock_host, None).await; + + // A prior row retained an obsolete boot interface id for this MAC. + let mut txn = env.pool.begin().await?; + db::retained_boot_interface::upsert(txn.as_mut(), inband_mac, "NIC.Old.9-9-9").await?; + txn.commit().await?; + + MockExploredHost::new(&env, mock_host) + .discover_dhcp_host_bmc(|result, _| { + assert!(result.is_ok()); + Ok(()) + }) + .await? + // Site-explorer mints a pending prediction with the current Redfish id. + .insert_site_exploration_results()? + .run_site_explorer_iteration() + .await + .mark_preingestion_complete() + .await? + .run_site_explorer_iteration() + .await + .then(move |mock| { + let pool = mock.test_env.pool.clone(); + async move { + let static_ip: std::net::IpAddr = "192.0.3.77".parse()?; + let mut txn = pool.begin().await?; + + // A `fixed_ip` declaration creates the row on the same + // HostInband segment before the NIC ever DHCPs; creation + // recovers the retained (obsolete) id onto it. + db::machine_interface::preallocate_machine_interface( + txn.as_mut(), + inband_mac, + static_ip, + None, + ) + .await?; + + let interfaces = + db::machine_interface::find_by_mac_address(txn.as_mut(), inband_mac).await?; + assert_eq!(interfaces.len(), 1); + assert_eq!( + interfaces[0].boot_interface_id.as_deref(), + Some("NIC.Old.9-9-9") + ); + txn.commit().await?; + Ok(()) + } + }) + .await? + // DHCP promotion must overwrite the preallocated row with the live id. + .discover_dhcp_host_primary_iface(|result, _| { + assert!(result.is_ok()); + Ok(()) + }) + .await? + .then(move |mock| { + let pool = mock.test_env.pool.clone(); + async move { + let mut txn = pool.begin().await?; + let interfaces = + db::machine_interface::find_by_mac_address(txn.as_mut(), inband_mac).await?; + assert_eq!(interfaces.len(), 1); + assert_eq!( + interfaces[0].boot_interface_id.as_deref(), + Some("NIC.Embedded.1-1-1"), + "the live predicted id outranks the preallocation-recovered retained id" + ); + Ok(()) + } + }) + .await?; + + Ok(()) +} + +/// A newly DHCP-created machine_interface recovers a retained boot +/// interface id -- recorded when a prior row for its MAC was deleted (e.g. +/// admin force-delete during a DPU-to-NIC mode migration) -- and consumes +/// the retention record. +#[sqlx_test] +async fn test_dhcp_created_interface_recovers_retained_boot_interface_id( + pool: PgPool, +) -> Result<(), Box> { + let env = common::api_fixtures::create_test_env_with_host_inband(pool.clone()).await; + + let mock_host = ManagedHostConfig { + dpus: vec![], + ..ManagedHostConfig::default() + }; + let inband_mac = *mock_host.non_dpu_macs.first().unwrap(); + api_fixtures::site_explorer::register_expected_machine(&env, &mock_host, None).await; + + // A prior interface row for this MAC was deleted with its boot pair + // retained; seed that record directly. + let mut txn = env.pool.begin().await?; + db::retained_boot_interface::upsert(txn.as_mut(), inband_mac, "NIC.Retained.7-1-1").await?; + txn.commit().await?; + + MockExploredHost::new(&env, mock_host) + // DHCP arrives before site-explorer ever runs: the brand-new row + // recovers the retained boot interface id on creation. + .discover_dhcp_host_primary_iface(|result, _| { + assert!(result.is_ok()); + Ok(()) + }) + .await? + .then(move |mock| { + let pool = mock.test_env.pool.clone(); + async move { + let mut txn = pool.begin().await?; + let interfaces = + db::machine_interface::find_by_mac_address(txn.as_mut(), inband_mac).await?; + assert_eq!(interfaces.len(), 1); + assert_eq!( + interfaces[0].boot_interface_id.as_deref(), + Some("NIC.Retained.7-1-1"), + "the new row should recover the retained boot interface id" + ); + assert!( + db::retained_boot_interface::find_by_mac(txn.as_mut(), inband_mac, None) + .await? + .is_none(), + "the retention record should be consumed once applied" + ); + Ok(()) + } + }) + .await?; + + Ok(()) +} + +/// Retention recovery is centralized at row creation, so even a static +/// preallocation (a declared `fixed_ip` reservation) recovers a retained +/// boot interface id -- the pair must not depend on WHICH path recreates +/// the row after a force-delete. +#[sqlx_test] +async fn test_preallocated_interface_recovers_retained_boot_interface_id( + pool: PgPool, +) -> Result<(), Box> { + let env = common::api_fixtures::create_test_env(pool.clone()).await; + + let mac: MacAddress = "aa:55:66:77:88:99".parse()?; + // An external static IP: preallocation homes it on the + // static-assignments anchor segment, no fixture segment needed. + let static_ip: std::net::IpAddr = "203.0.113.7".parse()?; + + // A prior row for this MAC was deleted with its boot pair retained. + let mut txn = env.pool.begin().await?; + db::retained_boot_interface::upsert(txn.as_mut(), mac, "NIC.Static.1-1-1").await?; + txn.commit().await?; + + // The static reservation recreates the row (the path a declared + // fixed_ip takes via DHCP discover or site-explorer reconciliation). + let mut txn = env.pool.begin().await?; + db::machine_interface::preallocate_machine_interface(txn.as_mut(), mac, static_ip, None) + .await?; + txn.commit().await?; + + let mut txn = env.pool.begin().await?; + let interfaces = db::machine_interface::find_by_mac_address(txn.as_mut(), mac).await?; + assert_eq!(interfaces.len(), 1); + assert_eq!( + interfaces[0].boot_interface_id.as_deref(), + Some("NIC.Static.1-1-1"), + "a preallocation-created row recovers the retained boot interface id" + ); + assert!( + db::retained_boot_interface::find_by_mac(txn.as_mut(), mac, None) + .await? + .is_none(), + "the retention record is consumed once applied" + ); + txn.rollback().await?; + + Ok(()) +} + +/// The expiry sweep removes only records older than the configured window +/// -- and removes nothing when no window is set (records wait forever for +/// their machine to come back). +#[sqlx_test] +async fn test_retained_boot_interface_sweep_removes_only_expired_records( + pool: PgPool, +) -> Result<(), Box> { + let _env = common::api_fixtures::create_test_env(pool.clone()).await; + + let old_mac: MacAddress = "aa:bb:cc:00:00:01".parse()?; + let recent_mac: MacAddress = "aa:bb:cc:00:00:02".parse()?; + + let mut txn = pool.begin().await?; + db::retained_boot_interface::upsert(txn.as_mut(), old_mac, "NIC.Old.1-1-1").await?; + db::retained_boot_interface::upsert(txn.as_mut(), recent_mac, "NIC.Recent.1-1-1").await?; + // Age one record past the window. + sqlx::query( + "UPDATE retained_boot_interfaces SET recorded_at = NOW() - INTERVAL '2 hours' \ + WHERE mac_address = $1", + ) + .bind(old_mac) + .execute(txn.as_mut()) + .await?; + txn.commit().await?; + + // No window -> nothing is swept. + let mut txn = pool.begin().await?; + assert_eq!( + db::retained_boot_interface::delete_expired(txn.as_mut(), None).await?, + 0, + "without a window the sweep must leave every record in place" + ); + let swept = + db::retained_boot_interface::delete_expired(txn.as_mut(), Some(chrono::Duration::hours(1))) + .await?; + assert_eq!(swept, 1, "only the aged-out record is swept"); + assert!( + db::retained_boot_interface::find_by_mac(txn.as_mut(), old_mac, None) + .await? + .is_none(), + "the aged-out record is gone" + ); + assert_eq!( + db::retained_boot_interface::find_by_mac(txn.as_mut(), recent_mac, None) + .await? + .as_deref(), + Some("NIC.Recent.1-1-1"), + "the in-window record survives the sweep" + ); + txn.rollback().await?; + + Ok(()) +} + +/// A prediction minted before the BMC report resolved the NIC's Redfish id +/// is refreshed by the next exploration that does resolve it -- pending +/// predictions stay as current as the live report until DHCP promotes them. +#[sqlx_test] +async fn test_exploration_refreshes_pending_predicted_boot_interface_id( + pool: PgPool, +) -> Result<(), Box> { + let env = common::api_fixtures::create_test_env_with_host_inband(pool.clone()).await; + + let mock_host = ManagedHostConfig { + dpus: vec![], + ..ManagedHostConfig::default() + }; + let inband_mac = *mock_host.non_dpu_macs.first().unwrap(); + api_fixtures::site_explorer::register_expected_machine(&env, &mock_host, None).await; + + // First exploration: the BMC reports the NIC's MAC but no Redfish id + // yet, so the minted prediction has no boot interface id. + let mut id_less_report: model::site_explorer::EndpointExplorationReport = + mock_host.clone().into(); + for system in id_less_report.systems.iter_mut() { + for iface in system.ethernet_interfaces.iter_mut() { + iface.id = None; + } + } + + let mock = MockExploredHost::new(&env, mock_host) + .discover_dhcp_host_bmc(|result, _| { + assert!(result.is_ok()); + Ok(()) + }) + .await?; + let host_bmc_ip = mock.host_bmc_ip.expect("host BMC should have DHCP'd"); + env.endpoint_explorer + .insert_endpoint_result(host_bmc_ip, Ok(id_less_report)); + + let mock = mock + .run_site_explorer_iteration() + .await + .mark_preingestion_complete() + .await? + .run_site_explorer_iteration() + .await + .then(move |mock| { + let pool = mock.test_env.pool.clone(); + async move { + let mut txn = pool.begin().await?; + let predicted = + db::predicted_machine_interface::find_by_mac_address(&mut txn, inband_mac) + .await? + .expect("zero-DPU ingest should have minted a predicted interface"); + assert!( + predicted.boot_interface_id.is_none(), + "an id-less report can't give the prediction a boot interface id" + ); + Ok(()) + } + }) + .await?; + + // Second exploration: the BMC now resolves the id; the pending + // prediction picks it up. + env.endpoint_explorer + .insert_endpoint_result(host_bmc_ip, Ok(mock.managed_host.clone().into())); + mock.run_site_explorer_iteration() + .await + .then(move |mock| { + let pool = mock.test_env.pool.clone(); + async move { + let mut txn = pool.begin().await?; + let predicted = + db::predicted_machine_interface::find_by_mac_address(&mut txn, inband_mac) + .await? + .expect("the prediction should still be pending"); + assert_eq!( + predicted.boot_interface_id.as_deref(), + Some("NIC.Embedded.1-1-1"), + "the next exploration that resolves the id refreshes the prediction" + ); + Ok(()) + } + }) + .await?; + + Ok(()) +} diff --git a/crates/api-core/src/tests/static_address_management.rs b/crates/api-core/src/tests/static_address_management.rs index 3d3dec84b0..ada9df9bd9 100644 --- a/crates/api-core/src/tests/static_address_management.rs +++ b/crates/api-core/src/tests/static_address_management.rs @@ -41,6 +41,7 @@ async fn test_assign_static_address(pool: sqlx::PgPool) -> Result<(), Box Result<(), Box, is_primary: Option, + retained_window: Option, ) -> DatabaseResult { let relaystr = relays .iter() @@ -452,8 +453,14 @@ pub async fn find_or_create_machine_interface( // at creation. If the caller has explicitly declared a *different* // NIC as this machine's primary (i.e. is_primary == false), override the // true/default here. - let mut interface = - validate_existing_mac_and_create(&mut *txn, mac_address, relays, host_nic).await?; + let mut interface = validate_existing_mac_and_create( + &mut *txn, + mac_address, + relays, + host_nic, + retained_window, + ) + .await?; if is_primary == Some(false) && interface.primary_interface { set_primary_interface(&interface.id, false, &mut *txn).await?; interface.primary_interface = false; @@ -486,6 +493,7 @@ pub async fn validate_existing_mac_and_create( mac_address: MacAddress, relays: &[IpAddr], host_nic: Option, + retained_window: Option, ) -> DatabaseResult { let mut interface_snapshot = find_by_mac_address(&mut *txn, mac_address).await?; match &interface_snapshot.len() { @@ -535,12 +543,14 @@ pub async fn validate_existing_mac_and_create( // Dynamic-pool allocation. // Any AddressSelectionStrategy::StaticIp flows will have happened as part of // preallocate_machine_interface or preallocate_bmc_machine_interface. + // (`create` recovers any retained boot interface id onto the new row.) let v = create( txn, &network_segments, &mac_address, true, AddressSelectionStrategy::NextAvailableIp, + retained_window, ) .await?; Ok(v) @@ -598,16 +608,32 @@ pub async fn preallocate_machine_interface( txn: &mut PgConnection, mac_address: MacAddress, static_ip: IpAddr, + retained_window: Option, ) -> DatabaseResult<()> { - preallocate_machine_interface_with_type(txn, mac_address, static_ip, InterfaceType::Data).await + preallocate_machine_interface_with_type( + txn, + mac_address, + static_ip, + InterfaceType::Data, + retained_window, + ) + .await } pub async fn preallocate_bmc_machine_interface( txn: &mut PgConnection, mac_address: MacAddress, static_ip: IpAddr, + retained_window: Option, ) -> DatabaseResult<()> { - preallocate_machine_interface_with_type(txn, mac_address, static_ip, InterfaceType::Bmc).await + preallocate_machine_interface_with_type( + txn, + mac_address, + static_ip, + InterfaceType::Bmc, + retained_window, + ) + .await } /// If a machine interface row already exists for `mac_address`, reconcile it against the @@ -646,6 +672,7 @@ async fn preallocate_machine_interface_with_type( mac_address: MacAddress, static_ip: IpAddr, interface_type: InterfaceType, + retained_window: Option, ) -> DatabaseResult<()> { // If there's already a matching record for (ip, mac), just return Ok, // instead of attempting to insert, getting a duplicate error, and then @@ -675,6 +702,7 @@ async fn preallocate_machine_interface_with_type( interface_type != InterfaceType::Bmc, AddressSelectionStrategy::StaticAddress(static_ip), interface_type, + retained_window, ) .await { @@ -711,6 +739,7 @@ pub async fn create( macaddr: &MacAddress, primary_interface: bool, address_strategy: AddressSelectionStrategy, + retained_window: Option, ) -> DatabaseResult { create_with_type( txn, @@ -719,6 +748,7 @@ pub async fn create( primary_interface, address_strategy, InterfaceType::Data, + retained_window, ) .await } @@ -730,8 +760,9 @@ pub async fn create_with_type( primary_interface: bool, address_strategy: AddressSelectionStrategy, interface_type: InterfaceType, + retained_window: Option, ) -> DatabaseResult { - match address_strategy { + let mut snapshot = match address_strategy { AddressSelectionStrategy::NextAvailableIp | AddressSelectionStrategy::Automatic => { create_fast_path(txn, segments, macaddr, primary_interface, interface_type).await } @@ -765,7 +796,22 @@ pub async fn create_with_type( ) .await } + }?; + + // Every brand-new row passes through here, whatever created it -- + // dynamic DHCP, a static preallocation, or predicted-interface + // promotion. A prior row for this MAC may have been deleted with its + // boot interface id retained; recover the pair onto the new row and + // consume the retention record. + if snapshot.boot_interface_id.is_none() + && let Some(boot_interface_id) = + crate::retained_boot_interface::take_by_mac(&mut *txn, *macaddr, retained_window) + .await? + { + set_boot_interface_id(*macaddr, &boot_interface_id, &mut *txn).await?; + snapshot.boot_interface_id = Some(boot_interface_id); } + Ok(snapshot) } #[allow(txn_held_across_await)] @@ -1349,6 +1395,7 @@ pub async fn move_predicted_machine_interface_to_machine( txn: &mut PgConnection, predicted_machine_interface: &PredictedMachineInterface, relay_ip: IpAddr, + retained_window: Option, ) -> Result<(), DatabaseError> { tracing::info!( machine_id=%predicted_machine_interface.machine_id, @@ -1373,51 +1420,61 @@ pub async fn move_predicted_machine_interface_to_machine( ))); } - let machine_interface_id = match self::find_by_mac_address( - &mut *txn, - predicted_machine_interface.mac_address, - ) - .await? - .into_iter() - .find(|machine_interface| machine_interface.segment_id == network_segment.id) + let existing_row = + self::find_by_mac_address(&mut *txn, predicted_machine_interface.mac_address) + .await? + .into_iter() + .find(|machine_interface| machine_interface.segment_id == network_segment.id); + + if let Some(machine_id) = existing_row + .as_ref() + .and_then(|machine_interface| machine_interface.machine_id.as_ref()) { - Some(machine_interface_snapshot) => { - match machine_interface_snapshot.machine_id.as_ref() { - None => { - // This host has already DHCP'd once and created an anonymous machine_interface, - // we will migrate it below. - machine_interface_snapshot.id - } - Some(machine_id) => { - if machine_id.ne(&predicted_machine_interface.machine_id) { - tracing::error!( - %machine_id, - "Can't migrate predicted_machine_interface to machine_interface: one already exists with this MAC address" - ); - return Err(DatabaseError::NetworkSegmentDuplicateMacAddress( - predicted_machine_interface.mac_address, - )); - } else { - tracing::warn!( - %machine_id, - "Bug: trying to move predicted_machine_interface to machine_interface, but it's already a part of this machine? Will proceed anyway." - ); - machine_interface_snapshot.id - } - } - } + if machine_id.ne(&predicted_machine_interface.machine_id) { + tracing::error!( + %machine_id, + "Can't migrate predicted_machine_interface to machine_interface: one already exists with this MAC address" + ); + return Err(DatabaseError::NetworkSegmentDuplicateMacAddress( + predicted_machine_interface.mac_address, + )); } + // To even get here, the interface must have been attached to the + // machine through some path that didn't clean up the prediction -- + // think a concurrent DHCP for the same MAC, or an attach flow that + // doesn't know predictions exist. There's nothing left to migrate, + // so just finish the bookkeeping below and remove the prediction. + tracing::warn!( + %machine_id, + "Bug: trying to move predicted_machine_interface to machine_interface, but it's already a part of this machine? Will proceed anyway." + ); + } + + let (machine_interface_id, current_boot_interface_id, row_created_here) = match existing_row { + // This host has already DHCP'd once and created a machine_interface; + // we will migrate it below. + Some(machine_interface_snapshot) => ( + machine_interface_snapshot.id, + machine_interface_snapshot.boot_interface_id, + false, + ), None => { // This host has never DHCP'd before, create a new machine_interface for it + // (`create` recovers any retained boot interface id onto it). let machine_interface = create( txn, &[network_segment], &predicted_machine_interface.mac_address, false, AddressSelectionStrategy::NextAvailableIp, + retained_window, ) .await?; - machine_interface.id + ( + machine_interface.id, + machine_interface.boot_interface_id, + true, + ) } }; @@ -1430,6 +1487,41 @@ pub async fn move_predicted_machine_interface_to_machine( ) .await?; + // Resolve the promoted row's boot interface id. The prediction's value + // comes from the live report and outranks an existing row value: that + // row may have been created from a static preallocation (an + // ExpectedMachine `fixed_ip` recorded while the prediction was pending) + // and recovered an older retained id. The retention record is consumed + // either way (creation already consumed it, or the take here does): + // from here on the MAC has a `machine_interfaces` row for explorations + // to keep up to date. + let retained_boot_interface_id = if row_created_here { + // Creation already consumed the record; any recovered value is on + // the row (`current_boot_interface_id`). + None + } else { + crate::retained_boot_interface::take_by_mac( + &mut *txn, + predicted_machine_interface.mac_address, + retained_window, + ) + .await? + }; + let predicted_boot_interface_id = predicted_machine_interface.boot_interface_id.clone(); + let resolved_boot_interface_id = predicted_boot_interface_id + .or(current_boot_interface_id.clone()) + .or(retained_boot_interface_id); + if let Some(boot_interface_id) = resolved_boot_interface_id + && current_boot_interface_id.as_deref() != Some(boot_interface_id.as_str()) + { + set_boot_interface_id( + predicted_machine_interface.mac_address, + &boot_interface_id, + &mut *txn, + ) + .await?; + } + crate::predicted_machine_interface::delete(predicted_machine_interface, txn).await?; Ok(()) } @@ -1443,6 +1535,7 @@ pub async fn create_host_machine_dpu_interface_proactively( txn: &mut PgConnection, hardware_info: Option<&HardwareInfo>, dpu_id: &MachineId, + retained_window: Option, ) -> Result { let admin_networks = crate::network_segment::admin(txn).await?; @@ -1479,9 +1572,16 @@ pub async fn create_host_machine_dpu_interface_proactively( } } - let machine_interface = - find_or_create_machine_interface(txn, existing_machine, host_mac, &gateways, None, None) - .await?; + let machine_interface = find_or_create_machine_interface( + txn, + existing_machine, + host_mac, + &gateways, + None, + None, + retained_window, + ) + .await?; associate_interface_with_dpu_machine(&machine_interface.id, dpu_id, txn).await?; Ok(machine_interface) @@ -2240,16 +2340,25 @@ pub async fn delete( interface_id: &MachineInterfaceId, txn: &mut PgConnection, ) -> Result<(), DatabaseError> { - let query = "DELETE FROM machine_interfaces WHERE id=$1"; + let query = + "DELETE FROM machine_interfaces WHERE id=$1 RETURNING mac_address, boot_interface_id"; crate::machine_interface_address::delete(txn, interface_id).await?; crate::dhcp_entry::delete(txn, interface_id).await?; - sqlx::query(query) + let deleted: Option<(MacAddress, Option)> = sqlx::query_as(query) .bind(*interface_id) - .execute(&mut *txn) + .fetch_optional(&mut *txn) .await - .map(|_| ()) .map_err(|e| DatabaseError::query(query, e))?; + // Every row deletion retains the boot pair: the vendor-named Redfish + // interface id is the one piece a future row for this MAC can't always + // rediscover (after a DPU/NIC mode flip the BMC can report the id + // without its MAC), so it outlives the row in `retained_boot_interfaces` + // no matter which caller deleted it. + if let Some((mac_address, Some(boot_interface_id))) = deleted { + crate::retained_boot_interface::upsert(&mut *txn, mac_address, &boot_interface_id).await?; + } + let query = "UPDATE machine_interfaces_deletion SET last_deletion=NOW() WHERE id = 1"; sqlx::query(query) .bind(*interface_id) diff --git a/crates/api-db/src/predicted_machine_interface.rs b/crates/api-db/src/predicted_machine_interface.rs index 586151b61d..3a96931136 100644 --- a/crates/api-db/src/predicted_machine_interface.rs +++ b/crates/api-db/src/predicted_machine_interface.rs @@ -54,6 +54,26 @@ pub async fn find_by<'a, C: ColumnInfo<'a, TableType = PredictedMachineInterface .map_err(|e| DatabaseError::query(query.sql(), e)) } +/// Records the vendor-named Redfish `EthernetInterface.Id` on the predicted +/// row(s) with the given MAC, keeping pending predictions as current as the +/// live report -- the same per-exploration refresh `machine_interfaces` +/// rows get. +pub async fn set_boot_interface_id( + txn: &mut PgConnection, + mac_address: MacAddress, + boot_interface_id: &str, +) -> Result<(), DatabaseError> { + let query = + "UPDATE predicted_machine_interfaces SET boot_interface_id = $1 WHERE mac_address = $2"; + sqlx::query(query) + .bind(boot_interface_id) + .bind(mac_address) + .execute(txn) + .await + .map(|_| ()) + .map_err(|e| DatabaseError::query(query, e)) +} + pub async fn delete( value: &PredictedMachineInterface, txn: &mut PgConnection, @@ -83,11 +103,12 @@ pub async fn create( value: NewPredictedMachineInterface<'_>, txn: &mut PgConnection, ) -> Result { - let query = "INSERT INTO predicted_machine_interfaces (machine_id, mac_address, expected_network_segment_type) VALUES ($1, $2, $3) RETURNING *"; + let query = "INSERT INTO predicted_machine_interfaces (machine_id, mac_address, expected_network_segment_type, boot_interface_id) VALUES ($1, $2, $3, $4) RETURNING *"; sqlx::query_as(query) .bind(value.machine_id) .bind(value.mac_address) .bind(value.expected_network_segment_type) + .bind(&value.boot_interface_id) .fetch_one(txn) .await .map_err(|e| DatabaseError::query(query, e)) diff --git a/crates/api-db/src/retained_boot_interface.rs b/crates/api-db/src/retained_boot_interface.rs new file mode 100644 index 0000000000..76acf57784 --- /dev/null +++ b/crates/api-db/src/retained_boot_interface.rs @@ -0,0 +1,125 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +//! Last-known boot interface pairs that outlive their `machine_interfaces` +//! rows. +//! +//! When an interface row is deleted, its `boot_interface_id` -- the +//! vendor-named Redfish `EthernetInterface.Id` -- is the one piece of state +//! a re-ingested machine cannot always rediscover on its own: after a +//! DPU/NIC mode flip the BMC can report the interface id without its MAC, +//! so the pair can't be re-derived later. Rows here are written by +//! `machine_interface::delete` itself, keyed by MAC with no foreign keys +//! (everything they referenced is gone), and consumed once the id lands on +//! a `machine_interfaces` row again. +//! +//! Records are honored for as long as the operator-configured +//! `retained_boot_interface_window` allows. The default (`None`) is +//! forever -- if the machine eventually comes back, the pair is waiting. +//! Setting a window bounds the reach of a recycled MAC: one reappearing on +//! different hardware long after a deletion (virtual-MAC reassignment, a +//! NIC moved between slots) should not inherit an interface id that aims +//! boot-order setup at a Redfish resource that no longer exists there. +//! Migrations consume their records within minutes either way. + +use mac_address::MacAddress; +use sqlx::PgConnection; + +use crate::DatabaseError; + +/// Record the boot interface pair for a MAC, overwriting any prior record +/// (the newest observation wins). +pub async fn upsert( + txn: &mut PgConnection, + mac_address: MacAddress, + boot_interface_id: &str, +) -> Result<(), DatabaseError> { + let query = "INSERT INTO retained_boot_interfaces (mac_address, boot_interface_id) \ + VALUES ($1, $2) \ + ON CONFLICT (mac_address) \ + DO UPDATE SET boot_interface_id = EXCLUDED.boot_interface_id, recorded_at = NOW()"; + sqlx::query(query) + .bind(mac_address) + .bind(boot_interface_id) + .execute(txn) + .await + .map_err(|e| DatabaseError::query(query, e))?; + Ok(()) +} + +/// Look up the retained boot interface id for a MAC without consuming it. +/// Records older than `window` (when one is set) are not returned. The +/// consuming `take_by_mac` is the one production flows use; this read is +/// for inspection (and test assertions). +pub async fn find_by_mac( + txn: &mut PgConnection, + mac_address: MacAddress, + window: Option, +) -> Result, DatabaseError> { + let query = "SELECT boot_interface_id FROM retained_boot_interfaces \ + WHERE mac_address = $1 \ + AND ($2::bigint IS NULL OR recorded_at > NOW() - ($2::bigint * INTERVAL '1 second'))"; + sqlx::query_scalar(query) + .bind(mac_address) + .bind(window.map(|w| w.num_seconds())) + .fetch_optional(txn) + .await + .map_err(|e| DatabaseError::query(query, e)) +} + +/// Consume the retained record for a MAC, returning its boot interface id +/// when the record is within `window` (always, when no window is set). The +/// record is removed either way -- a `machine_interfaces` row now +/// exists for the MAC, so future explorations keep it current and the +/// retention copy is done. +pub async fn take_by_mac( + txn: &mut PgConnection, + mac_address: MacAddress, + window: Option, +) -> Result, DatabaseError> { + let query = "DELETE FROM retained_boot_interfaces WHERE mac_address = $1 \ + RETURNING boot_interface_id, \ + ($2::bigint IS NULL OR recorded_at > NOW() - ($2::bigint * INTERVAL '1 second')) AS applicable"; + let row: Option<(String, bool)> = sqlx::query_as(query) + .bind(mac_address) + .bind(window.map(|w| w.num_seconds())) + .fetch_optional(txn) + .await + .map_err(|e| DatabaseError::query(query, e))?; + Ok(row.and_then(|(boot_interface_id, applicable)| applicable.then_some(boot_interface_id))) +} + +/// Remove records that have aged out of `window`. A no-op when no window +/// is set -- without one, every record waits forever for its machine to +/// come back. Reads already ignore expired records; this sweep keeps MACs +/// that never return from occupying table rows indefinitely. +pub async fn delete_expired( + txn: &mut PgConnection, + window: Option, +) -> Result { + let Some(window) = window else { + return Ok(0); + }; + let query = "DELETE FROM retained_boot_interfaces \ + WHERE recorded_at <= NOW() - ($1::bigint * INTERVAL '1 second')"; + let result = sqlx::query(query) + .bind(window.num_seconds()) + .execute(txn) + .await + .map_err(|e| DatabaseError::query(query, e))?; + Ok(result.rows_affected()) +} diff --git a/crates/api-model/src/predicted_machine_interface.rs b/crates/api-model/src/predicted_machine_interface.rs index fc7af612f3..1c806e6a21 100644 --- a/crates/api-model/src/predicted_machine_interface.rs +++ b/crates/api-model/src/predicted_machine_interface.rs @@ -27,6 +27,10 @@ pub struct PredictedMachineInterface { pub machine_id: MachineId, pub mac_address: MacAddress, pub expected_network_segment_type: NetworkSegmentType, + /// The last known vendor-named Redfish `EthernetInterface.Id` for this + /// MAC, handed to the `machine_interfaces` row at DHCP promotion so + /// the host's boot target is a full pair from its first owned interface. + pub boot_interface_id: Option, } #[derive(Debug, Clone)] @@ -34,4 +38,5 @@ pub struct NewPredictedMachineInterface<'a> { pub machine_id: &'a MachineId, pub mac_address: MacAddress, pub expected_network_segment_type: NetworkSegmentType, + pub boot_interface_id: Option, } diff --git a/crates/site-explorer/src/config.rs b/crates/site-explorer/src/config.rs index 5d2f101e7a..bfc7305bca 100644 --- a/crates/site-explorer/src/config.rs +++ b/crates/site-explorer/src/config.rs @@ -39,6 +39,13 @@ pub struct SiteExplorerConfig { serialize_with = "serialize_arc_atomic_bool" )] pub enabled: Arc, + /// How long a retained boot interface pair stays applicable (`None` = + /// forever). Deliberately not part of the `[site_explorer]` section + /// (serde skips it): setup copies the top-level + /// `retained_boot_interface_window` here so site-explorer's ingest paths + /// honor the same knob as DHCP. + #[serde(skip)] + pub retained_boot_interface_window: Option, /// The interval at which site explorer runs. /// Defaults to 5 Minutes if not specified. #[serde( @@ -192,6 +199,7 @@ impl Default for SiteExplorerConfig { fn default() -> Self { SiteExplorerConfig { enabled: Arc::new(true.into()), + retained_boot_interface_window: None, run_interval: Self::default_run_interval(), concurrent_explorations: Self::default_concurrent_explorations(), explorations_per_run: Self::default_explorations_per_run(), diff --git a/crates/site-explorer/src/lib.rs b/crates/site-explorer/src/lib.rs index 18a3d135dd..b4f54d24c3 100644 --- a/crates/site-explorer/src/lib.rs +++ b/crates/site-explorer/src/lib.rs @@ -734,6 +734,29 @@ impl SiteExplorer { self.audit_exploration_results(metrics, &expected_endpoint_index) .await?; + // Retained boot interface records that aged out of the configured + // window are already ignored at read time; sweep them once per pass + // so MACs that never return don't occupy table rows indefinitely. + // (A no-op without a window: records wait for their machine.) + if self.config.retained_boot_interface_window.is_some() { + let mut txn = self + .database_connection + .begin() + .await + .map_err(|e| DatabaseError::new("begin retained boot interface sweep", e))?; + let swept = db::retained_boot_interface::delete_expired( + &mut txn, + self.config.retained_boot_interface_window, + ) + .await?; + txn.commit() + .await + .map_err(|e| DatabaseError::new("end retained boot interface sweep", e))?; + if swept > 0 { + tracing::info!(swept, "Removed expired retained boot interface records"); + } + } + Ok(identified_hosts) } @@ -1379,6 +1402,9 @@ impl SiteExplorer { // Record each host NIC's Redfish id on its machine_interfaces row so the // primary-flagged row is the host's complete boot interface (MAC + id). + // Pending predicted interfaces get the same refresh, so a prediction + // minted before the report resolved the id stays as current as the + // live report until DHCP promotes it. for boot_interface in &nic_boot_interfaces { db::machine_interface::set_boot_interface_id( boot_interface.mac_address, @@ -1386,6 +1412,12 @@ impl SiteExplorer { &mut txn, ) .await?; + db::predicted_machine_interface::set_boot_interface_id( + &mut txn, + boot_interface.mac_address, + &boot_interface.interface_id, + ) + .await?; } txn.commit().await?; @@ -1578,6 +1610,7 @@ impl SiteExplorer { bmc_ip, InterfaceType::Bmc, "expected_machine BMC", + self.config.retained_boot_interface_window, ) .await; } @@ -1591,6 +1624,7 @@ impl SiteExplorer { ip, InterfaceType::Data, "expected_machine host NIC", + self.config.retained_boot_interface_window, ) .await; } @@ -1604,6 +1638,7 @@ impl SiteExplorer { bmc_ip, InterfaceType::Bmc, "expected_switch BMC", + self.config.retained_boot_interface_window, ) .await; } @@ -1620,6 +1655,7 @@ impl SiteExplorer { nvos_ip, InterfaceType::Data, "expected_switch NVOS", + self.config.retained_boot_interface_window, ) .await; } @@ -1643,6 +1679,7 @@ impl SiteExplorer { bmc_ip, InterfaceType::Bmc, "expected_power_shelf BMC", + self.config.retained_boot_interface_window, ) .await; } @@ -2828,6 +2865,7 @@ pub async fn try_preallocate_one( ip: IpAddr, interface_type: InterfaceType, kind: &'static str, + retained_window: Option, ) { let mut txn = match db::Transaction::begin(pool).await { Ok(t) => t, @@ -2841,10 +2879,22 @@ pub async fn try_preallocate_one( }; let result = match interface_type { InterfaceType::Bmc => { - db::machine_interface::preallocate_bmc_machine_interface(txn.as_pgconn(), mac, ip).await + db::machine_interface::preallocate_bmc_machine_interface( + txn.as_pgconn(), + mac, + ip, + retained_window, + ) + .await } InterfaceType::Data => { - db::machine_interface::preallocate_machine_interface(txn.as_pgconn(), mac, ip).await + db::machine_interface::preallocate_machine_interface( + txn.as_pgconn(), + mac, + ip, + retained_window, + ) + .await } }; match result { diff --git a/crates/site-explorer/src/machine_creator.rs b/crates/site-explorer/src/machine_creator.rs index b7404edc35..daa8d226ba 100644 --- a/crates/site-explorer/src/machine_creator.rs +++ b/crates/site-explorer/src/machine_creator.rs @@ -311,6 +311,18 @@ impl MachineCreator { // can't rely on matching the machine_id, as it may have migrated to a stable MachineID // already. let mac_addresses = host_mac_addresses_for_predicted_machine(report, machine_data); + + // Resolve each MAC's Redfish interface id from the live report up + // front (`generate_machine_id` below takes a mutable borrow of the + // report that lives for the rest of this function). + let report_boot_interface_ids: Vec<(MacAddress, String)> = mac_addresses + .iter() + .filter_map(|mac| { + report + .find_interface_id_for_mac(*mac) + .map(|id| (*mac, id.to_string())) + }) + .collect(); for mac_address in &mac_addresses { if db::machine::find_by_mac_address(txn, mac_address) .await? @@ -411,11 +423,24 @@ impl MachineCreator { .await?; } } else { + // Give the predicted interface its boot interface id when + // the live report resolves one, so the promoted row starts + // with the full boot pair. Retained ids are deliberately + // NOT copied here: a prediction has no recorded_at, so a + // copy would dodge the `retained_boot_interface_window` + // check. The retained pair instead lands on the row at + // creation (see `create_with_type`), where the window is + // checked at DHCP time. + let boot_interface_id = report_boot_interface_ids + .iter() + .find(|(mac, _)| *mac == mac_address) + .map(|(_, id)| id.clone()); db::predicted_machine_interface::create( NewPredictedMachineInterface { machine_id, mac_address, expected_network_segment_type: NetworkSegmentType::HostInband, + boot_interface_id, }, txn, ) @@ -584,6 +609,7 @@ impl MachineCreator { txn, Some(&dpu_hw_info), explored_dpu.report.machine_id.as_ref().unwrap(), + self.config.retained_boot_interface_window, ) .await?; diff --git a/crates/site-explorer/tests/reconcile.rs b/crates/site-explorer/tests/reconcile.rs index 72f4fc24fe..56878bdcb8 100644 --- a/crates/site-explorer/tests/reconcile.rs +++ b/crates/site-explorer/tests/reconcile.rs @@ -128,6 +128,7 @@ async fn test_site_explorer_reconcile_creates_missing_preallocations( ip, model::machine_interface::InterfaceType::Bmc, kind, + None, ) .await; } @@ -191,6 +192,7 @@ async fn test_site_explorer_reconcile_is_idempotent( bmc_ip, model::machine_interface::InterfaceType::Bmc, "expected_machine BMC", + None, ) .await; } @@ -250,6 +252,7 @@ async fn test_site_explorer_reconcile_preallocates_host_nic_fixed_ip( parsed_fixed_ip, model::machine_interface::InterfaceType::Data, "expected_machine host NIC", + None, ) .await; @@ -322,6 +325,7 @@ async fn test_site_explorer_reconcile_tolerates_per_entry_conflicts( ip, model::machine_interface::InterfaceType::Bmc, "expected_machine BMC", + None, ) .await; } @@ -400,6 +404,7 @@ async fn test_site_explorer_reconcile_preallocates_nvos_ip( nvos_ip, model::machine_interface::InterfaceType::Data, "expected_switch NVOS", + None, ) .await; diff --git a/crates/site-explorer/tests/site_explorer.rs b/crates/site-explorer/tests/site_explorer.rs index 56f0a175cf..46d3eca7c5 100644 --- a/crates/site-explorer/tests/site_explorer.rs +++ b/crates/site-explorer/tests/site_explorer.rs @@ -176,6 +176,7 @@ async fn test_handle_redfish_error_powers_on_machine( let explorer_config = SiteExplorerConfig { enabled: Arc::new(true.into()), + retained_boot_interface_window: None, explorations_per_run: 1, concurrent_explorations: 1, run_interval: std::time::Duration::from_secs(1), @@ -255,6 +256,7 @@ async fn test_site_explorer_skips_unexpected_zero_dpu_host( let explorer_config = SiteExplorerConfig { enabled: Arc::new(true.into()), + retained_boot_interface_window: None, explorations_per_run: 1, concurrent_explorations: 1, run_interval: std::time::Duration::from_secs(1), @@ -347,6 +349,7 @@ async fn test_site_explorer_ingests_nic_mode_host_with_no_observed_dpus( let explorer_config = SiteExplorerConfig { enabled: Arc::new(true.into()), + retained_boot_interface_window: None, explorations_per_run: 1, concurrent_explorations: 1, run_interval: std::time::Duration::from_secs(1), @@ -422,6 +425,7 @@ async fn test_site_explorer_ingests_no_dpu_host( let explorer_config = SiteExplorerConfig { enabled: Arc::new(true.into()), + retained_boot_interface_window: None, explorations_per_run: 1, concurrent_explorations: 1, run_interval: std::time::Duration::from_secs(1), @@ -479,6 +483,7 @@ async fn test_site_explorer_unknown_vendor(pool: PgPool) -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box(d: &Option, serializer: S) -> Result +where + S: Serializer, +{ + match d { + Some(d) => serializer.serialize_some(&format!("{}s", d.num_seconds())), + None => serializer.serialize_none(), + } +}