diff --git a/Cargo.lock b/Cargo.lock index 22ac6881..3db5c45d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -876,7 +876,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "fbuild-bench-fastled-examples" -version = "2.3.0" +version = "2.3.1" dependencies = [ "fbuild-library-select", "fbuild-packages", @@ -887,7 +887,7 @@ dependencies = [ [[package]] name = "fbuild-build" -version = "2.3.0" +version = "2.3.1" dependencies = [ "async-trait", "blake3", @@ -919,7 +919,7 @@ dependencies = [ [[package]] name = "fbuild-cli" -version = "2.3.0" +version = "2.3.1" dependencies = [ "blake3", "clap", @@ -949,7 +949,7 @@ dependencies = [ [[package]] name = "fbuild-config" -version = "2.3.0" +version = "2.3.1" dependencies = [ "fbuild-core", "include_dir", @@ -963,9 +963,10 @@ dependencies = [ [[package]] name = "fbuild-core" -version = "2.3.0" +version = "2.3.1" dependencies = [ "libc", + "prost", "running-process", "serde", "serde_json", @@ -981,7 +982,7 @@ dependencies = [ [[package]] name = "fbuild-daemon" -version = "2.3.0" +version = "2.3.1" dependencies = [ "async-trait", "axum", @@ -1021,7 +1022,7 @@ dependencies = [ [[package]] name = "fbuild-deploy" -version = "2.3.0" +version = "2.3.1" dependencies = [ "async-trait", "espflash", @@ -1044,7 +1045,7 @@ dependencies = [ [[package]] name = "fbuild-header-scan" -version = "2.3.0" +version = "2.3.1" dependencies = [ "criterion", "rayon", @@ -1054,7 +1055,7 @@ dependencies = [ [[package]] name = "fbuild-library-select" -version = "2.3.0" +version = "2.3.1" dependencies = [ "bincode", "blake3", @@ -1073,7 +1074,7 @@ dependencies = [ [[package]] name = "fbuild-packages" -version = "2.3.0" +version = "2.3.1" dependencies = [ "axum", "bzip2", @@ -1100,14 +1101,14 @@ dependencies = [ [[package]] name = "fbuild-paths" -version = "2.3.0" +version = "2.3.1" dependencies = [ "fbuild-core", ] [[package]] name = "fbuild-python" -version = "2.3.0" +version = "2.3.1" dependencies = [ "base64", "fbuild-core", @@ -1129,7 +1130,7 @@ dependencies = [ [[package]] name = "fbuild-serial" -version = "2.3.0" +version = "2.3.1" dependencies = [ "async-trait", "base64", @@ -1151,7 +1152,7 @@ dependencies = [ [[package]] name = "fbuild-test-support" -version = "2.3.0" +version = "2.3.1" dependencies = [ "fbuild-config", "fbuild-header-scan", diff --git a/crates/fbuild-cli/src/cli/port_scan.rs b/crates/fbuild-cli/src/cli/port_scan.rs index f6460659..2cf50ecb 100644 --- a/crates/fbuild-cli/src/cli/port_scan.rs +++ b/crates/fbuild-cli/src/cli/port_scan.rs @@ -67,9 +67,8 @@ fn run_scan(offline: bool) -> Result<()> { Ok(()) } -/// Fetch the FastLED/fbuild `online-data` branch's `usb-vid.json` (the -/// tier-2 overlay backing [`fbuild_core::usb::resolve`]) into the -/// local cache root, then install it. +/// Fetch the FastLED/boards `usb-vids.proto.zstd` tier-2 overlay backing +/// [`fbuild_core::usb::resolve`] into the local cache root, then install it. /// /// Best-effort: any I/O / network / parse failure is swallowed and the /// resolver degrades to tier-1 (embedded vendor archive). The cache is @@ -86,14 +85,14 @@ fn populate_online_overlay() { ); } } - fbuild_core::usb::install_online_cache(&cache_path); + fbuild_core::usb::install_online_cache_proto_zstd(&cache_path); } fn overlay_cache_path() -> Option { let root = fbuild_paths::get_cache_root(); let dir = root.join("usb"); std::fs::create_dir_all(&dir).ok()?; - Some(dir.join("usb-vid.json")) + Some(dir.join("usb-vids.proto.zstd")) } /// 7-day cache TTL — fbuild's online-data branch refreshes nightly; @@ -131,8 +130,16 @@ fn fetch_overlay_to_inner(path: &std::path::Path) -> std::result::Result<(), Str .timeout(Duration::from_secs(15)) .build() .map_err(|e| format!("client build: {e}"))?; + fetch_overlay_to_inner_with_client(path, &client, fbuild_core::usb::USB_VIDS_PROTO_ZSTD_URL) +} + +fn fetch_overlay_to_inner_with_client( + path: &std::path::Path, + client: &reqwest::blocking::Client, + url: &str, +) -> std::result::Result<(), String> { let response = client - .get(fbuild_core::usb::USB_VID_JSON_URL) + .get(url) .send() .map_err(|e| format!("http get: {e}"))?; if !response.status().is_success() { @@ -141,7 +148,7 @@ fn fetch_overlay_to_inner(path: &std::path::Path) -> std::result::Result<(), Str let body = response.bytes().map_err(|e| format!("body read: {e}"))?; // Atomic write via a `.tmp` sibling + rename — partial writes from // a Ctrl+C mid-fetch don't poison the cache. - let tmp = path.with_extension("json.tmp"); + let tmp = path.with_extension("proto.zstd.tmp"); std::fs::write(&tmp, &body).map_err(|e| format!("tmp write: {e}"))?; std::fs::rename(&tmp, path).map_err(|e| format!("rename: {e}"))?; tracing::debug!( @@ -315,6 +322,33 @@ fn render_non_usb(out: &mut String, name: &str, kind: &str) { mod tests { use super::*; use serialport::{SerialPortInfo, SerialPortType, UsbPortInfo}; + use std::io::{Read, Write}; + use std::net::TcpListener; + use std::sync::Mutex; + + static ENV_LOCK: Mutex<()> = Mutex::new(()); + + struct EnvVarGuard { + name: &'static str, + previous: Option, + } + + impl EnvVarGuard { + fn set(name: &'static str, value: impl AsRef) -> Self { + let previous = std::env::var_os(name); + std::env::set_var(name, value); + Self { name, previous } + } + } + + impl Drop for EnvVarGuard { + fn drop(&mut self) { + match self.previous.take() { + Some(value) => std::env::set_var(self.name, value), + None => std::env::remove_var(self.name), + } + } + } fn usb_port( name: &str, @@ -335,6 +369,51 @@ mod tests { } } + #[test] + fn overlay_cache_path_uses_fbuild_cache_dir() { + let _env = ENV_LOCK.lock().unwrap(); + let tmp = tempfile::tempdir().unwrap(); + let _guard = EnvVarGuard::set("FBUILD_CACHE_DIR", tmp.path()); + + let path = overlay_cache_path().expect("cache path"); + + assert_eq!(path, tmp.path().join("usb").join("usb-vids.proto.zstd")); + assert!(tmp.path().join("usb").is_dir()); + } + + #[test] + fn fetch_overlay_writes_cache_file_atomically() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let expected = b"fake proto zstd bytes".to_vec(); + let server_expected = expected.clone(); + let handle = std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let mut request = [0_u8; 1024]; + let _ = stream.read(&mut request).unwrap(); + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_expected.len() + ); + stream.write_all(response.as_bytes()).unwrap(); + stream.write_all(&server_expected).unwrap(); + }); + + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("usb-vids.proto.zstd"); + let client = reqwest::blocking::Client::builder() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(); + + fetch_overlay_to_inner_with_client(&path, &client, &format!("http://{addr}/data")) + .expect("fetch should write cache"); + + handle.join().unwrap(); + assert_eq!(std::fs::read(&path).unwrap(), expected); + assert!(!path.with_extension("proto.zstd.tmp").exists()); + } + #[test] fn empty_port_list_renders_canonical_message() { assert_eq!(render_scan(&[]), "no serial ports visible\n"); diff --git a/crates/fbuild-core/Cargo.toml b/crates/fbuild-core/Cargo.toml index f89393bb..a429a409 100644 --- a/crates/fbuild-core/Cargo.toml +++ b/crates/fbuild-core/Cargo.toml @@ -23,6 +23,7 @@ usb-ids = { workspace = true } # wire format without per-crate version drift. zstd = { workspace = true } tar = { workspace = true } +prost = { workspace = true } # Process containment primitive (Job Objects on Windows; process groups + # PR_SET_PDEATHSIG on Linux; process groups on macOS). The single global # `ContainedProcessGroup` owned by the daemon ensures every child process diff --git a/crates/fbuild-core/src/usb/data.rs b/crates/fbuild-core/src/usb/data.rs index 7f4cca1a..4884b6a8 100644 --- a/crates/fbuild-core/src/usb/data.rs +++ b/crates/fbuild-core/src/usb/data.rs @@ -1,7 +1,25 @@ -//! Tier-2 online overlay: an optional per-VID JSON map loaded from disk +//! Tier-2 online overlay: an optional per-VID protobuf map loaded from disk //! at runtime. //! -//! Schema on disk (from `online-data/data/usb-vid.json`): +//! Current on-disk schema is `usb-vids.proto.zstd`, published by +//! : +//! +//! ```protobuf +//! message UsbVidDatabase { +//! repeated Vendor vendors = 1; +//! } +//! message Vendor { +//! uint32 vid = 1; +//! string name = 2; +//! repeated Product products = 3; +//! } +//! message Product { +//! uint32 pid = 1; +//! string name = 2; +//! } +//! ``` +//! +//! The legacy JSON loader remains for old caches/tests. Its schema was: //! //! ```json //! { @@ -27,35 +45,61 @@ //! shape on disk just avoids duplicating the vendor name for every //! product entry under a VID (significantly smaller payload). //! -//! The daemon downloads the JSON from the repo's `online-data` branch, -//! writes it to a cache path, and calls [`install_online_cache`] to plug -//! it into the resolver. Replacing the cache is supported (`RwLock`, not -//! `OnceLock`) so the daemon can refresh during a long-running session -//! without a restart. +//! The CLI downloads the zstd-compressed protobuf from FastLED/boards, +//! writes it to the global fbuild cache root, and calls +//! [`install_online_cache_proto_zstd`] to plug it into the resolver. +//! Replacing the cache is supported (`RwLock`, not `OnceLock`) so the +//! daemon/CLI can refresh during a long-running session without a restart. //! //! All errors here are swallowed by design — if the overlay can't load, the //! resolver simply degrades to tier-1 + tier-3. use super::UsbInfo; +use prost::Message; use serde::Deserialize; use std::collections::HashMap; use std::path::Path; use std::sync::RwLock; -/// URL of the dataset index produced by the `online-data` branch's nightly -/// workflow. Clients can `GET` this, parse the JSON, and pull the -/// `datasets["usb-vid"].url` field to find the live `usb-vid.json`. +/// Legacy URL of the dataset index produced by the `online-data` branch's +/// nightly workflow. pub const MANIFEST_URL: &str = "https://raw.githubusercontent.com/fastled/fbuild/online-data/manifest.json"; -/// Direct convenience URL for the merged dataset itself. Kept in sync with -/// [`MANIFEST_URL`]'s `datasets["usb-vid"].url` by the nightly workflow. -/// Clients that don't want to parse the manifest can fetch this directly. +/// Legacy JSON overlay URL. Kept for compatibility with older callers and +/// tests; new code should use [`USB_VIDS_PROTO_ZSTD_URL`]. pub const USB_VID_JSON_URL: &str = "https://raw.githubusercontent.com/fastled/fbuild/online-data/data/usb-vid.json"; +/// Current compact USB VID:PID overlay published by FastLED/boards. +pub const USB_VIDS_PROTO_ZSTD_URL: &str = "https://fastled.github.io/boards/usb-vids.proto.zstd"; + static ONLINE_MAP: RwLock>> = RwLock::new(None); +#[derive(Clone, PartialEq, Message)] +struct UsbVidDatabase { + #[prost(message, repeated, tag = "1")] + vendors: Vec, +} + +#[derive(Clone, PartialEq, Message)] +struct Vendor { + #[prost(uint32, tag = "1")] + vid: u32, + #[prost(string, tag = "2")] + name: String, + #[prost(message, repeated, tag = "3")] + products: Vec, +} + +#[derive(Clone, PartialEq, Message)] +struct Product { + #[prost(uint32, tag = "1")] + pid: u32, + #[prost(string, tag = "2")] + name: String, +} + /// On-disk representation: one entry per VID, with the vendor name shared /// across all products of that VID. Each product is a two-element /// `[pid_hex, product_name]` tuple. @@ -109,6 +153,72 @@ pub fn install_online_cache(path: &Path) { tracing::debug!(path = %path.display(), entries = count, "usb online overlay installed"); } +/// Install the overlay from the current `usb-vids.proto.zstd` cache file. +/// Silently no-ops on any IO, zstd, or protobuf decode error so USB +/// resolution always degrades to the embedded vendor archive instead of +/// failing port enumeration. +pub fn install_online_cache_proto_zstd(path: &Path) { + let raw = match std::fs::read(path) { + Ok(bytes) => bytes, + Err(e) => { + tracing::debug!(?path, error = %e, "usb online overlay: read failed"); + return; + } + }; + match decode_proto_zstd_bytes(&raw) { + Ok(map) => { + let count = map.len(); + install_online_cache_map(map); + tracing::debug!( + path = %path.display(), + entries = count, + "usb online protobuf overlay installed" + ); + } + Err(e) => { + tracing::warn!( + ?path, + error = %e, + "usb online protobuf overlay decode failed" + ); + } + } +} + +fn decode_proto_zstd_bytes(raw: &[u8]) -> Result, String> { + let mut decoded = Vec::with_capacity(raw.len() * 4); + zstd::stream::copy_decode(raw, &mut decoded).map_err(|e| format!("zstd: {e}"))?; + decode_proto_bytes(&decoded) +} + +fn decode_proto_bytes(raw: &[u8]) -> Result, String> { + let db = UsbVidDatabase::decode(raw).map_err(|e| format!("protobuf: {e}"))?; + Ok(proto_to_map(db)) +} + +fn proto_to_map(db: UsbVidDatabase) -> HashMap { + let product_count = db.vendors.iter().map(|vendor| vendor.products.len()).sum(); + let mut packed: HashMap = HashMap::with_capacity(product_count); + for vendor in db.vendors { + let Ok(vid) = u16::try_from(vendor.vid) else { + continue; + }; + for product in vendor.products { + let Ok(pid) = u16::try_from(product.pid) else { + continue; + }; + packed.insert( + pack(vid, pid), + UsbInfo { + vendor: vendor.name.clone(), + product: product.name, + }, + ); + } + } + packed +} + /// Replace the overlay with a pre-built map. Exposed at `pub(crate)` so /// the daemon could in principle skip the file dance — primary user is the /// resolver's own test suite. @@ -188,6 +298,57 @@ mod tests { clear_online_cache_for_tests(); } + #[test] + fn install_online_cache_proto_zstd_round_trip() { + let _guard = OVERLAY_LOCK.lock().unwrap(); + clear_online_cache_for_tests(); + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("usb-vids.proto.zstd"); + let db = UsbVidDatabase { + vendors: vec![Vendor { + vid: 0x303A, + name: "Espressif Systems".to_string(), + products: vec![ + Product { + pid: 0x0002, + name: "ESP32-S2".to_string(), + }, + Product { + pid: 0x1001, + name: "USB JTAG/serial debug unit".to_string(), + }, + ], + }], + }; + let mut encoded = Vec::new(); + db.encode(&mut encoded).unwrap(); + let compressed = zstd::stream::encode_all(encoded.as_slice(), 19).unwrap(); + std::fs::write(&path, compressed).unwrap(); + + install_online_cache_proto_zstd(&path); + + let a = lookup(0x303A, 0x0002).expect("pid 0002 parsed"); + assert_eq!(a.vendor, "Espressif Systems"); + assert_eq!(a.product, "ESP32-S2"); + + let b = lookup(0x303A, 0x1001).expect("pid 1001 parsed"); + assert_eq!(b.vendor, "Espressif Systems"); + assert_eq!(b.product, "USB JTAG/serial debug unit"); + + clear_online_cache_for_tests(); + } + + #[test] + fn install_online_cache_proto_zstd_bad_file_is_silent() { + let _guard = OVERLAY_LOCK.lock().unwrap(); + clear_online_cache_for_tests(); + let tmp = tempfile::tempdir().unwrap(); + let path = tmp.path().join("bad.proto.zstd"); + std::fs::write(&path, b"not zstd").unwrap(); + install_online_cache_proto_zstd(&path); + assert!(lookup(0x303A, 0x1001).is_none()); + } + #[test] fn install_online_cache_missing_file_is_silent() { let _guard = OVERLAY_LOCK.lock().unwrap(); diff --git a/crates/fbuild-core/src/usb/mod.rs b/crates/fbuild-core/src/usb/mod.rs index 56733eae..11a273a9 100644 --- a/crates/fbuild-core/src/usb/mod.rs +++ b/crates/fbuild-core/src/usb/mod.rs @@ -30,6 +30,9 @@ pub mod data; pub mod embedded; pub mod resolver; -pub use data::{install_online_cache, MANIFEST_URL, USB_VID_JSON_URL}; +pub use data::{ + install_online_cache, install_online_cache_proto_zstd, MANIFEST_URL, USB_VIDS_PROTO_ZSTD_URL, + USB_VID_JSON_URL, +}; pub use embedded::vendor_name as embedded_vendor_name; pub use resolver::{pretty, resolve, resolve_bundled, try_resolve, UsbInfo};