diff --git a/.intentionally-empty-file.o b/.intentionally-empty-file.o new file mode 100644 index 00000000..48cdce85 --- /dev/null +++ b/.intentionally-empty-file.o @@ -0,0 +1 @@ +placeholder diff --git a/Cargo.lock b/Cargo.lock index d31aa595..68e19614 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1229,6 +1229,7 @@ dependencies = [ name = "fbuild-core" version = "2.3.12" dependencies = [ + "async-trait", "libc", "prost", "running-process", @@ -1340,6 +1341,7 @@ dependencies = [ name = "fbuild-packages" version = "2.3.12" dependencies = [ + "async-trait", "axum 0.7.9", "bzip2", "fbuild-config", diff --git a/crates/fbuild-build/Cargo.toml b/crates/fbuild-build/Cargo.toml index 25d3267d..e119d115 100644 --- a/crates/fbuild-build/Cargo.toml +++ b/crates/fbuild-build/Cargo.toml @@ -45,3 +45,5 @@ zccache = { git = "https://github.com/zackees/zccache", rev = "73d3f84542deb16f7 [dev-dependencies] filetime = { workspace = true } fbuild-test-support = { path = "../fbuild-test-support" } +async-trait = { workspace = true } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "sync"] } diff --git a/crates/fbuild-build/src/apollo3/mod.rs b/crates/fbuild-build/src/apollo3/mod.rs index 22bcd93c..809dcd5d 100644 --- a/crates/fbuild-build/src/apollo3/mod.rs +++ b/crates/fbuild-build/src/apollo3/mod.rs @@ -1,4 +1,4 @@ -//! Apollo3 platform build support (Ambiq Micro Apollo3 / SparkFun Artemis). +//! Apollo3 platform build support (Ambiq Micro Apollo3 / SparkFun Artemis). pub mod mcu_config; pub mod orchestrator; @@ -8,15 +8,16 @@ pub use orchestrator::Apollo3Orchestrator; /// Apollo3 platform support. pub struct Apollo3PlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for Apollo3PlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::ArmGcc8Toolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("ARM GCC 8 toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/apollo3/orchestrator.rs b/crates/fbuild-build/src/apollo3/orchestrator.rs index f6b476b4..2cdddb90 100644 --- a/crates/fbuild-build/src/apollo3/orchestrator.rs +++ b/crates/fbuild-build/src/apollo3/orchestrator.rs @@ -1,4 +1,4 @@ -//! Apollo3 build orchestrator — wires together config, packages, compiler, linker. +//! Apollo3 build orchestrator — wires together config, packages, compiler, linker. //! //! Build phases: //! 1. Parse platformio.ini @@ -25,16 +25,17 @@ use crate::{BuildOrchestrator, BuildParams, BuildResult, SourceScanner}; /// Apollo3 platform build orchestrator. pub struct Apollo3Orchestrator; +#[async_trait::async_trait] impl BuildOrchestrator for Apollo3Orchestrator { fn platform(&self) -> Platform { Platform::Apollo3 } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = pipeline::BuildContext::new(params)?; + let mut ctx = pipeline::BuildContext::new(params).await?; // Compute eh_frame strip policy once per build (FastLED/fbuild#244). let eh_frame_policy = @@ -42,7 +43,7 @@ impl BuildOrchestrator for Apollo3Orchestrator { // 3. Ensure ARM GCC 8 toolchain (Apollo3/mbed-os requires GCC 8) let toolchain = fbuild_packages::toolchain::ArmGcc8Toolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-gcc8 toolchain at {}", toolchain_dir.display()); use fbuild_packages::Toolchain; @@ -50,11 +51,12 @@ impl BuildOrchestrator for Apollo3Orchestrator { &toolchain.get_gcc_path(), "arm-none-eabi-gcc", &mut ctx.build_log, - ); + ) + .await; // 4. Ensure Apollo3 cores (SparkFun Arduino Apollo3 core) let framework = fbuild_packages::library::Apollo3Cores::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("Apollo3 cores at {}", framework_dir.display()); // 5. Scan sources (core + variant) @@ -270,6 +272,7 @@ impl BuildOrchestrator for Apollo3Orchestrator { "APOLLO3", start, ) + .await } } diff --git a/crates/fbuild-build/src/avr/avr_compiler.rs b/crates/fbuild-build/src/avr/avr_compiler.rs index f5c4ed58..ba7cf0cd 100644 --- a/crates/fbuild-build/src/avr/avr_compiler.rs +++ b/crates/fbuild-build/src/avr/avr_compiler.rs @@ -98,8 +98,9 @@ impl AvrCompiler { } } +#[async_trait::async_trait] impl Compiler for AvrCompiler { - fn compile_one( + async fn compile_one( &self, compiler_path: &Path, source: &Path, @@ -119,6 +120,7 @@ impl Compiler for AvrCompiler { None, &[], ) + .await } fn build_unflags(&self) -> &[String] { diff --git a/crates/fbuild-build/src/avr/avr_linker.rs b/crates/fbuild-build/src/avr/avr_linker.rs index c227a4b9..0901a8af 100644 --- a/crates/fbuild-build/src/avr/avr_linker.rs +++ b/crates/fbuild-build/src/avr/avr_linker.rs @@ -135,12 +135,13 @@ impl AvrLinker { } } +#[async_trait::async_trait] impl Linker for AvrLinker { - fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { - crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "avr-ar") + async fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { + crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "avr-ar").await } - fn link( + async fn link( &self, objects: &[PathBuf], archives: &[PathBuf], @@ -171,7 +172,8 @@ impl Linker for AvrLinker { &raw_objects, &framework_archive, "avr-gcc-ar", - )?; + ) + .await?; linker_archives.push(framework_archive); } linker_archives.extend(existing_archives); @@ -184,7 +186,7 @@ impl Linker for AvrLinker { } let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, None, None)?; + let result = run_command(&args_ref, None, None, None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -196,7 +198,7 @@ impl Linker for AvrLinker { Ok(elf_path) } - fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { + async fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { crate::linker::LinkerBase::objcopy_firmware( &self.objcopy_path, elf_path, @@ -205,6 +207,7 @@ impl Linker for AvrLinker { &self.mcu_config.objcopy.remove_sections, "avr-objcopy", ) + .await } fn size_tool_path(&self) -> &Path { @@ -223,7 +226,7 @@ impl Linker for AvrLinker { Some(&self.gcc_path) } - fn report_size(&self, elf_path: &Path) -> Result { + async fn report_size(&self, elf_path: &Path) -> Result { crate::linker::LinkerBase::report_size( &self.size_path, elf_path, @@ -231,6 +234,7 @@ impl Linker for AvrLinker { self.max_ram, "avr-size", ) + .await } } diff --git a/crates/fbuild-build/src/avr/mod.rs b/crates/fbuild-build/src/avr/mod.rs index edcf7138..1eda621a 100644 --- a/crates/fbuild-build/src/avr/mod.rs +++ b/crates/fbuild-build/src/avr/mod.rs @@ -1,4 +1,4 @@ -//! AVR platform build support (Arduino Uno, Mega, Nano, etc.) +//! AVR platform build support (Arduino Uno, Mega, Nano, etc.) pub mod avr_compiler; pub mod avr_linker; @@ -12,15 +12,16 @@ pub use orchestrator::AvrOrchestrator; /// AVR platform support (AtmelAvr + AtmelMegaAvr). pub struct AvrPlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for AvrPlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::AvrToolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("AVR toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/avr/orchestrator.rs b/crates/fbuild-build/src/avr/orchestrator.rs index ced117dc..e6554d31 100644 --- a/crates/fbuild-build/src/avr/orchestrator.rs +++ b/crates/fbuild-build/src/avr/orchestrator.rs @@ -1,4 +1,4 @@ -//! AVR build orchestrator — wires together config, packages, compiler, linker. +//! AVR build orchestrator — wires together config, packages, compiler, linker. //! //! Build phases: //! 1. Parse platformio.ini @@ -38,7 +38,7 @@ pub struct AvrOrchestrator; /// Any field that can change the produced firmware belongs here; /// a change flips the hash and forces a full rebuild. Keep this in /// sync with what [`AvrCompiler`] / [`AvrLinker`] actually read off -/// of `BoardConfig` — extra fields only cost a tiny amount of CPU, +/// of `BoardConfig` — extra fields only cost a tiny amount of CPU, /// but missing fields silently let stale artifacts get reused. #[derive(Debug, Serialize)] struct AvrFingerprintMetadata { @@ -67,17 +67,18 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +#[async_trait::async_trait] impl BuildOrchestrator for AvrOrchestrator { fn platform(&self) -> Platform { Platform::AtmelAvr } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); // Env-gated per-phase timer (FBUILD_PERF_LOG=1); zero-overhead when unset. let mut perf = crate::perf_log::PerfTimer::new("avr-orchestrator"); - // Wrapper-binary discovery removed in FastLED/fbuild#800 — every + // Wrapper-binary discovery removed in FastLED/fbuild#800 — every // compile dispatches through the embedded zccache service. The // `compiler_cache: Option` slot is retained as a dead // pass-through for the per-platform compiler API until a future @@ -88,7 +89,7 @@ impl BuildOrchestrator for AvrOrchestrator { // collect flags. `new_with_perf` records its own sub-phases // (config-parse, board-load, build-dirs, flag-collect) into // the shared `perf` timer. - let mut ctx = pipeline::BuildContext::new_with_perf(params, Some(&mut perf))?; + let mut ctx = pipeline::BuildContext::new_with_perf(params, Some(&mut perf)).await?; // Compute eh_frame strip policy once per build (FastLED/fbuild#244). // No sdkconfig on AVR. @@ -99,13 +100,14 @@ impl BuildOrchestrator for AvrOrchestrator { let (toolchain, toolchain_dir) = { let _g = perf.phase("toolchain-ensure"); let toolchain = fbuild_packages::toolchain::AvrToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; (toolchain, toolchain_dir) }; tracing::info!("avr-gcc toolchain at {}", toolchain_dir.display()); use fbuild_packages::Toolchain as _; - pipeline::log_toolchain_version(&toolchain.get_gcc_path(), "avr-gcc", &mut ctx.build_log); + pipeline::log_toolchain_version(&toolchain.get_gcc_path(), "avr-gcc", &mut ctx.build_log) + .await; // 4. Ensure Arduino core let (_framework_dir, core_dir, variant_dir) = { @@ -115,7 +117,8 @@ impl BuildOrchestrator for AvrOrchestrator { &ctx.board.core, &ctx.board.variant, ctx.board.platform(), - )? + ) + .await? }; // 4.5. Warm-build fast path. @@ -211,7 +214,7 @@ impl BuildOrchestrator for AvrOrchestrator { // 6. Build include dirs + compiler let defines = ctx.board.get_defines(); - // Use the resolved core_dir/variant_dir directly — board.get_include_paths() + // Use the resolved core_dir/variant_dir directly — board.get_include_paths() // uses the raw board core name which may differ from the actual directory // (e.g. MiniCore's core dir is "MCUdude_corefiles", not "MiniCore"). let mut include_dirs = vec![core_dir.clone(), variant_dir.clone()]; @@ -236,7 +239,7 @@ impl BuildOrchestrator for AvrOrchestrator { .with_build_unflags(ctx.build_unflags.clone()) .with_eh_frame_policy(eh_frame_policy); - // 7. Create linker — pass gcc-ar so framework .o inputs can be + // 7. Create linker — pass gcc-ar so framework .o inputs can be // archived into libframework.a with the LTO bytecode plugin index // intact (preserves `-fuse-linker-plugin`). See FastLED/fbuild#304. let linker = AvrLinker::new( @@ -286,11 +289,12 @@ impl BuildOrchestrator for AvrOrchestrator { TargetArchitecture::Avr, "AVR", start, - )?; + ) + .await?; // 10. Persist fingerprint so the next warm invocation can hit the // fast path. Skip this for compile-db-only / symbol-analysis runs - // — they don't produce the full artifact set the fast path + // — they don't produce the full artifact set the fast path // requires. if build_result.success && !params.compiledb_only @@ -325,7 +329,7 @@ pub fn create() -> Box { /// to `"arduino_megaavr"` so they get `ArduinoCore-megaavr` (which contains the /// megaAVR variants like `nona4809`) instead of `ArduinoCore-avr`. /// Returns (framework_root, core_dir, variant_dir). -fn ensure_avr_framework( +async fn ensure_avr_framework( project_dir: &Path, core_name: &str, variant_name: &str, @@ -343,7 +347,7 @@ fn ensure_avr_framework( }; let framework = fbuild_packages::library::AvrFramework::for_core(lookup_key, project_dir)?; - let framework_dir = framework.ensure_installed()?; + let framework_dir = framework.ensure_installed().await?; tracing::info!( "AVR framework for core '{}' (lookup '{}') at {}", core_name, diff --git a/crates/fbuild-build/src/ch32v/ch32v_compiler.rs b/crates/fbuild-build/src/ch32v/ch32v_compiler.rs index b0d4c512..5fa4d9a1 100644 --- a/crates/fbuild-build/src/ch32v/ch32v_compiler.rs +++ b/crates/fbuild-build/src/ch32v/ch32v_compiler.rs @@ -125,8 +125,9 @@ fn framework_suppression_flags() -> &'static [&'static str] { &["-w"] } +#[async_trait::async_trait] impl Compiler for Ch32vCompiler { - fn compile_one( + async fn compile_one( &self, compiler_path: &Path, source: &Path, @@ -166,6 +167,7 @@ impl Compiler for Ch32vCompiler { None, &[], ) + .await } fn gcc_path(&self) -> &Path { diff --git a/crates/fbuild-build/src/ch32v/ch32v_linker.rs b/crates/fbuild-build/src/ch32v/ch32v_linker.rs index 928bbec0..fb77f763 100644 --- a/crates/fbuild-build/src/ch32v/ch32v_linker.rs +++ b/crates/fbuild-build/src/ch32v/ch32v_linker.rs @@ -54,12 +54,14 @@ impl Ch32vLinker { } } +#[async_trait::async_trait] impl Linker for Ch32vLinker { - fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { + async fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "riscv-none-elf-ar") + .await } - fn link( + async fn link( &self, objects: &[PathBuf], archives: &[PathBuf], @@ -110,7 +112,7 @@ impl Linker for Ch32vLinker { } let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, None, None)?; + let result = run_command(&args_ref, None, None, None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -122,7 +124,7 @@ impl Linker for Ch32vLinker { Ok(elf_path) } - fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { + async fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { crate::linker::LinkerBase::objcopy_firmware( &self.objcopy_path, elf_path, @@ -131,6 +133,7 @@ impl Linker for Ch32vLinker { &self.mcu_config.objcopy.remove_sections, "riscv-none-elf-objcopy", ) + .await } fn size_tool_path(&self) -> &Path { @@ -149,7 +152,7 @@ impl Linker for Ch32vLinker { Some(&self.gcc_path) } - fn report_size(&self, elf_path: &Path) -> Result { + async fn report_size(&self, elf_path: &Path) -> Result { crate::linker::LinkerBase::report_size( &self.size_path, elf_path, @@ -157,6 +160,7 @@ impl Linker for Ch32vLinker { self.max_ram, "riscv-none-elf-size", ) + .await } } diff --git a/crates/fbuild-build/src/ch32v/mod.rs b/crates/fbuild-build/src/ch32v/mod.rs index e5db5994..1954e335 100644 --- a/crates/fbuild-build/src/ch32v/mod.rs +++ b/crates/fbuild-build/src/ch32v/mod.rs @@ -1,4 +1,4 @@ -//! CH32V RISC-V platform build support (WCH CH32V003, CH32V203, etc.) +//! CH32V RISC-V platform build support (WCH CH32V003, CH32V203, etc.) pub mod ch32v_compiler; pub mod ch32v_linker; @@ -12,15 +12,16 @@ pub use orchestrator::Ch32vOrchestrator; /// CH32V platform support. pub struct Ch32vPlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for Ch32vPlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::RiscvToolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("RISC-V toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/ch32v/orchestrator.rs b/crates/fbuild-build/src/ch32v/orchestrator.rs index 2300401c..0b159424 100644 --- a/crates/fbuild-build/src/ch32v/orchestrator.rs +++ b/crates/fbuild-build/src/ch32v/orchestrator.rs @@ -1,4 +1,4 @@ -//! CH32V build orchestrator — wires together config, packages, compiler, linker. +//! CH32V build orchestrator — wires together config, packages, compiler, linker. //! //! Build phases: //! 1. Parse platformio.ini @@ -27,20 +27,21 @@ use super::ch32v_linker::Ch32vLinker; /// CH32V platform build orchestrator. pub struct Ch32vOrchestrator; +#[async_trait::async_trait] impl BuildOrchestrator for Ch32vOrchestrator { fn platform(&self) -> Platform { Platform::Ch32v } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = pipeline::BuildContext::new(params)?; + let mut ctx = pipeline::BuildContext::new(params).await?; // 3. Ensure RISC-V GCC toolchain let toolchain = fbuild_packages::toolchain::RiscvToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("riscv-gcc toolchain at {}", toolchain_dir.display()); use fbuild_packages::Toolchain; @@ -48,11 +49,12 @@ impl BuildOrchestrator for Ch32vOrchestrator { &toolchain.get_gcc_path(), "riscv-none-elf-gcc", &mut ctx.build_log, - ); + ) + .await; // 4. Ensure OpenWCH CH32V cores let framework = fbuild_packages::library::Ch32vCores::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("CH32V cores at {}", framework_dir.display()); // 5. Resolve series/variant selection used by both scanning and compile flags. @@ -97,14 +99,14 @@ impl BuildOrchestrator for Ch32vOrchestrator { let mut defines = ctx.board.get_defines(); defines.extend(mcu_config.defines_map()); defines.insert(system_series.clone(), "1".to_string()); - // CH32V cores use `#include VARIANT_H` — define it from the variant dir + // CH32V cores use `#include VARIANT_H` — define it from the variant dir if let Some(vh) = resolve_variant_h(&variant_dir, ctx.board.variant_h.as_deref()) { defines.insert("VARIANT_H".to_string(), format!("\\\"{}\\\"", vh)); } if series == "ch32x035" { defines.insert("RCC_BackupResetCmd(x)".to_string(), "((void)0)".to_string()); } - // Use resolved core_dir/variant_dir directly — board.get_include_paths() + // Use resolved core_dir/variant_dir directly — board.get_include_paths() // uses the raw board core name which may differ from the actual directory // (e.g. OpenWCH core dir is "arduino", not "openwch"). let mut include_dirs = vec![core_dir.clone(), variant_dir.clone()]; @@ -210,6 +212,7 @@ impl BuildOrchestrator for Ch32vOrchestrator { "CH32V", start, ) + .await } } diff --git a/crates/fbuild-build/src/compile_many.rs b/crates/fbuild-build/src/compile_many.rs index 57cbc0cf..d0798a57 100644 --- a/crates/fbuild-build/src/compile_many.rs +++ b/crates/fbuild-build/src/compile_many.rs @@ -1,4 +1,4 @@ -//! Two-stage `compile-many` primitive (FastLED/fbuild#238). +//! Two-stage `compile-many` primitive (FastLED/fbuild#238). //! //! Compiles a list of sketches against the same board with the framework + //! library archives built **once**, then fans out per-sketch compile + link @@ -41,7 +41,7 @@ //! writes of distinct keys never block. fbuild itself adds no //! in-process locks around zccache (see `crates/fbuild-build/src/zccache.rs`), //! so stage-2 contention is bounded by the zccache daemon's own -//! concurrency model — well below the parallelism cap we set. +//! concurrency model — well below the parallelism cap we set. //! //! ## Routing through the existing orchestrator //! @@ -53,8 +53,6 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Mutex; use std::time::Instant; use fbuild_core::{BuildProfile, FbuildError, Platform, Result}; @@ -90,7 +88,7 @@ pub fn default_sketch_jobs() -> usize { /// Routes through [`fbuild_paths::BuildLayout`] so layout decisions /// (env-segment auto-collapse, `FBUILD_BUILD_DIR` override) match /// what the daemon and the per-platform orchestrators use. Centralized -/// here so the stage-1→stage-2 cache-seeding code can address that path +/// here so the stage-1→stage-2 cache-seeding code can address that path /// without re-deriving the same string in three places. See /// FastLED/fbuild#335. pub fn project_build_dir(sketch: &Path, env: &str, profile: BuildProfile) -> PathBuf { @@ -104,18 +102,18 @@ pub fn project_build_dir(sketch: &Path, env: &str, profile: BuildProfile) -> Pat /// /// Without this, every stage-2 project dir gets its own empty /// `.fbuild/build///core/`, so the framework is rebuilt -/// from scratch in every worker — the 25s-per-sketch path FastLED hit on +/// from scratch in every worker — the 25s-per-sketch path FastLED hit on /// its teensy41 / esp32s3 runs (FastLED/fbuild#335). /// /// Uses hardlinks to keep the seed near-free; falls back to a byte copy /// if hardlinking fails (cross-filesystem, target FS lacks hardlink -/// support, etc.). Both modes preserve the .o mtime — important because +/// support, etc.). Both modes preserve the .o mtime — important because /// `needs_rebuild` consults dep-file mtimes against the .o mtime. /// /// Idempotent: skips files that already exist at the target. A pre- /// existing stage-2 partial build won't get clobbered. /// -/// Errors are non-fatal and tracked by the caller — on any failure the +/// Errors are non-fatal and tracked by the caller — on any failure the /// orchestrator simply falls back to its full framework-recompile path /// (no worse than the pre-#335 behavior). fn seed_stage2_core_from_stage1(stage1_core: &Path, stage2_core: &Path) -> std::io::Result<()> { @@ -142,7 +140,7 @@ fn seed_stage2_core_from_stage1(stage1_core: &Path, stage2_core: &Path) -> std:: if std::fs::hard_link(&src, &dst).is_ok() { n_linked += 1; } else { - // Hardlink failed (cross-fs, NTFS junction, etc.) — fall + // Hardlink failed (cross-fs, NTFS junction, etc.) — fall // back to a byte copy. `copy` preserves nothing about the // source mtime by default; we explicitly set it from the // stage-1 metadata so depfile freshness comparison still @@ -177,14 +175,14 @@ fn seed_stage2_core_from_stage1(stage1_core: &Path, stage2_core: &Path) -> std:: /// single TU (sketch.cpp) against a pre-built framework archive. In /// practice consumers stage each sketch in its own project dir (so two /// sketches with different `.ino` content can build in parallel), and -/// each project dir has its own `.fbuild/build///` — which +/// each project dir has its own `.fbuild/build///` — which /// means stage 2 rebuilds the framework from scratch per sketch. With /// `jobs=1` that framework rebuild is serial inside each worker, and the /// per-sketch wall time becomes "sum of framework TU times" instead of /// "max of framework TU times". See FastLED/fbuild#335. /// /// Splitting cores across workers keeps total in-flight compile slots -/// at roughly `cores` so we don't oversubscribe small runners — each +/// at roughly `cores` so we don't oversubscribe small runners — each /// worker gets `max(1, cores / sketch_jobs)`. pub fn stage2_jobs_per_worker(sketch_jobs: usize) -> usize { let cores = std::thread::available_parallelism() @@ -302,7 +300,7 @@ impl CompileManyResult { /// 1. An environment literally named ``. /// 2. The first environment whose `board = `. /// -/// Returns `Err` when neither is found — this is a contract violation that +/// Returns `Err` when neither is found — this is a contract violation that /// should surface immediately rather than guessing. fn resolve_env_for_board(project_dir: &Path, board: &str) -> Result { let ini_path = project_dir.join("platformio.ini"); @@ -341,7 +339,7 @@ fn platform_for_board(board: &str, project_dir: Option<&std::path::Path>) -> Res /// /// `jobs` controls intra-build parallelism (passed through to the /// orchestrator's per-build thread pool). -fn build_one_sketch(inputs: SketchBuildInputs) -> SketchResult { +async fn build_one_sketch(inputs: SketchBuildInputs) -> SketchResult { let SketchBuildInputs { sketch, env_name, @@ -376,7 +374,7 @@ fn build_one_sketch(inputs: SketchBuildInputs) -> SketchResult { }; let outcome = match get_orchestrator(platform) { - Ok(orch) => orch.build(¶ms), + Ok(orch) => orch.build(¶ms).await, Err(e) => Err(e), }; @@ -462,8 +460,12 @@ pub struct SketchBuildInputs { /// Trait used by [`compile_many_with`] to run a single sketch. Tests /// inject a mock implementation that records stage / concurrency / output /// path uniqueness without dragging in a real toolchain. -pub trait SketchBuilder: Sync { - fn build(&self, inputs: SketchBuildInputs) -> SketchResult; +/// +/// FastLED/fbuild#820 (Phase B of #813): `build` is `async` so per-sketch +/// dispatch can `.await` the platform orchestrator's async build trait. +#[async_trait::async_trait] +pub trait SketchBuilder: Sync + Send { + async fn build(&self, inputs: SketchBuildInputs) -> SketchResult; } /// Production [`SketchBuilder`] that drives the real platform @@ -471,27 +473,28 @@ pub trait SketchBuilder: Sync { /// from `compile_many`, so tests can swap it out wholesale. pub struct OrchestratorBuilder; +#[async_trait::async_trait] impl SketchBuilder for OrchestratorBuilder { - fn build(&self, inputs: SketchBuildInputs) -> SketchResult { - build_one_sketch(inputs) + async fn build(&self, inputs: SketchBuildInputs) -> SketchResult { + build_one_sketch(inputs).await } } /// Run the two-stage `compile-many` flow. /// /// Returns once every sketch has been attempted. Individual sketch failures -/// do not short-circuit subsequent sketches — the caller inspects +/// do not short-circuit subsequent sketches — the caller inspects /// [`CompileManyResult::all_success`] / [`CompileManyResult::results`]. -pub fn compile_many(req: CompileManyRequest) -> Result { - compile_many_with(req, &OrchestratorBuilder) +pub async fn compile_many(req: CompileManyRequest) -> Result { + compile_many_with(req, &OrchestratorBuilder).await } /// Like [`compile_many`] but parameterized over the [`SketchBuilder`] /// used to actually build each sketch. Public for tests; production /// callers should use [`compile_many`]. -pub fn compile_many_with( +pub async fn compile_many_with( req: CompileManyRequest, - builder: &dyn SketchBuilder, + builder: &(dyn SketchBuilder + Send + Sync), ) -> Result { if req.sketches.is_empty() { return Err(FbuildError::Other( @@ -539,22 +542,24 @@ pub fn compile_many_with( let (first_sketch, first_env) = resolved[0].clone(); // Remember stage 1's resolved (sketch_dir, env) so stage 2 can find the // pre-built framework artifacts and seed each worker's `core/` from - // them — see `seed_stage2_core_from_stage1` and FastLED/fbuild#335. + // them — see `seed_stage2_core_from_stage1` and FastLED/fbuild#335. let stage1_sketch = first_sketch.clone(); let stage1_env = first_env.clone(); - let first_result = builder.build(SketchBuildInputs { - sketch: first_sketch, - env_name: first_env, - platform, - profile: req.profile, - jobs: framework_jobs, - verbose: req.verbose, - stage: Stage::Stage1Framework, - pio_env: req.pio_env.clone(), - }); + let first_result = builder + .build(SketchBuildInputs { + sketch: first_sketch, + env_name: first_env, + platform, + profile: req.profile, + jobs: framework_jobs, + verbose: req.verbose, + stage: Stage::Stage1Framework, + pio_env: req.pio_env.clone(), + }) + .await; let stage1_secs = stage1_start.elapsed().as_secs_f64(); - // If stage 1 failed there is no point fanning out — every stage-2 + // If stage 1 failed there is no point fanning out — every stage-2 // worker would re-run the framework build (which we just proved // broken) and report the same error. Return what we have so far. if !first_result.success { @@ -576,7 +581,7 @@ pub fn compile_many_with( // Stage-1 has finished and (on success) written its framework `core/` // artifacts to disk. Compute that path once so every stage-2 worker // can hardlink them into its own per-sketch `core/` and skip the - // framework recompile entirely — FastLED/fbuild#335. + // framework recompile entirely — FastLED/fbuild#335. let stage1_core_seed: Option = if first_result.success { Some(project_build_dir(&stage1_sketch, &stage1_env, req.profile).join("core")) } else { @@ -597,6 +602,7 @@ pub fn compile_many_with( &stage1_env, req.diag_stage2, ) + .await }; let stage2_secs = stage2_start.elapsed().as_secs_f64(); @@ -629,13 +635,13 @@ pub fn compile_many_with( /// (FastLED/fbuild#335). Pass `None` to disable seeding (e.g. stage 1 /// failed). #[allow(clippy::too_many_arguments)] -fn run_stage2( +async fn run_stage2( rest: &[(PathBuf, String)], platform: Platform, profile: BuildProfile, sketch_jobs: usize, verbose: bool, - builder: &dyn SketchBuilder, + builder: &(dyn SketchBuilder + Send + Sync), pio_env: &HashMap, stage1_core_seed: Option<&Path>, stage1_env: &str, @@ -649,95 +655,96 @@ fn run_stage2( cap ); - // Work queue: indices into `rest`. We dispatch by index so the result - // slot lives at a stable position regardless of completion order. - let next = AtomicUsize::new(0); - let mut results: Vec> = (0..total).map(|_| None).collect(); - let results_slot: Vec>> = - results.iter_mut().map(|_| Mutex::new(None)).collect(); - - // Split available cores across stage-2 workers so each worker has - // real compile parallelism. See `stage2_jobs_per_worker` for why the - // old `jobs=1` hardcoding was a 2-3x regression vs the single-build - // path on cold cache — FastLED/fbuild#335. + // FastLED/fbuild#820 (Phase B of #813): replaces the old + // `std::thread::scope` worker-pool with a `tokio::task::JoinSet` + // gated by a semaphore. Each per-sketch task `.await`s the async + // `SketchBuilder::build`, so the orchestrator's per-sketch + // compile / link / size pipeline runs cooperatively on the + // daemon's tokio runtime instead of stealing OS threads. let jobs_per_worker = stage2_jobs_per_worker(cap); - - std::thread::scope(|scope| { - let handles: Vec<_> = (0..cap) - .map(|worker_index| { - let next = &next; - let results_slot = &results_slot; - scope.spawn(move || loop { - let idx = next.fetch_add(1, Ordering::Relaxed); - if idx >= rest.len() { - break; - } - let entry = &rest[idx]; - let (sketch, env_name) = (&entry.0, &entry.1); - // Seed framework `core/` from stage 1 when the env - // matches. Different envs may bake different flags - // into framework objects, so we only reuse when the - // env name is identical (the common case for a single - // board across many FastLED examples). Errors are - // logged and ignored — orchestrator falls back to a - // full framework recompile, no worse than pre-#335. - let seed_started = Instant::now(); - let mut seed_applied = false; - if let Some(seed) = stage1_core_seed { - if env_name == stage1_env { - let target_core = - project_build_dir(sketch, env_name, profile).join("core"); - seed_applied = seed.is_dir(); - if let Err(e) = seed_stage2_core_from_stage1(seed, &target_core) { - tracing::warn!( - "compile-many stage 2: failed to seed core/ \ - for {}: {} — falling back to full framework \ - recompile", - sketch.display(), - e - ); - } - } - } - let seed_time_secs = seed_started.elapsed().as_secs_f64(); - let mut res = builder.build(SketchBuildInputs { - sketch: sketch.clone(), - env_name: env_name.clone(), - platform, - profile, - jobs: jobs_per_worker, - verbose, - stage: Stage::Stage2Sketch, - pio_env: pio_env.clone(), - }); - res.worker_index = Some(worker_index); - res.seed_time_secs = seed_time_secs; - res.seed_applied = seed_applied; - if diag_stage2 { - tracing::info!( - "compile-many stage2 diag worker={} index={} sketch={} env={} seed_applied={} seed_secs={:.6} build_secs={:.6} success={}", - worker_index, - idx, + let semaphore = std::sync::Arc::new(tokio::sync::Semaphore::new(cap)); + let mut joinset: tokio::task::JoinSet<(usize, SketchResult)> = tokio::task::JoinSet::new(); + + // SAFETY: the JoinSet is drained before this function returns, so the + // borrows below stay alive for the duration of every spawned task. + let builder_ptr: &'static (dyn SketchBuilder + Send + Sync) = + unsafe { std::mem::transmute(builder) }; + let stage1_env_owned = stage1_env.to_string(); + let stage1_core_seed_owned: Option = stage1_core_seed.map(|p| p.to_path_buf()); + + for (idx, entry) in rest.iter().enumerate() { + let sketch = entry.0.clone(); + let env_name = entry.1.clone(); + let pio_env = pio_env.clone(); + let sem = semaphore.clone(); + let seed = stage1_core_seed_owned.clone(); + let stage1_env_cloned = stage1_env_owned.clone(); + let worker_index = idx % cap; + joinset.spawn(async move { + let _permit = sem.acquire().await.expect("compile-many semaphore closed"); + let seed_started = Instant::now(); + let mut seed_applied = false; + if let Some(seed_path) = seed.as_deref() { + if env_name == stage1_env_cloned { + let target_core = + project_build_dir(&sketch, &env_name, profile).join("core"); + seed_applied = seed_path.is_dir(); + if let Err(e) = seed_stage2_core_from_stage1(seed_path, &target_core) { + tracing::warn!( + "compile-many stage 2: failed to seed core/ \ + for {}: {} — falling back to full framework \ + recompile", sketch.display(), - env_name, - seed_applied, - seed_time_secs, - res.build_time_secs, - res.success + e ); } - *results_slot[idx].lock().unwrap() = Some(res); + } + } + let seed_time_secs = seed_started.elapsed().as_secs_f64(); + let mut res = builder_ptr + .build(SketchBuildInputs { + sketch: sketch.clone(), + env_name: env_name.clone(), + platform, + profile, + jobs: jobs_per_worker, + verbose, + stage: Stage::Stage2Sketch, + pio_env, }) - }) - .collect(); - for h in handles { - let _ = h.join(); - } - }); + .await; + res.worker_index = Some(worker_index); + res.seed_time_secs = seed_time_secs; + res.seed_applied = seed_applied; + if diag_stage2 { + tracing::info!( + "compile-many stage2 diag worker={} index={} sketch={} env={} seed_applied={} seed_secs={:.6} build_secs={:.6} success={}", + worker_index, + idx, + sketch.display(), + env_name, + seed_applied, + seed_time_secs, + res.build_time_secs, + res.success + ); + } + (idx, res) + }); + } - results_slot + let mut results: Vec> = (0..total).map(|_| None).collect(); + while let Some(joined) = joinset.join_next().await { + match joined { + Ok((idx, res)) => results[idx] = Some(res), + Err(e) => { + tracing::error!("compile-many stage 2 worker join error: {e}"); + } + } + } + results .into_iter() - .map(|slot| slot.into_inner().unwrap().expect("worker filled slot")) + .map(|slot| slot.expect("stage-2 worker filled every slot")) .collect() } @@ -772,7 +779,7 @@ mod tests { let absent_source = tmp.path().join("missing"); let target = tmp.path().join("target"); assert!(seed_stage2_core_from_stage1(&absent_source, &target).is_ok()); - // Must not create the target dir for an absent source — otherwise + // Must not create the target dir for an absent source — otherwise // we'd leave litter on disk for envs that don't match. assert!(!target.exists()); } @@ -828,7 +835,7 @@ mod tests { // Locks the on-disk convention the AVR/ESP32/etc. orchestrators // all derive their per-(env,profile) build root from. Any change // here that doesn't also update the orchestrators will silently - // break the stage-1→stage-2 core/ handoff in FastLED/fbuild#335. + // break the stage-1→stage-2 core/ handoff in FastLED/fbuild#335. let p = project_build_dir(Path::new("/tmp/sketch"), "uno", BuildProfile::Release); assert!(p.ends_with("sketch/.fbuild/build/uno/release")); let q = project_build_dir(Path::new("/tmp/sketch"), "esp32s3", BuildProfile::Quick); @@ -895,8 +902,8 @@ mod tests { assert_eq!(p, Platform::AtmelAvr); } - #[test] - fn empty_sketch_list_errors_out() { + #[tokio::test] + async fn empty_sketch_list_errors_out() { let req = CompileManyRequest { board: "uno".to_string(), sketches: Vec::new(), @@ -907,11 +914,11 @@ mod tests { pio_env: HashMap::new(), diag_stage2: false, }; - assert!(compile_many(req).is_err()); + assert!(compile_many(req).await.is_err()); } - #[test] - fn missing_sketch_dir_errors_out() { + #[tokio::test] + async fn missing_sketch_dir_errors_out() { let tmp = tempfile::tempdir().unwrap(); let missing = tmp.path().join("nope"); let req = CompileManyRequest { @@ -924,11 +931,11 @@ mod tests { pio_env: HashMap::new(), diag_stage2: false, }; - assert!(compile_many(req).is_err()); + assert!(compile_many(req).await.is_err()); } - #[test] - fn sketch_without_matching_board_errors_out() { + #[tokio::test] + async fn sketch_without_matching_board_errors_out() { let tmp = tempfile::tempdir().unwrap(); std::fs::write( tmp.path().join("platformio.ini"), @@ -945,6 +952,6 @@ mod tests { pio_env: HashMap::new(), diag_stage2: false, }; - assert!(compile_many(req).is_err()); + assert!(compile_many(req).await.is_err()); } } diff --git a/crates/fbuild-build/src/compiler.rs b/crates/fbuild-build/src/compiler.rs index aac02aea..c2e72144 100644 --- a/crates/fbuild-build/src/compiler.rs +++ b/crates/fbuild-build/src/compiler.rs @@ -63,12 +63,20 @@ pub struct CompileResult { static COMPILER_IDENTITY_CACHE: OnceLock>> = OnceLock::new(); /// Trait for platform-specific compilers. +/// +/// FastLED/fbuild#820 (Phase B of #813): `compile_one` is `async` so +/// per-TU zccache dispatch can `.await` `ZccacheService::compile` +/// directly, with no `Handle::block_on`. The default-method bodies +/// (`compile_c` / `compile_cpp` / `compile`) are `async fn` too so +/// they propagate the `.await` to platform-specific `compile_one` +/// impls. +#[async_trait::async_trait] pub trait Compiler: Send + Sync { /// Platform-specific compilation dispatch. /// /// Routes to `compile_source()` with platform-specific parameters /// (temp dir, response file prefix, compiler cache, extra pre-flags). - fn compile_one( + async fn compile_one( &self, compiler_path: &Path, source: &Path, @@ -91,7 +99,7 @@ pub trait Compiler: Send + Sync { } /// Compile a C source file to an object file. - fn compile_c( + async fn compile_c( &self, source: &Path, output: &Path, @@ -100,10 +108,11 @@ pub trait Compiler: Send + Sync { let flags = self.c_flags(); let (flags, extra) = apply_compile_unflags(flags, extra_flags, self.build_unflags()); self.compile_one(self.gcc_path(), source, output, &flags, &extra) + .await } /// Compile a C++ source file to an object file. - fn compile_cpp( + async fn compile_cpp( &self, source: &Path, output: &Path, @@ -112,10 +121,11 @@ pub trait Compiler: Send + Sync { let flags = self.cpp_flags(); let (flags, extra) = apply_compile_unflags(flags, extra_flags, self.build_unflags()); self.compile_one(self.gxx_path(), source, output, &flags, &extra) + .await } /// Compile a source file (auto-detect C vs C++). - fn compile( + async fn compile( &self, source: &Path, output: &Path, @@ -127,8 +137,8 @@ pub trait Compiler: Send + Sync { .to_string_lossy() .to_lowercase(); match ext.as_str() { - "c" | "s" => self.compile_c(source, output, extra_flags), - _ => self.compile_cpp(source, output, extra_flags), + "c" | "s" => self.compile_c(source, output, extra_flags).await, + _ => self.compile_cpp(source, output, extra_flags).await, } } @@ -433,9 +443,32 @@ fn compiler_identity(path: &Path) -> String { } fn compiler_version(path: &Path) -> String { - let program = path.to_string_lossy(); - let args = [program.as_ref(), "-dumpversion"]; - match fbuild_core::subprocess::run_command(&args, None, None, None) { + // FastLED/fbuild#820 (Phase B of #813): `fbuild_core::subprocess:: + // run_command` is now `async`. `compiler_version` is called from + // the sync `rebuild_signature` trait method (which is in turn + // called from sync rebuild-check code paths), so we bridge to the + // ambient tokio runtime via `block_in_place` + `block_on`. This is + // safe because the daemon runs on a multi-thread tokio runtime and + // `block_in_place` permits this exact pattern. + let program = path.to_string_lossy().to_string(); + let result = match tokio::runtime::Handle::try_current() { + Ok(handle) => tokio::task::block_in_place(|| { + handle.block_on(async { + let args = [program.as_str(), "-dumpversion"]; + fbuild_core::subprocess::run_command(&args, None, None, None).await + }) + }), + Err(_) => { + // No ambient runtime — happens in unit-test contexts that + // don't spin up a tokio runtime. Returning an empty version + // is a graceful degradation: rebuild-signature loses the + // compiler-version contribution but still encodes path + + // flags, which is enough for the tests that don't touch a + // real toolchain. + return String::new(); + } + }; + match result { Ok(output) if output.success() => output.stdout.trim().to_string(), _ => String::new(), } @@ -495,8 +528,12 @@ pub fn windows_temp_dir() -> PathBuf { /// Write flags to a temporary GCC response file (`@file` syntax). /// /// Delegates to [`fbuild_core::response_file::write_response_file`]. -pub fn write_response_file(flags: &[String], temp_dir: &Path, prefix: &str) -> Result { - fbuild_core::response_file::write_response_file(flags, temp_dir, prefix) +pub async fn write_response_file( + flags: &[String], + temp_dir: &Path, + prefix: &str, +) -> Result { + fbuild_core::response_file::write_response_file(flags, temp_dir, prefix).await } /// Response-file directory for a given output object. Used by the @@ -570,7 +607,7 @@ pub fn build_cpp_flags(common_flags: Vec, config: &dyn McuConfig) -> Vec /// and `extra_flags` (ESP32 uses this for include flags deferred /// from `common_flags`). #[allow(clippy::too_many_arguments)] -pub fn compile_source( +pub async fn compile_source( compiler: &Path, source: &Path, output: &Path, @@ -640,7 +677,6 @@ pub fn compile_source( ) })?; let svc = global.service(); - let runtime = global.runtime(); let cwd = compile_cwd .clone() .or_else(|| std::env::current_dir().ok()) @@ -662,8 +698,14 @@ pub fn compile_source( ); } + // FastLED/fbuild#820 (Phase B of #813): direct `.await` on the + // async zccache service. The legacy `compile_blocking` path is gone + // — every per-TU compile is reached through the async build trait + // chain, so the daemon's tokio runtime drives `ZccacheService:: + // compile` natively. let outcome = svc - .compile_blocking(runtime, compiler, sanitized, cwd, Vec::new()) + .compile(compiler, sanitized, cwd, Vec::new()) + .await .map_err(|err| { fbuild_core::FbuildError::BuildFailed(format!( "embedded zccache compile failed for {}: {err}", diff --git a/crates/fbuild-build/src/compiler_tests.rs b/crates/fbuild-build/src/compiler_tests.rs index 8ff7a99f..87a63343 100644 --- a/crates/fbuild-build/src/compiler_tests.rs +++ b/crates/fbuild-build/src/compiler_tests.rs @@ -199,9 +199,11 @@ fn test_prepare_flags_and_response_file_produce_same_define_value() { let exec_result = prepare_flags_for_exec(vec![input.clone()]); assert_eq!(exec_result[0], r#"-DFOO="bar""#); - // Response file path + // Response file path — sync test uses the blocking bridge. let tmp = tempfile::TempDir::new().unwrap(); - let rsp = write_response_file(&[input], tmp.path(), "test").unwrap(); + let rsp = + fbuild_core::response_file::write_response_file_blocking(&[input], tmp.path(), "test") + .unwrap(); let content = std::fs::read_to_string(rsp).unwrap(); // Response file wraps in single quotes with unescaped " assert_eq!(content, r#"'-DFOO="bar"'"#); @@ -216,7 +218,9 @@ fn test_response_file_preserves_bare_quoted_define_value() { let input = r#"-DARDUINO_BSP_VERSION="1.6.1""#.to_string(); let tmp = tempfile::TempDir::new().unwrap(); - let rsp = write_response_file(&[input], tmp.path(), "test").unwrap(); + let rsp = + fbuild_core::response_file::write_response_file_blocking(&[input], tmp.path(), "test") + .unwrap(); let content = std::fs::read_to_string(rsp).unwrap(); assert_eq!(content, r#"'-DARDUINO_BSP_VERSION="1.6.1"'"#); } diff --git a/crates/fbuild-build/src/esp32/esp32_compiler.rs b/crates/fbuild-build/src/esp32/esp32_compiler.rs index 002f2bf9..70f1e039 100644 --- a/crates/fbuild-build/src/esp32/esp32_compiler.rs +++ b/crates/fbuild-build/src/esp32/esp32_compiler.rs @@ -150,8 +150,9 @@ impl Esp32Compiler { } } +#[async_trait::async_trait] impl Compiler for Esp32Compiler { - fn compile_one( + async fn compile_one( &self, compiler_path: &Path, source: &Path, @@ -172,6 +173,7 @@ impl Compiler for Esp32Compiler { self.compiler_cache.as_deref(), &include_flags, ) + .await } fn build_unflags(&self) -> &[String] { @@ -308,13 +310,15 @@ mod tests { assert!(include_flags.iter().any(|f: &String| f.contains("-I"))); } - #[test] - fn test_response_file_generation() { + #[tokio::test] + async fn test_response_file_generation() { let tmp = tempfile::TempDir::new().unwrap(); let flags: Vec = (0..200) .map(|i| format!("-I/path/to/include/{}", i)) .collect(); - let path = crate::compiler::write_response_file(&flags, tmp.path(), "esp32").unwrap(); + let path = crate::compiler::write_response_file(&flags, tmp.path(), "esp32") + .await + .unwrap(); assert!(path.exists()); let content = std::fs::read_to_string(&path).unwrap(); assert!(content.contains("-I/path/to/include/0")); diff --git a/crates/fbuild-build/src/esp32/esp32_linker.rs b/crates/fbuild-build/src/esp32/esp32_linker.rs index 70398725..65b59cfc 100644 --- a/crates/fbuild-build/src/esp32/esp32_linker.rs +++ b/crates/fbuild-build/src/esp32/esp32_linker.rs @@ -291,12 +291,13 @@ impl Esp32Linker { } } +#[async_trait::async_trait] impl Linker for Esp32Linker { - fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { - crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "ar") + async fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { + crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "ar").await } - fn link( + async fn link( &self, objects: &[PathBuf], archives: &[PathBuf], @@ -320,12 +321,13 @@ impl Linker for Esp32Linker { &flags_for_rsp, &rsp_dir, "esp32_link", - )?; + ) + .await?; let rsp_args = [link_args[0].as_str(), &format!("@{}", rsp_path.display())]; - run_command(&rsp_args, None, None, None)? + run_command(&rsp_args, None, None, None).await? } else { let args_ref: Vec<&str> = link_args.iter().map(|s| s.as_str()).collect(); - run_command(&args_ref, None, None, None)? + run_command(&args_ref, None, None, None).await? }; if !result.success() { @@ -338,7 +340,7 @@ impl Linker for Esp32Linker { Ok(elf_path) } - fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { + async fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { // Copy ELF to output directory let elf_out = output_dir.join("firmware.elf"); if elf_path != elf_out { @@ -378,7 +380,7 @@ impl Linker for Esp32Linker { tracing::info!("elf2image: {}", args.join(" ")); - match run_command(&args, None, None, Some(std::time::Duration::from_secs(30))) { + match run_command(&args, None, None, Some(std::time::Duration::from_secs(30))).await { Ok(result) if result.success() => { let cache = self.current_bin_cache(&elf_out, &flash_size)?; if let Err(e) = save_json(&self.bin_cache_path(output_dir), &cache) { @@ -415,7 +417,7 @@ impl Linker for Esp32Linker { Some(&self.gcc_path) } - fn report_size(&self, elf_path: &Path) -> Result { + async fn report_size(&self, elf_path: &Path) -> Result { if let Some(size_info) = self.load_cached_size(elf_path) { tracing::info!("size: firmware.elf is unchanged, reusing cached size report"); return Ok(size_info); @@ -427,7 +429,8 @@ impl Linker for Esp32Linker { self.max_flash, self.max_ram, "size", - )?; + ) + .await?; self.save_size_cache(elf_path, &size_info); Ok(size_info) } diff --git a/crates/fbuild-build/src/esp32/mod.rs b/crates/fbuild-build/src/esp32/mod.rs index 78cf6032..cbcd228b 100644 --- a/crates/fbuild-build/src/esp32/mod.rs +++ b/crates/fbuild-build/src/esp32/mod.rs @@ -1,4 +1,4 @@ -//! ESP32 platform build support (all variants: ESP32, C2, C3, C5, C6, P4, S3) +//! ESP32 platform build support (all variants: ESP32, C2, C3, C5, C6, P4, S3) pub mod esp32_compiler; pub mod esp32_linker; @@ -13,19 +13,20 @@ pub use orchestrator::Esp32Orchestrator; /// ESP32 platform support. pub struct Esp32PlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for Esp32PlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::esp32::Esp32Toolchain::new( project_dir, false, "xtensa-esp-elf", ); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("ESP32 toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/esp32/orchestrator/boot_artifacts.rs b/crates/fbuild-build/src/esp32/orchestrator/boot_artifacts.rs index 47d78090..2203438b 100644 --- a/crates/fbuild-build/src/esp32/orchestrator/boot_artifacts.rs +++ b/crates/fbuild-build/src/esp32/orchestrator/boot_artifacts.rs @@ -10,7 +10,7 @@ use super::super::mcu_config::Esp32McuConfig; /// Stage boot/partition/boot_app0 binaries into `build_dir`. Logs warnings on /// missing inputs but does not error — the linker output remains usable for /// in-tree flows even when the boot artifacts cannot be produced. -pub(super) fn prepare_boot_artifacts( +pub(super) async fn prepare_boot_artifacts( build_dir: &Path, framework: &fbuild_packages::library::Esp32Framework, board: &fbuild_config::BoardConfig, @@ -92,7 +92,9 @@ pub(super) fn prepare_boot_artifacts( None, None, Some(std::time::Duration::from_secs(30)), - ) { + ) + .await + { Ok(result) if result.success() => { tracing::info!("converted bootloader ELF → bootloader.bin"); } @@ -143,7 +145,9 @@ pub(super) fn prepare_boot_artifacts( None, None, Some(std::time::Duration::from_secs(10)), - ) { + ) + .await + { Ok(result) if result.success() => { tracing::info!("generated partitions.bin from {}", partitions_name); } diff --git a/crates/fbuild-build/src/esp32/orchestrator/build.rs b/crates/fbuild-build/src/esp32/orchestrator/build.rs index 03866ba0..905150c5 100644 --- a/crates/fbuild-build/src/esp32/orchestrator/build.rs +++ b/crates/fbuild-build/src/esp32/orchestrator/build.rs @@ -1,4 +1,4 @@ -//! `impl BuildOrchestrator for Esp32Orchestrator` — the high-level build flow. +//! `impl BuildOrchestrator for Esp32Orchestrator` — the high-level build flow. //! //! Most heavy work delegates to sibling submodules (`packages`, `framework_libs`, //! `local_libs`, `boot_artifacts`, `embed_stage`, `helpers`). @@ -30,22 +30,23 @@ use crate::flag_overlay::{apply_overlay_flags, LanguageExtraFlags}; use crate::linker::LinkerScripts; use crate::{BuildOrchestrator, BuildParams, BuildResult, SourceScanner}; +#[async_trait::async_trait] impl BuildOrchestrator for Esp32Orchestrator { fn platform(&self) -> Platform { Platform::Espressif32 } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); // Env-gated per-phase timer (FBUILD_PERF_LOG=1); zero overhead when unset. let mut perf = crate::perf_log::PerfTimer::new("esp32-orchestrator"); - // Wrapper-binary discovery removed in FastLED/fbuild#800 — every + // Wrapper-binary discovery removed in FastLED/fbuild#800 — every // compile dispatches through the embedded zccache service. let compiler_cache: Option = None; // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = crate::pipeline::BuildContext::new_with_perf(params, Some(&mut perf))?; + let mut ctx = crate::pipeline::BuildContext::new_with_perf(params, Some(&mut perf)).await?; // Compute eh_frame strip policy once per build (FastLED/fbuild#243). // Reads sdkconfig from the project dir (ESP32 only) so panic-backtrace @@ -71,7 +72,7 @@ impl BuildOrchestrator for Esp32Orchestrator { // 4-6. Resolve platform, toolchain, and framework let _resolve_phase = perf.phase("pioarduino-resolve"); let (toolchain, framework) = - resolve_pioarduino_packages(¶ms.project_dir, &ctx.board.mcu, &mcu_config)?; + resolve_pioarduino_packages(¶ms.project_dir, &ctx.board.mcu, &mcu_config).await?; drop(_resolve_phase); let _toolchain_cache_dir = fbuild_packages::Package::get_info(&toolchain).install_path; let _framework_cache_dir = fbuild_packages::Package::get_info(&framework).install_path; @@ -82,7 +83,7 @@ impl BuildOrchestrator for Esp32Orchestrator { let src_build_dir = &ctx.src_build_dir; // SDK directory selector: matches the chip's ROM revision (e.g. - // `esp32p4_es` for ESP32-P4 eco0–eco2). Falls back to `mcu`. + // `esp32p4_es` for ESP32-P4 eco0–eco2). Falls back to `mcu`. let sdk_variant = ctx.board.sdk_variant().to_string(); // Read link-affecting config before the expensive include/library/source discovery steps @@ -214,7 +215,7 @@ impl BuildOrchestrator for Esp32Orchestrator { // FastLED/fbuild#800: the `zccache start` daemon-spawn was deleted. // The embedded service is part of fbuild-daemon's own lifecycle. - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!( "ESP32 {} toolchain at {}", if mcu_config.is_riscv() { @@ -225,7 +226,7 @@ impl BuildOrchestrator for Esp32Orchestrator { toolchain_dir.display() ); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("ESP32 framework at {}", framework_dir.display()); let tc_label = if mcu_config.is_riscv() { @@ -237,7 +238,8 @@ impl BuildOrchestrator for Esp32Orchestrator { &toolchain.get_gcc_path(), tc_label, &mut ctx.build_log, - ); + ) + .await; let core_dir = framework.get_core_dir(&ctx.board.core); let variant_dir = framework.get_variant_dir(&ctx.board.variant); @@ -274,7 +276,7 @@ impl BuildOrchestrator for Esp32Orchestrator { // Toolchain sysroot includes (xtensa headers, etc.) include_dirs.extend(toolchain.get_include_dirs()); - // Read SDK flags early — needed to check LTO before compiling. + // Read SDK flags early — needed to check LTO before compiling. let sdk_ld_flags = framework.get_sdk_ld_flags(&sdk_variant); let sdk_lib_flags = framework.get_sdk_lib_flags(&sdk_variant, sdk_memory_type.as_deref()); let sdk_ld_scripts = @@ -294,7 +296,7 @@ impl BuildOrchestrator for Esp32Orchestrator { use fbuild_packages::Toolchain; let mut library_archives = Vec::new(); - // Read user build_flags early — needed for both library and sketch compilation. + // Read user build_flags early — needed for both library and sketch compilation. // SDK defines (from flags/defines) are prepended so user flags can override them. let mut user_flags = sdk_defines; let mut user_build_flags = ctx.config.get_build_flags(¶ms.env_name)?; @@ -366,7 +368,7 @@ impl BuildOrchestrator for Esp32Orchestrator { &c_flags, &cpp_flags, ); - let lib_result = fbuild_packages::library::library_manager::ensure_libraries_sync( + let lib_result = fbuild_packages::library::library_manager::ensure_libraries( &lib_deps, &lib_ignore, &toolchain.get_gcc_path(), @@ -379,7 +381,8 @@ impl BuildOrchestrator for Esp32Orchestrator { params.verbose, jobs, compiler_cache.as_deref(), - )?; + ) + .await?; // Add library include dirs to the main include list include_dirs.extend(lib_result.include_dirs); @@ -392,7 +395,7 @@ impl BuildOrchestrator for Esp32Orchestrator { ); } - // 8.5b. Project-as-library compilation — shared with sequential pipeline. + // 8.5b. Project-as-library compilation — shared with sequential pipeline. // When the project root contains library.json or library.properties (e.g., FastLED), // the project's own src/ directory is compiled as a library archive so that example // sketches can link against it. Centralized in pipeline::compile_project_as_library @@ -460,7 +463,9 @@ impl BuildOrchestrator for Esp32Orchestrator { build_dir, &lib_env, &existing_lib_names, - )? { + ) + .await? + { library_archives.push(archive); } } @@ -485,7 +490,8 @@ impl BuildOrchestrator for Esp32Orchestrator { build_dir, compiler_cache.as_deref(), &mut library_archives, - )?; + ) + .await?; } // 9. Scan sources @@ -638,7 +644,8 @@ impl BuildOrchestrator for Esp32Orchestrator { &user_overlay, jobs, Some(&build_log_mutex), - )? + ) + .await? }; if let Some(cache) = core_cache.as_ref() { let _g = perf.phase("core-cache-store"); @@ -672,7 +679,8 @@ impl BuildOrchestrator for Esp32Orchestrator { &src_overlay, jobs, Some(&build_log_mutex), - )? + ) + .await? }; // Unwrap build log and flush collected warnings @@ -699,7 +707,8 @@ impl BuildOrchestrator for Esp32Orchestrator { params.verbose, compiler_cache.as_deref(), &mut library_archives, - )?; + ) + .await?; } // 11.5. Process embedded files (board_build.embed_files + embed_txtfiles) @@ -720,7 +729,8 @@ impl BuildOrchestrator for Esp32Orchestrator { &objcopy_path, &mcu_config, params.verbose, - )?; + ) + .await?; sketch_objects.extend(embed_objects); } @@ -780,7 +790,8 @@ impl BuildOrchestrator for Esp32Orchestrator { bloat_analysis: false, }, params.symbol_analysis, - )? + ) + .await? }; // 14. Prepare boot artifacts for deployment / emulation @@ -791,7 +802,8 @@ impl BuildOrchestrator for Esp32Orchestrator { &mcu_config, &flash_freq, &mut perf, - )?; + ) + .await?; // 15. Size reporting + result assembly let fingerprint_started = Instant::now(); diff --git a/crates/fbuild-build/src/esp32/orchestrator/embed.rs b/crates/fbuild-build/src/esp32/orchestrator/embed.rs index aafacf74..2a9e2311 100644 --- a/crates/fbuild-build/src/esp32/orchestrator/embed.rs +++ b/crates/fbuild-build/src/esp32/orchestrator/embed.rs @@ -13,7 +13,7 @@ use fbuild_core::Result; /// - `embed_files`: embedded as-is (binary) /// - `embed_txtfiles`: a null-terminated copy is created first, then embedded #[allow(clippy::too_many_arguments)] -pub(super) fn process_embed_files( +pub(super) async fn process_embed_files( embed_files: &[String], embed_txtfiles: &[String], project_dir: &Path, @@ -69,7 +69,7 @@ pub(super) fn process_embed_files( } let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, Some(project_dir), None, None)?; + let result = run_command(&args_ref, Some(project_dir), None, None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -129,7 +129,7 @@ pub(super) fn process_embed_files( // Run from embed_dir so objcopy generates symbols from the relative path let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, Some(embed_dir), None, None)?; + let result = run_command(&args_ref, Some(embed_dir), None, None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( diff --git a/crates/fbuild-build/src/esp32/orchestrator/embed_stage.rs b/crates/fbuild-build/src/esp32/orchestrator/embed_stage.rs index 381fad81..cfbdbf53 100644 --- a/crates/fbuild-build/src/esp32/orchestrator/embed_stage.rs +++ b/crates/fbuild-build/src/esp32/orchestrator/embed_stage.rs @@ -11,7 +11,7 @@ use super::embed::process_embed_files; /// cache, then convert each entry into a linkable ELF object. Returns the /// list of object files to be appended to the sketch link set. #[allow(clippy::too_many_arguments)] -pub(super) fn stage_embed_files( +pub(super) async fn stage_embed_files( embed_files: &[String], embed_txtfiles: &[String], project_dir: &Path, @@ -68,4 +68,5 @@ pub(super) fn stage_embed_files( binary_arch, verbose, ) + .await } diff --git a/crates/fbuild-build/src/esp32/orchestrator/framework_libs.rs b/crates/fbuild-build/src/esp32/orchestrator/framework_libs.rs index d792fc64..b358680a 100644 --- a/crates/fbuild-build/src/esp32/orchestrator/framework_libs.rs +++ b/crates/fbuild-build/src/esp32/orchestrator/framework_libs.rs @@ -21,7 +21,7 @@ use crate::BuildParams; /// Compile every Arduino built-in library shipped with the ESP32 framework. /// Library archives are appended to `library_archives`. #[allow(clippy::too_many_arguments)] -pub(super) fn compile_framework_builtin_libs( +pub(super) async fn compile_framework_builtin_libs( params: &BuildParams, perf: &mut crate::perf_log::PerfTimer, framework: &fbuild_packages::library::Esp32Framework, @@ -174,7 +174,9 @@ pub(super) fn compile_framework_builtin_libs( params.verbose, fw_jobs, compiler_cache, - ) { + ) + .await + { Ok(Some(archive)) => { let _ = std::fs::remove_file(&failure_marker); library_archives.push(archive); diff --git a/crates/fbuild-build/src/esp32/orchestrator/local_libs.rs b/crates/fbuild-build/src/esp32/orchestrator/local_libs.rs index 3d235c0e..7011998e 100644 --- a/crates/fbuild-build/src/esp32/orchestrator/local_libs.rs +++ b/crates/fbuild-build/src/esp32/orchestrator/local_libs.rs @@ -11,7 +11,7 @@ use crate::flag_overlay::{apply_overlay_flags, LanguageExtraFlags}; /// Walk `project_dir/lib/*` and compile each subdirectory as a library archive. /// Archives are appended to `library_archives`. #[allow(clippy::too_many_arguments)] -pub(super) fn compile_local_libraries( +pub(super) async fn compile_local_libraries( project_dir: &Path, build_dir: &Path, compiler: &Esp32Compiler, @@ -83,7 +83,9 @@ pub(super) fn compile_local_libraries( verbose, jobs, compiler_cache, - ) { + ) + .await + { Ok(Some(archive)) => { library_archives.push(archive); } diff --git a/crates/fbuild-build/src/esp32/orchestrator/packages.rs b/crates/fbuild-build/src/esp32/orchestrator/packages.rs index d1f2df0d..74ac0ea3 100644 --- a/crates/fbuild-build/src/esp32/orchestrator/packages.rs +++ b/crates/fbuild-build/src/esp32/orchestrator/packages.rs @@ -8,7 +8,7 @@ use fbuild_core::Result; /// /// Downloads pioarduino platform.json, resolves toolchain via metadata, /// and downloads the split framework + libs packages. -pub(super) fn resolve_pioarduino_packages( +pub(super) async fn resolve_pioarduino_packages( project_dir: &Path, mcu: &str, mcu_config: &super::super::mcu_config::Esp32McuConfig, @@ -18,7 +18,7 @@ pub(super) fn resolve_pioarduino_packages( )> { // Ensure pioarduino platform (contains platform.json with metadata URLs) let platform = fbuild_packages::library::Esp32Platform::new(project_dir); - fbuild_packages::Package::ensure_installed(&platform)?; + fbuild_packages::Package::ensure_installed(&platform).await?; // Resolve toolchain via metadata let toolchain = resolve_and_create_toolchain(&platform, project_dir, mcu_config)?; @@ -45,11 +45,11 @@ pub(super) fn resolve_pioarduino_packages( }; // Ensure framework is installed before trying to install libs - let _ = fbuild_packages::Package::ensure_installed(&framework)?; + let _ = fbuild_packages::Package::ensure_installed(&framework).await?; // Ensure SDK libs (split package in pioarduino 3.3.7+) if let Ok(libs_url) = platform.get_package_url("framework-arduinoespressif32-libs") { - framework.ensure_libs(&libs_url)?; + framework.ensure_libs(&libs_url).await?; } // Ensure MCU-specific skeleton libs (e.g. ESP32-C2, ESP32-C61). @@ -58,7 +58,7 @@ pub(super) fn resolve_pioarduino_packages( if !mcu_suffix.is_empty() { let skeleton_name = format!("framework-arduino-{}-skeleton-lib", mcu_suffix); if let Ok(skeleton_url) = platform.get_package_url(&skeleton_name) { - framework.ensure_mcu_libs(&skeleton_url, mcu)?; + framework.ensure_mcu_libs(&skeleton_url, mcu).await?; } } diff --git a/crates/fbuild-build/src/esp8266/esp8266_compiler.rs b/crates/fbuild-build/src/esp8266/esp8266_compiler.rs index 6b4a7b54..f398addc 100644 --- a/crates/fbuild-build/src/esp8266/esp8266_compiler.rs +++ b/crates/fbuild-build/src/esp8266/esp8266_compiler.rs @@ -110,8 +110,9 @@ impl Esp8266Compiler { } } +#[async_trait::async_trait] impl Compiler for Esp8266Compiler { - fn compile_one( + async fn compile_one( &self, compiler_path: &Path, source: &Path, @@ -131,21 +132,23 @@ impl Compiler for Esp8266Compiler { None, &[], ) + .await } - fn compile( + async fn compile( &self, source: &Path, output: &Path, extra_flags: &[String], ) -> Result { match source.extension().and_then(|ext| ext.to_str()) { - Some("c") => self.compile_c(source, output, extra_flags), + Some("c") => self.compile_c(source, output, extra_flags).await, Some("S") | Some("s") => { let flags = self.asm_flags(); self.compile_one(self.gcc_path(), source, output, &flags, extra_flags) + .await } - _ => self.compile_cpp(source, output, extra_flags), + _ => self.compile_cpp(source, output, extra_flags).await, } } diff --git a/crates/fbuild-build/src/esp8266/esp8266_linker.rs b/crates/fbuild-build/src/esp8266/esp8266_linker.rs index d4c72ec5..5ff54f97 100644 --- a/crates/fbuild-build/src/esp8266/esp8266_linker.rs +++ b/crates/fbuild-build/src/esp8266/esp8266_linker.rs @@ -1,4 +1,4 @@ -//! ESP8266 linker implementation. +//! ESP8266 linker implementation. //! //! Links ESP8266 object files into `firmware.elf`, converts to `firmware.bin` //! using esptool (with objcopy fallback), and reports size. @@ -24,9 +24,9 @@ pub struct Esp8266Linker { size_path: PathBuf, /// Path to `tools/sdk/lib/` inside the framework. sdk_lib_dir: PathBuf, - /// Path to `tools/sdk/lib/NONOSDK305/` — NonOS SDK version-specific libraries. + /// Path to `tools/sdk/lib/NONOSDK305/` — NonOS SDK version-specific libraries. sdk_nonosdk_lib_dir: PathBuf, - /// Path to `tools/sdk/ld/` — needed by `generate_linker_scripts()` for template lookup. + /// Path to `tools/sdk/ld/` — needed by `generate_linker_scripts()` for template lookup. sdk_ld_dir: PathBuf, /// Board linker script + search directories. linker_scripts: LinkerScripts, @@ -81,13 +81,13 @@ impl Esp8266Linker { } impl Esp8266Linker { - /// Preprocess `eagle.app.v6.common.ld.h` → `local.eagle.app.v6.common.ld`. + /// Preprocess `eagle.app.v6.common.ld.h` → `local.eagle.app.v6.common.ld`. /// /// The board linker script (e.g. `eagle.flash.4m1m.ld`) includes /// `local.eagle.app.v6.common.ld` via the linker's `INCLUDE` directive. - /// This file doesn't ship pre-built — it must be generated by running + /// This file doesn't ship pre-built — it must be generated by running /// the GCC preprocessor on the `.ld.h` template with the correct defines. - fn generate_linker_scripts(&self, output_dir: &Path) -> Result<()> { + async fn generate_linker_scripts(&self, output_dir: &Path) -> Result<()> { let ld_template = self.sdk_ld_dir.join("eagle.app.v6.common.ld.h"); if !ld_template.exists() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -118,7 +118,7 @@ impl Esp8266Linker { args.extend(["-o".to_string(), output_ld.to_string_lossy().to_string()]); let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, None, None)?; + let result = run_command(&args_ref, None, None, None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( "ESP8266 linker script preprocessing failed:\n{}", @@ -130,12 +130,14 @@ impl Esp8266Linker { } } +#[async_trait::async_trait] impl Linker for Esp8266Linker { - fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { + async fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "xtensa-lx106-elf-ar") + .await } - fn link( + async fn link( &self, objects: &[PathBuf], archives: &[PathBuf], @@ -145,7 +147,7 @@ impl Linker for Esp8266Linker { std::fs::create_dir_all(output_dir)?; // Preprocess linker script template before linking - self.generate_linker_scripts(output_dir)?; + self.generate_linker_scripts(output_dir).await?; let elf_path = output_dir.join("firmware.elf"); let mut positional_archives: Vec = archives @@ -172,7 +174,7 @@ impl Linker for Esp8266Linker { if !loose_objects.is_empty() { let bundled_archive = output_dir.join("libfbuild-core.a"); - self.archive(&loose_objects, &bundled_archive)?; + self.archive(&loose_objects, &bundled_archive).await?; positional_archives.insert(0, bundled_archive); } @@ -187,7 +189,7 @@ impl Linker for Esp8266Linker { } args.extend(extra.flags.iter().cloned()); - // Build output dir — contains generated local.eagle.app.v6.common.ld + // Build output dir — contains generated local.eagle.app.v6.common.ld args.push(format!("-L{}", output_dir.to_string_lossy())); // Board linker script + SDK ld search directory args.extend(self.linker_scripts.to_args()); @@ -225,7 +227,7 @@ impl Linker for Esp8266Linker { } let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, None, None)?; + let result = run_command(&args_ref, None, None, None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -237,7 +239,7 @@ impl Linker for Esp8266Linker { Ok(elf_path) } - fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { + async fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { let bin_path = output_dir.join("firmware.bin"); // Try esptool first @@ -259,14 +261,14 @@ impl Linker for Esp8266Linker { &bin_str, ]; - match run_command(&args, None, None, Some(std::time::Duration::from_secs(30))) { + match run_command(&args, None, None, Some(std::time::Duration::from_secs(30))).await { Ok(result) if result.success() => { // Verify the output file was actually created with content. // ESP8266 esptool may create segmented files (firmware.bin-0x00000.bin) // instead of a single firmware.bin, leaving it empty. let size = std::fs::metadata(&bin_path).map(|m| m.len()).unwrap_or(0); if size > 0 { - tracing::info!("converted ELF → firmware.bin via esptool"); + tracing::info!("converted ELF → firmware.bin via esptool"); return Ok(bin_path); } tracing::debug!("esptool produced empty firmware.bin (falling back to objcopy)"); @@ -291,6 +293,7 @@ impl Linker for Esp8266Linker { &self.mcu_config.objcopy.remove_sections, "xtensa-lx106-elf-objcopy", ) + .await } fn size_tool_path(&self) -> &Path { @@ -309,7 +312,7 @@ impl Linker for Esp8266Linker { Some(&self.gcc_path) } - fn report_size(&self, elf_path: &Path) -> Result { + async fn report_size(&self, elf_path: &Path) -> Result { crate::linker::LinkerBase::report_size( &self.size_path, elf_path, @@ -317,6 +320,7 @@ impl Linker for Esp8266Linker { self.max_ram, "xtensa-lx106-elf-size", ) + .await } } diff --git a/crates/fbuild-build/src/esp8266/mod.rs b/crates/fbuild-build/src/esp8266/mod.rs index 8a152f84..7e29e5d0 100644 --- a/crates/fbuild-build/src/esp8266/mod.rs +++ b/crates/fbuild-build/src/esp8266/mod.rs @@ -1,4 +1,4 @@ -//! ESP8266 platform build support (NodeMCU, Wemos D1, etc.) +//! ESP8266 platform build support (NodeMCU, Wemos D1, etc.) pub mod esp8266_compiler; pub mod esp8266_linker; @@ -12,15 +12,16 @@ pub use orchestrator::Esp8266Orchestrator; /// ESP8266 platform support. pub struct Esp8266PlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for Esp8266PlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::Esp8266Toolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("ESP8266 toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/esp8266/orchestrator.rs b/crates/fbuild-build/src/esp8266/orchestrator.rs index bd1eb06e..719f9b57 100644 --- a/crates/fbuild-build/src/esp8266/orchestrator.rs +++ b/crates/fbuild-build/src/esp8266/orchestrator.rs @@ -1,4 +1,4 @@ -//! ESP8266 build orchestrator — wires together config, packages, compiler, linker. +//! ESP8266 build orchestrator — wires together config, packages, compiler, linker. //! //! Build phases: //! 1. Parse platformio.ini @@ -28,16 +28,17 @@ use super::mcu_config::get_esp8266_config; /// ESP8266 platform build orchestrator. pub struct Esp8266Orchestrator; +#[async_trait::async_trait] impl BuildOrchestrator for Esp8266Orchestrator { fn platform(&self) -> Platform { Platform::Espressif8266 } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = pipeline::BuildContext::new(params)?; + let mut ctx = pipeline::BuildContext::new(params).await?; // Compute eh_frame strip policy once per build (FastLED/fbuild#244). // No sdkconfig on ESP8266. @@ -46,7 +47,7 @@ impl BuildOrchestrator for Esp8266Orchestrator { // 3. Ensure toolchain let toolchain = fbuild_packages::toolchain::Esp8266Toolchain::new(¶ms.project_dir); - let _toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let _toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("ESP8266 toolchain ready"); use fbuild_packages::Toolchain as _; @@ -54,11 +55,12 @@ impl BuildOrchestrator for Esp8266Orchestrator { &toolchain.get_gcc_path(), "xtensa-lx106-elf-gcc", &mut ctx.build_log, - ); + ) + .await; // 4. Ensure framework let framework = fbuild_packages::library::Esp8266Framework::new(¶ms.project_dir); - let _framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let _framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("ESP8266 framework ready"); let board_id = ctx .config @@ -110,7 +112,7 @@ impl BuildOrchestrator for Esp8266Orchestrator { // SDK include paths include_dirs.extend(framework.get_sdk_include_dirs()); // Toolchain sysroot includes (xtensa/coreasm.h, etc.) - // Required by .S assembly files — see platform.txt compiler.S.flags. + // Required by .S assembly files — see platform.txt compiler.S.flags. include_dirs.extend(toolchain.get_include_dirs()); // SDK libc headers (platform.txt compiler.libc.path) include_dirs.extend(framework.get_libc_include_dirs()); @@ -218,6 +220,7 @@ impl BuildOrchestrator for Esp8266Orchestrator { "ESP8266", start, ) + .await } } diff --git a/crates/fbuild-build/src/framework_core_cache.rs b/crates/fbuild-build/src/framework_core_cache.rs index b56e7a0c..611b258a 100644 --- a/crates/fbuild-build/src/framework_core_cache.rs +++ b/crates/fbuild-build/src/framework_core_cache.rs @@ -200,8 +200,9 @@ mod tests { } } + #[async_trait::async_trait] impl Compiler for FakeCompiler { - fn compile_one( + async fn compile_one( &self, _compiler_path: &Path, _source: &Path, diff --git a/crates/fbuild-build/src/generic_arm/arm_compiler.rs b/crates/fbuild-build/src/generic_arm/arm_compiler.rs index 3f00eacf..1c4e1945 100644 --- a/crates/fbuild-build/src/generic_arm/arm_compiler.rs +++ b/crates/fbuild-build/src/generic_arm/arm_compiler.rs @@ -113,8 +113,9 @@ impl ArmCompiler { } } +#[async_trait::async_trait] impl Compiler for ArmCompiler { - fn compile_one( + async fn compile_one( &self, compiler_path: &Path, source: &Path, @@ -134,6 +135,7 @@ impl Compiler for ArmCompiler { self.compiler_cache.as_deref(), &[], ) + .await } fn gcc_path(&self) -> &Path { diff --git a/crates/fbuild-build/src/generic_arm/arm_linker.rs b/crates/fbuild-build/src/generic_arm/arm_linker.rs index a845dfe2..bd2ff0a2 100644 --- a/crates/fbuild-build/src/generic_arm/arm_linker.rs +++ b/crates/fbuild-build/src/generic_arm/arm_linker.rs @@ -1,4 +1,4 @@ -//! Generic ARM linker implementation. +//! Generic ARM linker implementation. //! //! Links ARM Cortex-M object files into firmware.elf, converts to firmware.hex, //! and reports size using arm-none-eabi-size. Used by STM32, RP2040, NRF52, etc. @@ -129,12 +129,14 @@ impl ArmLinker { } } +#[async_trait::async_trait] impl Linker for ArmLinker { - fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { + async fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "arm-none-eabi-ar") + .await } - fn link( + async fn link( &self, objects: &[PathBuf], archives: &[PathBuf], @@ -151,7 +153,7 @@ impl Linker for ArmLinker { tracing::info!("link: {}", args.join(" ")); } - // GCC LTO temp dir for MSYS-safe paths — see FastLED/fbuild#261. + // GCC LTO temp dir for MSYS-safe paths — see FastLED/fbuild#261. let lto_env = fbuild_core::subprocess::link_env_for_build(output_dir)?; let env_slice: Vec<(&str, &str)> = lto_env .iter() @@ -168,12 +170,13 @@ impl Linker for ArmLinker { &rsp_content, &temp_dir, "arm_link", - )?; + ) + .await?; let rsp_arg = format!("@{}", rsp_path.display()); - run_command(&[args[0].as_str(), &rsp_arg], None, Some(&env_slice), None)? + run_command(&[args[0].as_str(), &rsp_arg], None, Some(&env_slice), None).await? } else { let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - run_command(&args_ref, None, Some(&env_slice), None)? + run_command(&args_ref, None, Some(&env_slice), None).await? }; if !result.success() { @@ -202,7 +205,7 @@ impl Linker for ArmLinker { Ok(elf_path) } - fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { + async fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { crate::linker::LinkerBase::objcopy_firmware( &self.objcopy_path, elf_path, @@ -211,6 +214,7 @@ impl Linker for ArmLinker { &self.mcu_config.objcopy.remove_sections, "arm-none-eabi-objcopy", ) + .await } fn size_tool_path(&self) -> &Path { @@ -229,7 +233,7 @@ impl Linker for ArmLinker { Some(&self.gcc_path) } - fn report_size(&self, elf_path: &Path) -> Result { + async fn report_size(&self, elf_path: &Path) -> Result { crate::linker::LinkerBase::report_size( &self.size_path, elf_path, @@ -237,6 +241,7 @@ impl Linker for ArmLinker { self.max_ram, "arm-none-eabi-size", ) + .await } } diff --git a/crates/fbuild-build/src/lib.rs b/crates/fbuild-build/src/lib.rs index aa012844..d232d09c 100644 --- a/crates/fbuild-build/src/lib.rs +++ b/crates/fbuild-build/src/lib.rs @@ -55,12 +55,16 @@ use fbuild_core::{BuildProfile, Platform, Result, SizeInfo}; /// dependency installation, and configuration. Adding a new platform requires: /// 1. Implement this trait in the platform module /// 2. Register in `get_platform_support()` +/// +/// FastLED/fbuild#820 (Phase B of #813): `install_deps` is `async` so +/// per-platform impls can `.await` `fbuild_packages::Package::ensure_installed`. +#[async_trait::async_trait] pub trait PlatformSupport: Send + Sync { /// Create the build orchestrator for this platform. fn create_orchestrator(&self) -> Box; /// Install platform-specific dependencies (toolchain, framework). - fn install_deps(&self, project_dir: &Path) -> Result<()>; + async fn install_deps(&self, project_dir: &Path) -> Result<()>; /// Default board ID used as fallback when none is specified. fn default_board_id(&self) -> &str; @@ -198,9 +202,15 @@ pub struct BuildParams { } /// Trait for platform-specific build orchestrators. +/// +/// FastLED/fbuild#820 (Phase B of #813): `build` is `async` so per-platform +/// orchestrators can `.await` toolchain install, subprocess invocation, +/// and per-TU zccache dispatch directly instead of `Handle::block_on`-ing +/// from a sync entry point. +#[async_trait::async_trait] pub trait BuildOrchestrator: Send + Sync { fn platform(&self) -> Platform; - fn build(&self, params: &BuildParams) -> Result; + async fn build(&self, params: &BuildParams) -> Result; } /// Select the appropriate orchestrator for a platform. @@ -209,8 +219,8 @@ pub fn get_orchestrator(platform: Platform) -> Result } /// Install platform-specific dependencies (toolchain, framework). -pub fn install_platform_deps(platform: Platform, project_dir: &Path) -> Result<()> { - get_platform_support(platform)?.install_deps(project_dir) +pub async fn install_platform_deps(platform: Platform, project_dir: &Path) -> Result<()> { + get_platform_support(platform)?.install_deps(project_dir).await } #[cfg(test)] diff --git a/crates/fbuild-build/src/linker.rs b/crates/fbuild-build/src/linker.rs index 48a8fde8..f0864ea5 100644 --- a/crates/fbuild-build/src/linker.rs +++ b/crates/fbuild-build/src/linker.rs @@ -56,12 +56,18 @@ fn elf_is_up_to_date<'a>(elf_path: &Path, inputs: impl Iterator Result<()>; + async fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()>; /// Link objects and archives into an ELF binary. - fn link( + async fn link( &self, objects: &[PathBuf], archives: &[PathBuf], @@ -70,10 +76,10 @@ pub trait Linker: Send + Sync { ) -> Result; /// Convert ELF to firmware format (.hex, .bin, etc.). - fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result; + async fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result; /// Report firmware size. - fn report_size(&self, elf_path: &Path) -> Result; + async fn report_size(&self, elf_path: &Path) -> Result; /// Path to the platform's size tool (e.g. `arm-none-eabi-size`). /// @@ -111,7 +117,7 @@ pub trait Linker: Send + Sync { /// Skips relinking when the existing firmware.elf is newer than all input /// objects and archives, saving ~10-14s on incremental builds where only /// the compilation step ran (or nothing changed at all). - fn link_all( + async fn link_all( &self, sketch_objects: &[PathBuf], core_objects: &[PathBuf], @@ -127,10 +133,12 @@ pub trait Linker: Send + Sync { ); if can_skip { tracing::info!("link: firmware.elf is up-to-date, skipping relink"); - let firmware_path = self.convert_firmware(&candidate_elf, output_dir)?; - let size_info = self.report_size(&candidate_elf).ok(); + let firmware_path = self.convert_firmware(&candidate_elf, output_dir).await?; + let size_info = self.report_size(&candidate_elf).await.ok(); let symbol_map = if symbol_analysis { - LinkerBase::analyze_symbols(self.size_tool_path(), &candidate_elf).ok() + LinkerBase::analyze_symbols(self.size_tool_path(), &candidate_elf) + .await + .ok() } else { None }; @@ -154,17 +162,21 @@ pub trait Linker: Send + Sync { // Pass core objects directly to linker (not archived) for LTO compatibility. // With LTO + archives, the linker can't see symbols across TUs properly. - let elf_path = self.link(sketch_objects, core_objects, output_dir, extra)?; + let elf_path = self + .link(sketch_objects, core_objects, output_dir, extra) + .await?; // Convert - let firmware_path = self.convert_firmware(&elf_path, output_dir)?; + let firmware_path = self.convert_firmware(&elf_path, output_dir).await?; // Size - let size_info = self.report_size(&elf_path).ok(); + let size_info = self.report_size(&elf_path).await.ok(); // Symbol analysis let symbol_map = if symbol_analysis { - LinkerBase::analyze_symbols(self.size_tool_path(), &elf_path).ok() + LinkerBase::analyze_symbols(self.size_tool_path(), &elf_path) + .await + .ok() } else { None }; @@ -300,7 +312,7 @@ impl LinkerBase { } /// Create a static archive (.a) from object files using `ar rcs`. - pub fn archive( + pub async fn archive( ar_path: &Path, objects: &[PathBuf], output: &Path, @@ -328,7 +340,7 @@ impl LinkerBase { } let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, None, None)?; + let result = run_command(&args_ref, None, None, None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -341,7 +353,7 @@ impl LinkerBase { } /// Report firmware size by running the size tool and parsing its output. - pub fn report_size( + pub async fn report_size( size_path: &Path, elf_path: &Path, max_flash: Option, @@ -356,7 +368,7 @@ impl LinkerBase { ]; let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, None, None)?; + let result = run_command(&args_ref, None, None, None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -394,7 +406,10 @@ impl LinkerBase { } /// Run `nm --print-size --size-sort --reverse-sort` on an ELF and parse the output. - pub fn analyze_symbols(size_path: &Path, elf_path: &Path) -> Result { + pub async fn analyze_symbols( + size_path: &Path, + elf_path: &Path, + ) -> Result { use fbuild_core::subprocess::run_command; let nm_path = Self::nm_path_from_size_path(size_path); @@ -406,7 +421,7 @@ impl LinkerBase { elf_path.to_string_lossy().to_string(), ]; let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, None, None)?; + let result = run_command(&args_ref, None, None, None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -421,7 +436,7 @@ impl LinkerBase { } /// Convert ELF to firmware using objcopy (shared by AVR and Teensy). - pub fn objcopy_firmware( + pub async fn objcopy_firmware( objcopy_path: &Path, elf_path: &Path, output_dir: &Path, @@ -452,7 +467,7 @@ impl LinkerBase { args.push(hex_path.to_string_lossy().to_string()); let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, None, None)?; + let result = run_command(&args_ref, None, None, None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( diff --git a/crates/fbuild-build/src/nrf52/mod.rs b/crates/fbuild-build/src/nrf52/mod.rs index b76d9da9..a7dbf0a1 100644 --- a/crates/fbuild-build/src/nrf52/mod.rs +++ b/crates/fbuild-build/src/nrf52/mod.rs @@ -1,4 +1,4 @@ -//! NRF52 platform build support (Nordic NRF52840, etc.) +//! NRF52 platform build support (Nordic NRF52840, etc.) pub mod mcu_config; pub mod nrf52_compiler; @@ -12,15 +12,16 @@ pub use orchestrator::Nrf52Orchestrator; /// NRF52 platform support. pub struct Nrf52PlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for Nrf52PlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("ARM toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/nrf52/nrf52_compiler.rs b/crates/fbuild-build/src/nrf52/nrf52_compiler.rs index 846f72ff..e761a260 100644 --- a/crates/fbuild-build/src/nrf52/nrf52_compiler.rs +++ b/crates/fbuild-build/src/nrf52/nrf52_compiler.rs @@ -106,8 +106,9 @@ impl Nrf52Compiler { } } +#[async_trait::async_trait] impl Compiler for Nrf52Compiler { - fn compile_one( + async fn compile_one( &self, compiler_path: &Path, source: &Path, @@ -147,6 +148,7 @@ impl Compiler for Nrf52Compiler { None, &[], ) + .await } fn gcc_path(&self) -> &Path { diff --git a/crates/fbuild-build/src/nrf52/nrf52_linker.rs b/crates/fbuild-build/src/nrf52/nrf52_linker.rs index 4ef2ae2d..2043ef0a 100644 --- a/crates/fbuild-build/src/nrf52/nrf52_linker.rs +++ b/crates/fbuild-build/src/nrf52/nrf52_linker.rs @@ -1,4 +1,4 @@ -//! NRF52 ARM linker implementation. +//! NRF52 ARM linker implementation. //! //! Links ARM Cortex-M4F object files into firmware.elf, converts to firmware.hex, //! and reports size using arm-none-eabi-size. @@ -57,12 +57,14 @@ impl Nrf52Linker { } } +#[async_trait::async_trait] impl Linker for Nrf52Linker { - fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { + async fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "arm-none-eabi-ar") + .await } - fn link( + async fn link( &self, objects: &[PathBuf], archives: &[PathBuf], @@ -117,7 +119,7 @@ impl Linker for Nrf52Linker { tracing::info!("link: {}", args.join(" ")); } - // GCC LTO temp dir for MSYS-safe paths — see FastLED/fbuild#261. + // GCC LTO temp dir for MSYS-safe paths — see FastLED/fbuild#261. let lto_env = fbuild_core::subprocess::link_env_for_build(output_dir)?; let env_slice: Vec<(&str, &str)> = lto_env .iter() @@ -125,7 +127,7 @@ impl Linker for Nrf52Linker { .collect(); let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, Some(&env_slice), None)?; + let result = run_command(&args_ref, None, Some(&env_slice), None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -137,7 +139,7 @@ impl Linker for Nrf52Linker { Ok(elf_path) } - fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { + async fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { crate::linker::LinkerBase::objcopy_firmware( &self.objcopy_path, elf_path, @@ -146,6 +148,7 @@ impl Linker for Nrf52Linker { &self.mcu_config.objcopy.remove_sections, "arm-none-eabi-objcopy", ) + .await } fn size_tool_path(&self) -> &Path { @@ -164,7 +167,7 @@ impl Linker for Nrf52Linker { Some(&self.gcc_path) } - fn report_size(&self, elf_path: &Path) -> Result { + async fn report_size(&self, elf_path: &Path) -> Result { crate::linker::LinkerBase::report_size( &self.size_path, elf_path, @@ -172,6 +175,7 @@ impl Linker for Nrf52Linker { self.max_ram, "arm-none-eabi-size", ) + .await } } diff --git a/crates/fbuild-build/src/nrf52/orchestrator.rs b/crates/fbuild-build/src/nrf52/orchestrator.rs index 743953b1..f684396b 100644 --- a/crates/fbuild-build/src/nrf52/orchestrator.rs +++ b/crates/fbuild-build/src/nrf52/orchestrator.rs @@ -1,4 +1,4 @@ -//! NRF52 build orchestrator — wires together config, packages, compiler, linker. +//! NRF52 build orchestrator — wires together config, packages, compiler, linker. //! //! Build phases: //! 1. Parse platformio.ini @@ -57,21 +57,22 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +#[async_trait::async_trait] impl BuildOrchestrator for Nrf52Orchestrator { fn platform(&self) -> Platform { Platform::NordicNrf52 } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); let compiler_cache: Option = None; // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = pipeline::BuildContext::new(params)?; + let mut ctx = pipeline::BuildContext::new(params).await?; // 3. Ensure ARM GCC toolchain let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-none-eabi toolchain at {}", toolchain_dir.display()); use fbuild_packages::Toolchain; @@ -79,11 +80,12 @@ impl BuildOrchestrator for Nrf52Orchestrator { &toolchain.get_gcc_path(), "arm-none-eabi-gcc", &mut ctx.build_log, - ); + ) + .await; // 4. Ensure NRF52 cores (Adafruit nRF52 Arduino core) let framework = fbuild_packages::library::Nrf52Cores::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("NRF52 cores at {}", framework_dir.display()); let build_dir = &ctx.build_dir; @@ -236,7 +238,7 @@ impl BuildOrchestrator for Nrf52Orchestrator { include_dirs.extend(toolchain.get_include_dirs()); // CMSIS Core includes (core_cm4.h, etc.) let cmsis = fbuild_packages::library::CmsisFramework::new(¶ms.project_dir); - let _cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis)?; + let _cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis).await?; tracing::info!("CMSIS framework installed"); include_dirs.push(cmsis.get_core_include_dir()); include_dirs.push(cmsis.get_dsp_include_dir()); @@ -300,14 +302,14 @@ impl BuildOrchestrator for Nrf52Orchestrator { params.verbose, ) .with_build_unflags(ctx.build_unflags.clone()) - // Scope `-Wno-array-bounds` to Adafruit nRF52 BSP sources only — + // Scope `-Wno-array-bounds` to Adafruit nRF52 BSP sources only — // FastLED + user sketch code still sees the full diagnostic. See // FastLED/fbuild#407. .with_framework_root(framework_dir.clone()); // 7. Create linker (reuse the alias-resolved linker_script_path // computed up front so the fingerprint hash and the actual linker - // input are always in sync — see comment above the metadata_hash). + // input are always in sync — see comment above the metadata_hash). let linker = Nrf52Linker::new( toolchain.get_gcc_path(), toolchain.get_ar_path(), @@ -355,7 +357,7 @@ impl BuildOrchestrator for Nrf52Orchestrator { TargetArchitecture::Arm, "NRF52", start, - )?; + ).await?; if build_result.success && !params.compiledb_only diff --git a/crates/fbuild-build/src/nxplpc/mod.rs b/crates/fbuild-build/src/nxplpc/mod.rs index 696478ae..aef41131 100644 --- a/crates/fbuild-build/src/nxplpc/mod.rs +++ b/crates/fbuild-build/src/nxplpc/mod.rs @@ -1,10 +1,10 @@ -//! NXP LPC8xx (Cortex-M0+) bare-metal build support. +//! NXP LPC8xx (Cortex-M0+) bare-metal build support. //! //! - Stage 1 (shipped): board/toolchain wiring, board JSON, dispatch entry. //! - Stage 2 (shipped): build orchestrator (see [`orchestrator`]). //! - Stage 3/4 (this module, #479 / #487): the orchestrator vendors the real //! Arduino framework [`zackees/ArduinoCore-LPC8xx`](https://github.com/zackees/ArduinoCore-LPC8xx) -//! via the package downloader (`ArduinoCoreLpc8xx`) — framework-owned +//! via the package downloader (`ArduinoCoreLpc8xx`) — framework-owned //! `main()`, startup + vector table, wiring, HardwareSerial, SPI, the GCC //! linker scripts, and per-board variants. The previously embedded //! `arduino_stub/`, device headers, startup `.S`, linker scripts, and @@ -15,7 +15,7 @@ pub mod mcu_config; pub mod orchestrator; // `platform_packages` lookup is now shared at the workspace level -// (FastLED/fbuild#681) — see `crate::package_override`. The per-platform +// (FastLED/fbuild#681) — see `crate::package_override`. The per-platform // parser introduced in #663 has been folded into // `fbuild_config::platform_packages` so every orchestrator gets the same // parser without duplication. @@ -27,22 +27,23 @@ use fbuild_core::Result; /// NXP LPC8xx platform support. pub struct NxpLpcPlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for NxpLpcPlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &Path) -> Result<()> { + async fn install_deps(&self, project_dir: &Path) -> Result<()> { // ARM GCC is the right toolchain for Cortex-M0+ bare metal. // Pre-install it (+ CMSIS + the Arduino core framework) so the // orchestrator can `ensure_installed` cheaply. use fbuild_packages::Package; let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; let cmsis = fbuild_packages::library::CmsisFramework::new(project_dir); - Package::ensure_installed(&cmsis)?; + Package::ensure_installed(&cmsis).await?; let core = fbuild_packages::library::ArduinoCoreLpc8xx::new(project_dir); - Package::ensure_installed(&core)?; + Package::ensure_installed(&core).await?; tracing::info!("ARM GCC toolchain + ArduinoCore-LPC8xx installed for NXP LPC8xx"); Ok(()) } diff --git a/crates/fbuild-build/src/nxplpc/orchestrator.rs b/crates/fbuild-build/src/nxplpc/orchestrator.rs index 61dd1671..a52061f0 100644 --- a/crates/fbuild-build/src/nxplpc/orchestrator.rs +++ b/crates/fbuild-build/src/nxplpc/orchestrator.rs @@ -1,19 +1,19 @@ -//! NXP LPC8xx build orchestrator — Stage 2 of #487. +//! NXP LPC8xx build orchestrator — Stage 2 of #487. //! -//! Compiles user sketch sources (.ino → .cpp + .c + .cpp + .S) together +//! Compiles user sketch sources (.ino → .cpp + .c + .cpp + .S) together //! with the per-MCU startup `.S` and the hand-rolled Arduino `main.cpp` //! shim, links against the per-MCU linker script, and emits `firmware.elf` //! + `firmware.bin` via objcopy. //! -//! No external framework is required at this stage — the test fixtures +//! No external framework is required at this stage — the test fixtures //! (`tests/platform/lpc845/lpc845.ino`, //! `tests/platform/lpc804/lpc804.ino`) are 3-line `setup()`/`loop()` stubs. //! Stage 3 (#479) replaces the embedded shim with the framework-owned //! `main()` from [`zackees/ArduinoCore-LPC8xx`](https://github.com/zackees/ArduinoCore-LPC8xx). //! //! Pattern mirrors the Apollo3 orchestrator -//! (`crates/fbuild-build/src/apollo3/orchestrator.rs`) — same Cortex-M -//! family, same `generic_arm::ArmCompiler` + `ArmLinker` pipeline — minus +//! (`crates/fbuild-build/src/apollo3/orchestrator.rs`) — same Cortex-M +//! family, same `generic_arm::ArmCompiler` + `ArmLinker` pipeline — minus //! the mbed-os framework machinery that Apollo3 needs. use std::path::PathBuf; @@ -80,18 +80,19 @@ fn collect_compilable_sources(dir: &std::path::Path) -> Result> { /// NXP LPC8xx (Cortex-M0+) build orchestrator. pub struct NxpLpcOrchestrator; +#[async_trait::async_trait] impl BuildOrchestrator for NxpLpcOrchestrator { fn platform(&self) -> Platform { Platform::NxpLpc } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); // 1-2. Parse platformio.ini, load board, setup build dirs. - let mut ctx = pipeline::BuildContext::new(params)?; + let mut ctx = pipeline::BuildContext::new(params).await?; - // eh_frame strip policy — same convention every other orchestrator + // eh_frame strip policy — same convention every other orchestrator // follows (#244). let eh_frame_policy = crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); @@ -100,11 +101,11 @@ impl BuildOrchestrator for NxpLpcOrchestrator { // the platform is dispatched, but ensure_installed is idempotent // and cheap when the toolchain is already on disk. let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-none-eabi-gcc toolchain at {}", toolchain_dir.display()); let cmsis = fbuild_packages::library::CmsisFramework::new(¶ms.project_dir); - let cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis)?; + let cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis).await?; tracing::info!("CMSIS framework at {}", cmsis_dir.display()); use fbuild_packages::Toolchain; @@ -112,7 +113,8 @@ impl BuildOrchestrator for NxpLpcOrchestrator { &toolchain.get_gcc_path(), "arm-none-eabi-gcc", &mut ctx.build_log, - ); + ) + .await; // 4. Vendor the Arduino LPC8xx core framework. This supersedes the // embedded `arduino_stub/` shim (FastLED/fbuild#479, #487): the @@ -144,7 +146,7 @@ impl BuildOrchestrator for NxpLpcOrchestrator { } None => fbuild_packages::library::ArduinoCoreLpc8xx::new(¶ms.project_dir), }; - let core_root = fbuild_packages::Package::ensure_installed(&core)?; + let core_root = fbuild_packages::Package::ensure_installed(&core).await?; tracing::info!("ArduinoCore-LPC8xx at {}", core_root.display()); // 5. Family + linker script. The board's `ldscript` is relative to @@ -269,7 +271,7 @@ impl BuildOrchestrator for NxpLpcOrchestrator { // see the same -D defines / -std overrides the sketch will see. // Without this fold, the only way to get `build_flags` defines into a // library compile was to bake them into the board JSON's `extra_flags` - // — exactly the workaround #576 installed for `lpc845brk` and that + // — exactly the workaround #576 installed for `lpc845brk` and that // this PR retires. Mirrors the ESP32 library-compile path at // `esp32/orchestrator/build.rs`; see FastLED/fbuild#587. let gcc_path = toolchain.get_gcc_path(); @@ -304,7 +306,8 @@ impl BuildOrchestrator for NxpLpcOrchestrator { compiler_cache: None, }; let extra_link_inputs = - pipeline::compile_extra_libraries(&extra_library_roots, &ctx.build_dir, &lib_env)?; + pipeline::compile_extra_libraries(&extra_library_roots, &ctx.build_dir, &lib_env) + .await?; // 11. Run the shared sequential build pipeline. pipeline::run_sequential_build_with_libs( @@ -319,6 +322,7 @@ impl BuildOrchestrator for NxpLpcOrchestrator { "NXPLPC", start, ) + .await } } diff --git a/crates/fbuild-build/src/parallel.rs b/crates/fbuild-build/src/parallel.rs index 7c8c1fc5..84ee9f14 100644 --- a/crates/fbuild-build/src/parallel.rs +++ b/crates/fbuild-build/src/parallel.rs @@ -1,13 +1,16 @@ //! Parallel source file compilation. //! -//! Uses `std::thread::scope` with a work-stealing pattern to compile -//! source files across N threads. No external dependencies needed. +//! FastLED/fbuild#820 (Phase B of #813): converted from +//! `std::thread::scope` work-stealing to `tokio::task::JoinSet` so +//! the per-TU `Compiler::compile` futures can `.await` the embedded +//! `ZccacheService` directly, with no `Handle::block_on` and no +//! dedicated OS threads for compile dispatch. use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::Mutex; +use std::sync::Arc; use fbuild_core::{BuildLog, FbuildError, Result}; +use tokio::sync::Semaphore; use crate::compiler::{Compiler, CompilerBase}; use crate::flag_overlay::LanguageExtraFlags; @@ -32,20 +35,28 @@ pub struct ParallelCompileResult { pub warnings: Vec, } -/// Compile source files in parallel using a thread pool. +/// Compile source files in parallel, gating concurrency with a +/// `tokio::sync::Semaphore`. /// -/// Spawns up to `jobs` threads, each pulling work from a shared queue. -/// Stops on first compilation error. -/// Returns object file paths (in source order) and collected warnings. +/// Spawns each per-file compile as a `JoinSet` task; the semaphore +/// permits cap concurrent in-flight compiles at `jobs`. Stops on first +/// compilation error and returns object file paths (in source order) +/// plus collected warnings. /// -/// If `build_log` is provided, progress milestones are streamed to it. -pub fn compile_sources_parallel( - compiler: &dyn Compiler, +/// FastLED/fbuild#820 (Phase B of #813): replaces the old +/// `std::thread::scope` work-stealing loop. The borrowed `&dyn +/// Compiler` is held across `.await` points safely because +/// `compile_sources_parallel` is `async fn` and the per-task futures +/// borrow `compiler` for the duration of the JoinSet — the outer fn +/// `.await`s every task before returning, so the borrow is alive +/// throughout. +pub async fn compile_sources_parallel( + compiler: &(dyn Compiler + Send + Sync), sources: &[PathBuf], build_dir: &Path, extra_flags: &LanguageExtraFlags, jobs: usize, - build_log: Option<&Mutex>, + build_log: Option<&std::sync::Mutex>, ) -> Result { // Build work list: (source, object) pairs needing rebuild let mut work: Vec<(PathBuf, PathBuf)> = Vec::new(); @@ -69,87 +80,102 @@ pub fn compile_sources_parallel( } let total = work.len(); - let thread_count = jobs.min(total); - tracing::info!("compiling {} files with {} threads", total, thread_count); - - let work_iter = Mutex::new(work.into_iter()); - let first_error: Mutex> = Mutex::new(None); - let compiled_count = AtomicUsize::new(0); - let all_warnings: Mutex> = Mutex::new(Vec::new()); - - std::thread::scope(|scope| { - let handles: Vec<_> = (0..thread_count) - .map(|_| { - scope.spawn(|| { - let mut local_warnings: Vec = Vec::new(); - loop { - // Check for early termination - if first_error.lock().unwrap().is_some() { - break; - } - - let job = work_iter.lock().unwrap().next(); - let (source, obj) = match job { - Some(j) => j, - None => break, - }; - - let source_flags = extra_flags.for_source(&source); - match compiler.compile(&source, &obj, &source_flags) { - Ok(result) if result.success => { - let stderr = result.stderr.trim().to_string(); - if !stderr.is_empty() { - local_warnings.push(stderr); - } - let count = compiled_count.fetch_add(1, Ordering::Relaxed) + 1; - if count % 20 == 0 || count == total { - tracing::info!("[{}/{}] compiled", count, total); - if let Some(log) = build_log { - if let Ok(mut log) = log.lock() { - log.push(format!("Compiled {}/{} files", count, total)); - } - } - } - } - Ok(result) => { - let mut err = first_error.lock().unwrap(); - if err.is_none() { - *err = Some(format!( - "compilation failed for {}:\n{}", - source.display(), - result.stderr - )); - } - } - Err(e) => { - let mut err = first_error.lock().unwrap(); - if err.is_none() { - *err = Some(e.to_string()); - } + let parallelism = jobs.min(total).max(1); + tracing::info!( + "compiling {} files with {} concurrent tasks", + total, + parallelism + ); + + let semaphore = Arc::new(Semaphore::new(parallelism)); + let compiled_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let mut warnings: Vec = Vec::new(); + let mut first_error: Option = None; + + // `JoinSet>` lets us cancel pending tasks the moment + // the first error appears. We accumulate Result outcomes and bail + // after draining. + let mut tasks: tokio::task::JoinSet, String>> = + tokio::task::JoinSet::new(); + + // SAFETY: we extend the borrow of `compiler` / `extra_flags` / + // `build_log` to `'static` for the duration of the JoinSet's + // lifetime. The outer `async fn` awaits every spawned task before + // returning, so the borrows are alive for as long as the tasks + // execute. `transmute` is the standard idiom for this scoped-task + // pattern in tokio (no scoped tasks in tokio today). + let compiler_ptr: &'static (dyn Compiler + Send + Sync) = + unsafe { std::mem::transmute(compiler) }; + let extra_flags_ptr: &'static LanguageExtraFlags = unsafe { std::mem::transmute(extra_flags) }; + let build_log_ptr: Option<&'static std::sync::Mutex> = + unsafe { std::mem::transmute(build_log) }; + + for (source, obj) in work.into_iter() { + let sem = semaphore.clone(); + let counter = compiled_count.clone(); + tasks.spawn(async move { + // Acquire permit; if Acquired returns Err, semaphore was closed + // (only happens on shutdown — propagate as immediate-error). + let _permit = sem + .acquire() + .await + .map_err(|e| format!("semaphore closed: {e}"))?; + + let source_flags = extra_flags_ptr.for_source(&source); + match compiler_ptr.compile(&source, &obj, &source_flags).await { + Ok(result) if result.success => { + let stderr = result.stderr.trim().to_string(); + let count = + counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; + if count % 20 == 0 || count == total { + tracing::info!("[{}/{}] compiled", count, total); + if let Some(log) = build_log_ptr { + if let Ok(mut log) = log.lock() { + log.push(format!("Compiled {}/{} files", count, total)); } } } - // Merge local warnings into shared collection - if !local_warnings.is_empty() { - all_warnings.lock().unwrap().extend(local_warnings); + if stderr.is_empty() { + Ok(None) + } else { + Ok(Some(stderr)) } - }) - }) - .collect(); + } + Ok(result) => Err(format!( + "compilation failed for {}:\n{}", + source.display(), + result.stderr + )), + Err(e) => Err(e.to_string()), + } + }); + } - for handle in handles { - handle.join().unwrap(); + while let Some(joined) = tasks.join_next().await { + match joined { + Ok(Ok(Some(warning))) => warnings.push(warning), + Ok(Ok(None)) => {} + Ok(Err(msg)) => { + if first_error.is_none() { + first_error = Some(msg); + // Abort remaining tasks; we already have an error. + tasks.abort_all(); + } + } + Err(join_err) => { + if first_error.is_none() { + first_error = Some(format!("compile task panicked or was cancelled: {join_err}")); + tasks.abort_all(); + } + } } - }); + } - if let Some(error) = first_error.into_inner().unwrap() { + if let Some(error) = first_error { return Err(FbuildError::BuildFailed(error)); } - Ok(ParallelCompileResult { - objects, - warnings: all_warnings.into_inner().unwrap(), - }) + Ok(ParallelCompileResult { objects, warnings }) } #[cfg(test)] diff --git a/crates/fbuild-build/src/pipeline/compile.rs b/crates/fbuild-build/src/pipeline/compile.rs index 18ba7efb..d9456f2e 100644 --- a/crates/fbuild-build/src/pipeline/compile.rs +++ b/crates/fbuild-build/src/pipeline/compile.rs @@ -15,7 +15,7 @@ use crate::flag_overlay::LanguageExtraFlags; /// [`super::run_sequential_build_with_libs`]; ESP32 calls `compile_sources_parallel` /// directly because it interleaves multiple compile phases through the same /// log Mutex. -pub fn compile_sources( +pub async fn compile_sources( compiler: &dyn Compiler, sources: &[PathBuf], build_dir: &Path, @@ -30,7 +30,8 @@ pub fn compile_sources( extra_flags, jobs, Some(build_log), - )?; + ) + .await?; if !result.warnings.is_empty() { let mut log = build_log.lock().unwrap(); for w in &result.warnings { @@ -45,7 +46,7 @@ pub fn compile_sources( /// Each library's source files are compiled in parallel via /// [`crate::parallel::compile_sources_parallel`]. Libraries themselves are /// processed one after another so the per-lib `jobs` budget isn't oversubscribed. -pub fn compile_local_libraries( +pub async fn compile_local_libraries( compiler: &dyn Compiler, project_dir: &Path, build_dir: &Path, @@ -96,6 +97,7 @@ pub fn compile_local_libraries( jobs, Some(build_log), ) + .await .map_err(|e| { fbuild_core::FbuildError::BuildFailed(format!( "local library '{}' compilation failed: {}", @@ -163,13 +165,15 @@ pub fn generate_compile_db( } /// Log the version of a GCC toolchain by running `gcc -dumpversion`. -pub fn log_toolchain_version(gcc_path: &Path, label: &str, build_log: &mut BuildLog) { +pub async fn log_toolchain_version(gcc_path: &Path, label: &str, build_log: &mut BuildLog) { if let Ok(ver_out) = fbuild_core::subprocess::run_command( &[gcc_path.to_string_lossy().as_ref(), "-dumpversion"], None, None, None, - ) { + ) + .await + { let version = ver_out.stdout.trim().to_string(); if !version.is_empty() { crate::build_output::log_toolchain_version(build_log, label, &version); diff --git a/crates/fbuild-build/src/pipeline/context.rs b/crates/fbuild-build/src/pipeline/context.rs index f47ba215..f11eb638 100644 --- a/crates/fbuild-build/src/pipeline/context.rs +++ b/crates/fbuild-build/src/pipeline/context.rs @@ -45,8 +45,8 @@ impl BuildContext { /// /// Takes `&BuildParams` so that new fields (e.g. `src_dir`) flow through /// automatically — orchestrators just pass `params` without listing every field. - pub fn new(params: &BuildParams) -> Result { - Self::new_with_perf(params, None) + pub async fn new(params: &BuildParams) -> Result { + Self::new_with_perf(params, None).await } /// Variant that records phase timings into an optional `PerfTimer`. @@ -54,7 +54,7 @@ impl BuildContext { /// Orchestrators that want per-phase visibility (see [`crate::perf_log`]) /// pass in a shared timer. Callers that don't care get zero overhead by /// passing `None`. - pub fn new_with_perf( + pub async fn new_with_perf( params: &BuildParams, mut perf: Option<&mut crate::perf_log::PerfTimer>, ) -> Result { @@ -70,7 +70,8 @@ impl BuildContext { let config = fbuild_config::PlatformIOConfig::from_path_with_overrides(&ini_path, pio_overrides)?; let overlay = - crate::script_runtime::resolve_extra_script_overlay(project_dir, env_name, &config)?; + crate::script_runtime::resolve_extra_script_overlay(project_dir, env_name, &config) + .await?; if let Some(p) = perf.as_mut() { p.record("config-parse", t0.elapsed()); } diff --git a/crates/fbuild-build/src/pipeline/library.rs b/crates/fbuild-build/src/pipeline/library.rs index d711d194..42ac20c4 100644 --- a/crates/fbuild-build/src/pipeline/library.rs +++ b/crates/fbuild-build/src/pipeline/library.rs @@ -148,7 +148,7 @@ pub fn add_extra_library_include_dirs(library_roots: &[PathBuf], include_dirs: & } /// Compile extra library roots from `lib_extra_dirs` into archives. -pub fn compile_extra_libraries( +pub async fn compile_extra_libraries( library_roots: &[PathBuf], build_dir: &Path, env: &LibraryBuildEnv<'_>, @@ -186,7 +186,9 @@ pub fn compile_extra_libraries( env.verbose, env.jobs, env.compiler_cache, - ) { + ) + .await + { Ok(Some(archive)) => archives.push(archive), Ok(None) => {} Err(e) => { @@ -210,7 +212,7 @@ pub fn compile_extra_libraries( /// library archive was produced. /// /// Matches PlatformIO's project-as-library convention; see ISSUES.md Issue 1. -pub fn compile_project_as_library( +pub async fn compile_project_as_library( project_dir: &Path, src_dir: &Path, build_dir: &Path, @@ -288,7 +290,9 @@ pub fn compile_project_as_library( env.verbose, env.jobs, env.compiler_cache, - ) { + ) + .await + { Ok(Some(archive)) => { tracing::info!( "project-as-library compiled: {} sources -> {}", @@ -417,8 +421,8 @@ mod project_as_library_tests { } } - #[test] - fn test_returns_none_when_not_a_library() { + #[tokio::test] + async fn test_returns_none_when_not_a_library() { let tmp = tempfile::TempDir::new().unwrap(); let project_dir = tmp.path(); // No library.json or library.properties @@ -437,12 +441,13 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - ); + ) + .await; assert!(matches!(result, Ok(None))); } - #[test] - fn test_returns_none_when_src_dir_equals_project_src() { + #[tokio::test] + async fn test_returns_none_when_src_dir_equals_project_src() { // Library project being built normally (not as an example) — must // NOT compile project-as-library or we'd double-compile sketch sources. let tmp = tempfile::TempDir::new().unwrap(); @@ -463,12 +468,13 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - ); + ) + .await; assert!(matches!(result, Ok(None))); } - #[test] - fn test_returns_none_when_src_dir_equals_project_dir() { + #[tokio::test] + async fn test_returns_none_when_src_dir_equals_project_dir() { // BuildContext::new falls back to project_dir when the resolved src // dir doesn't exist. In that fallback, the sketch scanner walks // project_dir recursively and would pick up library sources — so we @@ -490,12 +496,13 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - ); + ) + .await; assert!(matches!(result, Ok(None))); } - #[test] - fn test_returns_none_when_no_src_dir() { + #[tokio::test] + async fn test_returns_none_when_no_src_dir() { // library.properties exists but no src/ directory. let tmp = tempfile::TempDir::new().unwrap(); let project_dir = tmp.path(); @@ -513,12 +520,13 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - ); + ) + .await; assert!(matches!(result, Ok(None))); } - #[test] - fn test_returns_none_when_header_only() { + #[tokio::test] + async fn test_returns_none_when_header_only() { // library.json + src/ but only headers — header-only library, not // an error, just nothing to compile. let tmp = tempfile::TempDir::new().unwrap(); @@ -541,12 +549,13 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - ); + ) + .await; assert!(matches!(result, Ok(None))); } - #[test] - fn test_returns_none_on_collision_with_lib_dir() { + #[tokio::test] + async fn test_returns_none_on_collision_with_lib_dir() { // If a user has both library.json AND lib//, the lib/ // version wins (matches PlatformIO behavior). Must skip project-as- // library to prevent two libfastled.a archives at link time. @@ -575,12 +584,13 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &existing, - ); + ) + .await; assert!(matches!(result, Ok(None))); } - #[test] - fn test_attempts_compile_when_building_example() { + #[tokio::test] + async fn test_attempts_compile_when_building_example() { // The positive case: library project + sketch lives elsewhere + has // sources + no name collision → must reach the compile path. We // verify this by passing a bogus gcc path and asserting the function @@ -608,7 +618,8 @@ mod project_as_library_tests { &project_dir.join("build"), &env, &HashSet::new(), - ); + ) + .await; // Must NOT be Ok(None) — that would mean a guard skipped compile. // Either Err (bogus tool failed) or Ok(Some(_)) (impossible without // a real toolchain) is acceptable. diff --git a/crates/fbuild-build/src/pipeline/sequential.rs b/crates/fbuild-build/src/pipeline/sequential.rs index aca2cfff..54c727a7 100644 --- a/crates/fbuild-build/src/pipeline/sequential.rs +++ b/crates/fbuild-build/src/pipeline/sequential.rs @@ -33,9 +33,9 @@ use super::link::{assemble_build_result, handle_link_result}; /// with the rest of the build. See [`compile_project_as_library`] and /// ISSUES.md Issue 1. #[allow(clippy::too_many_arguments)] -pub fn run_sequential_build_with_libs( - compiler: &dyn Compiler, - linker: &dyn crate::linker::Linker, +pub async fn run_sequential_build_with_libs( + compiler: &(dyn Compiler + Send + Sync), + linker: &(dyn crate::linker::Linker + Send + Sync), mut ctx: BuildContext, params: &BuildParams, sources: &SourceCollection, @@ -162,7 +162,8 @@ pub fn run_sequential_build_with_libs( &user_overlay, jobs, &build_log_mutex, - )? + ) + .await? }; let variant_objects = { let _g = perf.phase("compile-variant"); @@ -173,7 +174,8 @@ pub fn run_sequential_build_with_libs( &user_overlay, jobs, &build_log_mutex, - )? + ) + .await? }; core_objects.extend(variant_objects); if let Some(cache) = core_cache.as_ref() { @@ -208,7 +210,8 @@ pub fn run_sequential_build_with_libs( &src_overlay, jobs, &build_log_mutex, - )? + ) + .await? }; // Compile local libraries (lib/* — loose objects, LTO-safe; per-lib parallel) @@ -221,7 +224,8 @@ pub fn run_sequential_build_with_libs( &src_overlay, jobs, &build_log_mutex, - )? + ) + .await? }; // Unwrap the build log Mutex back into the context for the remaining @@ -255,7 +259,8 @@ pub fn run_sequential_build_with_libs( &ctx.build_dir, env, &existing_lib_names, - )? + ) + .await? } else { None } @@ -303,7 +308,8 @@ pub fn run_sequential_build_with_libs( bloat_analysis: params.bloat_analysis, }, params.symbol_analysis, - )? + ) + .await? }; // Emit build_info_.json (and the generic fallback) so downstream diff --git a/crates/fbuild-build/src/renesas/mod.rs b/crates/fbuild-build/src/renesas/mod.rs index a334b5d3..c3b9a457 100644 --- a/crates/fbuild-build/src/renesas/mod.rs +++ b/crates/fbuild-build/src/renesas/mod.rs @@ -1,4 +1,4 @@ -//! Renesas RA platform build support (Arduino UNO R4, etc.) +//! Renesas RA platform build support (Arduino UNO R4, etc.) pub mod mcu_config; pub mod orchestrator; @@ -12,15 +12,16 @@ pub use renesas_linker::RenesasLinker; /// Renesas RA platform support. pub struct RenesasPlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for RenesasPlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("ARM toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/renesas/orchestrator.rs b/crates/fbuild-build/src/renesas/orchestrator.rs index 9698d995..aa46922d 100644 --- a/crates/fbuild-build/src/renesas/orchestrator.rs +++ b/crates/fbuild-build/src/renesas/orchestrator.rs @@ -1,4 +1,4 @@ -//! Renesas RA build orchestrator — wires together config, packages, compiler, linker. +//! Renesas RA build orchestrator — wires together config, packages, compiler, linker. //! //! Build phases: //! 1. Parse platformio.ini @@ -58,21 +58,22 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +#[async_trait::async_trait] impl BuildOrchestrator for RenesasOrchestrator { fn platform(&self) -> Platform { Platform::RenesasRa } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); let compiler_cache: Option = None; // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = pipeline::BuildContext::new(params)?; + let mut ctx = pipeline::BuildContext::new(params).await?; // 3. Ensure ARM GCC toolchain let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-gcc toolchain at {}", toolchain_dir.display()); use fbuild_packages::Toolchain; @@ -80,11 +81,12 @@ impl BuildOrchestrator for RenesasOrchestrator { &toolchain.get_gcc_path(), "arm-none-eabi-gcc", &mut ctx.build_log, - ); + ) + .await; // 4. Ensure Renesas cores (ArduinoCore-renesas) let framework = fbuild_packages::library::RenesasCores::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("Renesas cores at {}", framework_dir.display()); // 5. Scan sources @@ -256,7 +258,7 @@ impl BuildOrchestrator for RenesasOrchestrator { TargetArchitecture::Arm, "Renesas RA", start, - )?; + ).await?; if build_result.success && !params.compiledb_only diff --git a/crates/fbuild-build/src/renesas/renesas_compiler.rs b/crates/fbuild-build/src/renesas/renesas_compiler.rs index d4444862..8314aa41 100644 --- a/crates/fbuild-build/src/renesas/renesas_compiler.rs +++ b/crates/fbuild-build/src/renesas/renesas_compiler.rs @@ -157,8 +157,9 @@ fn is_c_source(source: &Path) -> bool { .unwrap_or(false) } +#[async_trait::async_trait] impl Compiler for RenesasCompiler { - fn compile_one( + async fn compile_one( &self, compiler_path: &Path, source: &Path, @@ -200,6 +201,7 @@ impl Compiler for RenesasCompiler { None, &[], ) + .await } fn gcc_path(&self) -> &Path { diff --git a/crates/fbuild-build/src/renesas/renesas_linker.rs b/crates/fbuild-build/src/renesas/renesas_linker.rs index e6eaae07..b06892cb 100644 --- a/crates/fbuild-build/src/renesas/renesas_linker.rs +++ b/crates/fbuild-build/src/renesas/renesas_linker.rs @@ -1,4 +1,4 @@ -//! Renesas RA ARM linker implementation. +//! Renesas RA ARM linker implementation. //! //! Links ARM Cortex-M4 object files into firmware.elf, converts to firmware.bin, //! and reports size using arm-none-eabi-size. @@ -54,12 +54,14 @@ impl RenesasLinker { } } +#[async_trait::async_trait] impl Linker for RenesasLinker { - fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { + async fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "arm-none-eabi-ar") + .await } - fn link( + async fn link( &self, objects: &[PathBuf], archives: &[PathBuf], @@ -122,7 +124,7 @@ impl Linker for RenesasLinker { tracing::info!("link: {}", args.join(" ")); } - // GCC LTO temp dir for MSYS-safe paths — see FastLED/fbuild#261. + // GCC LTO temp dir for MSYS-safe paths — see FastLED/fbuild#261. let lto_env = fbuild_core::subprocess::link_env_for_build(output_dir)?; let env_slice: Vec<(&str, &str)> = lto_env .iter() @@ -130,7 +132,7 @@ impl Linker for RenesasLinker { .collect(); let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, Some(&env_slice), None)?; + let result = run_command(&args_ref, None, Some(&env_slice), None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -142,7 +144,7 @@ impl Linker for RenesasLinker { Ok(elf_path) } - fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { + async fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { crate::linker::LinkerBase::objcopy_firmware( &self.objcopy_path, elf_path, @@ -151,6 +153,7 @@ impl Linker for RenesasLinker { &self.mcu_config.objcopy.remove_sections, "arm-none-eabi-objcopy", ) + .await } fn size_tool_path(&self) -> &Path { @@ -169,7 +172,7 @@ impl Linker for RenesasLinker { Some(&self.gcc_path) } - fn report_size(&self, elf_path: &Path) -> Result { + async fn report_size(&self, elf_path: &Path) -> Result { crate::linker::LinkerBase::report_size( &self.size_path, elf_path, @@ -177,6 +180,7 @@ impl Linker for RenesasLinker { self.max_ram, "arm-none-eabi-size", ) + .await } } diff --git a/crates/fbuild-build/src/rp2040/mod.rs b/crates/fbuild-build/src/rp2040/mod.rs index 34136015..6541df13 100644 --- a/crates/fbuild-build/src/rp2040/mod.rs +++ b/crates/fbuild-build/src/rp2040/mod.rs @@ -1,4 +1,4 @@ -//! RP2040/RP2350 platform build support (Raspberry Pi Pico, etc.) +//! RP2040/RP2350 platform build support (Raspberry Pi Pico, etc.) pub mod mcu_config; pub mod orchestrator; @@ -8,15 +8,16 @@ pub use orchestrator::Rp2040Orchestrator; /// RP2040 platform support. pub struct Rp2040PlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for Rp2040PlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("ARM toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/rp2040/orchestrator.rs b/crates/fbuild-build/src/rp2040/orchestrator.rs index bc341fea..2a01c5ce 100644 --- a/crates/fbuild-build/src/rp2040/orchestrator.rs +++ b/crates/fbuild-build/src/rp2040/orchestrator.rs @@ -1,4 +1,4 @@ -//! RP2040/RP2350 build orchestrator — wires together config, packages, compiler, linker. +//! RP2040/RP2350 build orchestrator — wires together config, packages, compiler, linker. //! //! Build phases: //! 1. Parse platformio.ini @@ -58,17 +58,18 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +#[async_trait::async_trait] impl BuildOrchestrator for Rp2040Orchestrator { fn platform(&self) -> Platform { Platform::RaspberryPi } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); let compiler_cache: Option = None; // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = pipeline::BuildContext::new(params)?; + let mut ctx = pipeline::BuildContext::new(params).await?; // Compute eh_frame strip policy once per build (FastLED/fbuild#244). let eh_frame_policy = @@ -76,7 +77,7 @@ impl BuildOrchestrator for Rp2040Orchestrator { // 3. Ensure the arduino-pico-matched pqt-gcc toolchain let toolchain = fbuild_packages::toolchain::Rp2040PqtToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("rp2040 pqt-gcc toolchain at {}", toolchain_dir.display()); use fbuild_packages::Toolchain; @@ -84,11 +85,12 @@ impl BuildOrchestrator for Rp2040Orchestrator { &toolchain.get_gcc_path(), "arm-none-eabi-gcc", &mut ctx.build_log, - ); + ) + .await; // 4. Ensure RP2040 cores (arduino-pico by earlephilhower) let framework = fbuild_packages::library::Rp2040Cores::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("RP2040 cores at {}", framework_dir.display()); let board_id = ctx .config @@ -269,7 +271,8 @@ impl BuildOrchestrator for Rp2040Orchestrator { .and_then(|props| props.get("boot2")) .map(String::as_str) .unwrap_or("boot2_w25q080_2_padded_checksum"), - )?; + ) + .await?; let mut mcu_config = mcu_config; add_rp_linker_flags(&framework_dir, &ctx.board.mcu, &mut mcu_config); let linker = ArmLinker::new( @@ -321,7 +324,7 @@ impl BuildOrchestrator for Rp2040Orchestrator { TargetArchitecture::Arm, "RP2040", start, - )?; + ).await?; if build_result.success && !params.compiledb_only @@ -630,7 +633,7 @@ fn generate_linker_script( Ok(output) } -fn compile_boot2_object( +async fn compile_boot2_object( compiler: &ArmCompiler, framework_dir: &Path, build_dir: &Path, @@ -683,7 +686,8 @@ fn compile_boot2_object( &boot2_object, &boot2_flags, &extra_flags, - )?; + ) + .await?; if !result.success { return Err(fbuild_core::FbuildError::BuildFailed(format!( "RP2040 boot2 compile failed for {}:\n{}", diff --git a/crates/fbuild-build/src/sam/mod.rs b/crates/fbuild-build/src/sam/mod.rs index b8339fa5..1021069f 100644 --- a/crates/fbuild-build/src/sam/mod.rs +++ b/crates/fbuild-build/src/sam/mod.rs @@ -1,4 +1,4 @@ -//! SAM platform build support (Atmel SAM3X8E / Arduino Due) +//! SAM platform build support (Atmel SAM3X8E / Arduino Due) pub mod mcu_config; pub mod orchestrator; @@ -12,15 +12,16 @@ pub use sam_linker::SamLinker; /// SAM platform support. pub struct SamPlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for SamPlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("ARM toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/sam/orchestrator.rs b/crates/fbuild-build/src/sam/orchestrator.rs index ed4e09ab..19f9926a 100644 --- a/crates/fbuild-build/src/sam/orchestrator.rs +++ b/crates/fbuild-build/src/sam/orchestrator.rs @@ -1,9 +1,9 @@ -//! SAM/SAMD build orchestrator — wires together config, packages, compiler, linker. +//! SAM/SAMD build orchestrator — wires together config, packages, compiler, linker. //! //! Handles both SAM (Due/SAM3X) and SAMD (SAMD21/SAMD51) boards under the //! `atmelsam` platform. Selects the correct Arduino core: -//! - SAM3X → ArduinoCore-sam -//! - SAMD21/51 → ArduinoCore-samd (Adafruit fork) +//! - SAM3X → ArduinoCore-sam +//! - SAMD21/51 → ArduinoCore-samd (Adafruit fork) //! //! Build phases: //! 1. Parse platformio.ini @@ -68,21 +68,22 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +#[async_trait::async_trait] impl BuildOrchestrator for SamOrchestrator { fn platform(&self) -> Platform { Platform::AtmelSam } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); let compiler_cache: Option = None; // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = pipeline::BuildContext::new(params)?; + let mut ctx = pipeline::BuildContext::new(params).await?; // 3. Ensure ARM GCC toolchain let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-none-eabi toolchain at {}", toolchain_dir.display()); use fbuild_packages::Toolchain; @@ -90,14 +91,15 @@ impl BuildOrchestrator for SamOrchestrator { &toolchain.get_gcc_path(), "arm-none-eabi-gcc", &mut ctx.build_log, - ); + ) + .await; // 4. Ensure correct Arduino core based on MCU family let (framework_dir, core_dir, variant_dir, linker_script_path, system_includes) = if is_samd_mcu(&ctx.board.mcu) { - install_samd_core(params, &ctx.board.core, &ctx.board.variant)? + install_samd_core(params, &ctx.board.core, &ctx.board.variant).await? } else { - install_sam_core(params, &ctx.board.core, &ctx.board.variant)? + install_sam_core(params, &ctx.board.core, &ctx.board.variant).await? }; let build_dir = &ctx.build_dir; @@ -276,7 +278,7 @@ impl BuildOrchestrator for SamOrchestrator { TargetArchitecture::Arm, "SAM", start, - )?; + ).await?; if build_result.success && !params.compiledb_only @@ -301,13 +303,13 @@ impl BuildOrchestrator for SamOrchestrator { /// Install ArduinoCore-sam for classic SAM3X boards (Due). /// /// Returns (framework_dir, core_dir, variant_dir, linker_script, system_includes). -fn install_sam_core( +async fn install_sam_core( params: &BuildParams, core_name: &str, variant_name: &str, ) -> Result<(PathBuf, PathBuf, PathBuf, PathBuf, Vec)> { let framework = fbuild_packages::library::SamCores::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("SAM cores at {}", framework_dir.display()); let core_dir = framework.get_core_dir(core_name); @@ -336,13 +338,13 @@ fn install_sam_core( /// Install Adafruit ArduinoCore-samd for SAMD21/SAMD51 boards. /// /// Returns (framework_dir, core_dir, variant_dir, linker_script, system_includes). -fn install_samd_core( +async fn install_samd_core( params: &BuildParams, core_name: &str, variant_name: &str, ) -> Result<(PathBuf, PathBuf, PathBuf, PathBuf, Vec)> { let framework = fbuild_packages::library::SamdCores::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("SAMD cores at {}", framework_dir.display()); let core_dir = framework.get_core_dir(core_name); @@ -351,11 +353,11 @@ fn install_samd_core( // SAMD core needs external CMSIS and CMSIS-Atmel packages for device headers let cmsis = fbuild_packages::library::CmsisFramework::new(¶ms.project_dir); - let cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis)?; + let cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis).await?; tracing::info!("CMSIS at {}", cmsis_dir.display()); let cmsis_atmel = fbuild_packages::library::CmsisAtmel::new(¶ms.project_dir); - let _cmsis_atmel_dir = fbuild_packages::Package::ensure_installed(&cmsis_atmel)?; + let _cmsis_atmel_dir = fbuild_packages::Package::ensure_installed(&cmsis_atmel).await?; tracing::info!("CMSIS-Atmel installed"); let mut includes = vec![ @@ -370,7 +372,7 @@ fn install_samd_core( // `BoardConfig::get_include_paths` already emits // `framework_root/cores/`, which for Adafruit SAMD boards is // a vendor-brand label (e.g. "adafruit") that the framework doesn't - // actually ship as a directory — only `cores/arduino/` exists. That + // actually ship as a directory — only `cores/arduino/` exists. That // literal path becomes a phantom `-I` and `#include "Arduino.h"` / // `#include "WVariant.h"` lookups miss. `core_dir` here was produced by // `SamdCores::get_core_dir` which falls back to `cores/arduino/` when the diff --git a/crates/fbuild-build/src/sam/sam_compiler.rs b/crates/fbuild-build/src/sam/sam_compiler.rs index 81695843..baac6534 100644 --- a/crates/fbuild-build/src/sam/sam_compiler.rs +++ b/crates/fbuild-build/src/sam/sam_compiler.rs @@ -75,8 +75,9 @@ impl SamCompiler { } } +#[async_trait::async_trait] impl Compiler for SamCompiler { - fn compile_one( + async fn compile_one( &self, compiler_path: &Path, source: &Path, @@ -96,6 +97,7 @@ impl Compiler for SamCompiler { None, &[], ) + .await } fn gcc_path(&self) -> &Path { diff --git a/crates/fbuild-build/src/sam/sam_linker.rs b/crates/fbuild-build/src/sam/sam_linker.rs index e58926d6..6f899b97 100644 --- a/crates/fbuild-build/src/sam/sam_linker.rs +++ b/crates/fbuild-build/src/sam/sam_linker.rs @@ -1,4 +1,4 @@ -//! SAM ARM linker implementation. +//! SAM ARM linker implementation. //! //! Links ARM Cortex-M3 object files into firmware.elf, converts to firmware.bin, //! and reports size using arm-none-eabi-size. @@ -68,12 +68,14 @@ impl SamLinker { } } +#[async_trait::async_trait] impl Linker for SamLinker { - fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { + async fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "arm-none-eabi-ar") + .await } - fn link( + async fn link( &self, objects: &[PathBuf], archives: &[PathBuf], @@ -138,7 +140,7 @@ impl Linker for SamLinker { tracing::info!("link: {}", args.join(" ")); } - // GCC LTO temp dir for MSYS-safe paths — see FastLED/fbuild#261. + // GCC LTO temp dir for MSYS-safe paths — see FastLED/fbuild#261. let lto_env = fbuild_core::subprocess::link_env_for_build(output_dir)?; let env_slice: Vec<(&str, &str)> = lto_env .iter() @@ -146,7 +148,7 @@ impl Linker for SamLinker { .collect(); let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, Some(&env_slice), None)?; + let result = run_command(&args_ref, None, Some(&env_slice), None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -158,7 +160,7 @@ impl Linker for SamLinker { Ok(elf_path) } - fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { + async fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { crate::linker::LinkerBase::objcopy_firmware( &self.objcopy_path, elf_path, @@ -167,6 +169,7 @@ impl Linker for SamLinker { &self.mcu_config.objcopy.remove_sections, "arm-none-eabi-objcopy", ) + .await } fn size_tool_path(&self) -> &Path { @@ -185,7 +188,7 @@ impl Linker for SamLinker { Some(&self.gcc_path) } - fn report_size(&self, elf_path: &Path) -> Result { + async fn report_size(&self, elf_path: &Path) -> Result { crate::linker::LinkerBase::report_size( &self.size_path, elf_path, @@ -193,6 +196,7 @@ impl Linker for SamLinker { self.max_ram, "arm-none-eabi-size", ) + .await } } diff --git a/crates/fbuild-build/src/script_runtime.rs b/crates/fbuild-build/src/script_runtime.rs index cc54dcfa..ea26caa1 100644 --- a/crates/fbuild-build/src/script_runtime.rs +++ b/crates/fbuild-build/src/script_runtime.rs @@ -37,7 +37,7 @@ struct ScriptRuntimeInput<'a> { platformio_home: String, } -pub fn resolve_extra_script_overlay( +pub async fn resolve_extra_script_overlay( project_dir: &Path, env_name: &str, config: &fbuild_config::PlatformIOConfig, @@ -61,7 +61,7 @@ pub fn resolve_extra_script_overlay( .to_string(), }; - let python = find_python().ok_or_else(|| { + let python = find_python().await.ok_or_else(|| { fbuild_core::FbuildError::BuildFailed( "extra_scripts detected but no Python interpreter was found; \ install Python or use --platformio" @@ -109,6 +109,7 @@ pub fn resolve_extra_script_overlay( argv.push(harness_path_str.as_ref()); argv.push(input_path_str.as_ref()); let output = fbuild_core::subprocess::run_command(&argv, Some(project_dir), None, None) + .await .map_err(|e| { fbuild_core::FbuildError::BuildFailed(format!( "failed to run extra_scripts runtime via '{}': {}", @@ -284,7 +285,7 @@ fn libs_to_flags( Ok(flags) } -fn find_python() -> Option> { +async fn find_python() -> Option> { let candidates: &[&[&str]] = if cfg!(windows) { &[&["python"], &["py", "-3"]] } else { @@ -294,7 +295,7 @@ fn find_python() -> Option> { for candidate in candidates { let mut argv: Vec<&str> = candidate.to_vec(); argv.push("--version"); - if let Ok(output) = fbuild_core::subprocess::run_command(&argv, None, None, None) { + if let Ok(output) = fbuild_core::subprocess::run_command(&argv, None, None, None).await { if output.success() { return Some(candidate.iter().map(|s| (*s).to_string()).collect()); } diff --git a/crates/fbuild-build/src/script_runtime_tests.rs b/crates/fbuild-build/src/script_runtime_tests.rs index 9de2e24c..b19352cb 100644 --- a/crates/fbuild-build/src/script_runtime_tests.rs +++ b/crates/fbuild-build/src/script_runtime_tests.rs @@ -1,4 +1,4 @@ -use super::*; +use super::*; use crate::flag_overlay::ScriptScopeState; use std::fs; @@ -27,10 +27,10 @@ extra_scripts = {} temp } -fn resolve_runtime_error(project_dir: &Path) -> String { +async fn resolve_runtime_error(project_dir: &Path) -> String { let config = fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")).unwrap(); - resolve_extra_script_overlay(project_dir, "demo", &config) + resolve_extra_script_overlay(project_dir, "demo", &config).await .unwrap_err() .to_string() } @@ -123,9 +123,9 @@ fn test_scope_to_link_overlay_maps_libpath_and_libs() { ); } -#[test] -fn test_resolve_extra_script_overlay_supports_dump_shim() { - if find_python().is_none() { +#[tokio::test] +async fn test_resolve_extra_script_overlay_supports_dump_shim() { + if find_python().await.is_none() { return; } @@ -158,16 +158,16 @@ env.Append(CPPDEFINES=[\"DUMP_SHIM_OK\"]) let config = fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")).unwrap(); // Pinned to MockEnv (see resolve_runtime_overlay note). - let overlay = resolve_extra_script_overlay(project_dir, "demo", &config).unwrap(); + let overlay = resolve_extra_script_overlay(project_dir, "demo", &config).await.unwrap(); assert!(overlay .global_compile .common .contains(&"-DDUMP_SHIM_OK".to_string())); } -#[test] -fn test_resolve_extra_script_overlay_supports_common_noop_scons_helpers() { - if find_python().is_none() { +#[tokio::test] +async fn test_resolve_extra_script_overlay_supports_common_noop_scons_helpers() { + if find_python().await.is_none() { return; } @@ -204,16 +204,16 @@ env.Append(CPPDEFINES=[\"HELPERS_SHIM_OK\"]) let config = fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")).unwrap(); // Pinned to MockEnv (see resolve_runtime_overlay note). - let overlay = resolve_extra_script_overlay(project_dir, "demo", &config).unwrap(); + let overlay = resolve_extra_script_overlay(project_dir, "demo", &config).await.unwrap(); assert!(overlay .global_compile .common .contains(&"-DHELPERS_SHIM_OK".to_string())); } -#[test] -fn test_resolve_extra_script_overlay_supports_board_config_shim() { - if find_python().is_none() { +#[tokio::test] +async fn test_resolve_extra_script_overlay_supports_board_config_shim() { + if find_python().await.is_none() { return; } @@ -247,16 +247,16 @@ env.Append(CPPDEFINES=[\"BOARD_CONFIG_SHIM_OK\"]) let config = fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")).unwrap(); // Pinned to MockEnv (see resolve_runtime_overlay note). - let overlay = resolve_extra_script_overlay(project_dir, "demo", &config).unwrap(); + let overlay = resolve_extra_script_overlay(project_dir, "demo", &config).await.unwrap(); assert!(overlay .global_compile .common .contains(&"-DBOARD_CONFIG_SHIM_OK".to_string())); } -#[test] -fn test_resolve_extra_script_overlay_supports_pio_platform_shim() { - if find_python().is_none() { +#[tokio::test] +async fn test_resolve_extra_script_overlay_supports_pio_platform_shim() { + if find_python().await.is_none() { return; } @@ -293,16 +293,16 @@ env.Append(CPPDEFINES=[\"PIO_PLATFORM_SHIM_OK\"]) let config = fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")).unwrap(); // Pinned to MockEnv (see resolve_runtime_overlay note). - let overlay = resolve_extra_script_overlay(project_dir, "demo", &config).unwrap(); + let overlay = resolve_extra_script_overlay(project_dir, "demo", &config).await.unwrap(); assert!(overlay .global_compile .common .contains(&"-DPIO_PLATFORM_SHIM_OK".to_string())); } -#[test] -fn test_resolve_extra_script_overlay_rejects_unsupported_script_prefix() { - if find_python().is_none() { +#[tokio::test] +async fn test_resolve_extra_script_overlay_rejects_unsupported_script_prefix() { + if find_python().await.is_none() { return; } @@ -313,7 +313,7 @@ fn test_resolve_extra_script_overlay_rejects_unsupported_script_prefix() { Import(\"env\") ", ); - let err = resolve_runtime_error(temp.path()); + let err = resolve_runtime_error(temp.path()).await; assert!( err.contains("unsupported extra_scripts prefix 'mid'"), "{err}" @@ -348,10 +348,12 @@ framework = arduino temp } -fn resolve_runtime_overlay(project_dir: &Path) -> BuildOverlay { +async fn resolve_runtime_overlay(project_dir: &Path) -> BuildOverlay { let config = fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")).unwrap(); - resolve_extra_script_overlay(project_dir, "demo", &config).unwrap() + resolve_extra_script_overlay(project_dir, "demo", &config) + .await + .unwrap() } // ---- SIMPLE tier ------------------------------------------------------ @@ -359,9 +361,9 @@ fn resolve_runtime_overlay(project_dir: &Path) -> BuildOverlay { /// Marlin `common-cxxflags.py`-style script: language-specific append, /// `GetBuildType()` gating, in-place `BUILD_FLAGS` append, and a no-op /// `AddPostAction`. Source: MarlinFirmware/Marlin buildroot scripts. -#[test] -fn test_shim_simple_marlin_cxxflags_style() { - if find_python().is_none() { +#[tokio::test] +async fn test_shim_simple_marlin_cxxflags_style() { + if find_python().await.is_none() { return; } @@ -382,7 +384,7 @@ env.AddPostAction(\"$PROGPATH\", lambda *a, **k: None) ", ); - let overlay = resolve_runtime_overlay(temp.path()); + let overlay = resolve_runtime_overlay(temp.path()).await; assert!( overlay .global_compile @@ -408,9 +410,9 @@ env.AddPostAction(\"$PROGPATH\", lambda *a, **k: None) /// Tuple-shaped `CPPDEFINES` appended in place via `__getitem__` must still /// emit `-Dkey=value`, not a malformed array entry. -#[test] -fn test_shim_simple_inplace_tuple_cppdefine() { - if find_python().is_none() { +#[tokio::test] +async fn test_shim_simple_inplace_tuple_cppdefine() { + if find_python().await.is_none() { return; } @@ -425,7 +427,7 @@ env.Append(CPPDEFINES=[\"PLAIN\"]) ", ); - let overlay = resolve_runtime_overlay(temp.path()); + let overlay = resolve_runtime_overlay(temp.path()).await; assert!( overlay .global_compile @@ -447,9 +449,9 @@ env.Append(CPPDEFINES=[\"PLAIN\"]) /// namf `platformio_script.py`-style script: obtains env via /// `from SCons.Script import DefaultEnvironment`, reads + rewrites /// `LINKFLAGS`, and registers a no-op post action. -#[test] -fn test_shim_medium_default_environment_linkflags() { - if find_python().is_none() { +#[tokio::test] +async fn test_shim_medium_default_environment_linkflags() { + if find_python().await.is_none() { return; } @@ -470,7 +472,7 @@ env.AddPostAction(\"$BUILD_DIR/firmware.bin\", after_build) ", ); - let overlay = resolve_runtime_overlay(temp.path()); + let overlay = resolve_runtime_overlay(temp.path()).await; assert!( overlay .link @@ -485,9 +487,9 @@ env.AddPostAction(\"$BUILD_DIR/firmware.bin\", after_build) /// note; under lite-SCons (the only backend post-#553 step 4) the value /// is stored on the construction env without a note. Either way the /// script must not hard-fail and the parallel flag mutation must land. -#[test] -fn test_shim_medium_nonflag_scope_does_not_reject() { - if find_python().is_none() { +#[tokio::test] +async fn test_shim_medium_nonflag_scope_does_not_reject() { + if find_python().await.is_none() { return; } @@ -502,7 +504,7 @@ env.Append(CPPDEFINES=[\"LFS_OK\"]) ", ); - let overlay = resolve_runtime_overlay(temp.path()); + let overlay = resolve_runtime_overlay(temp.path()).await; assert!( overlay .global_compile diff --git a/crates/fbuild-build/src/shrink/probe.rs b/crates/fbuild-build/src/shrink/probe.rs index 52faae7a..47e199ae 100644 --- a/crates/fbuild-build/src/shrink/probe.rs +++ b/crates/fbuild-build/src/shrink/probe.rs @@ -187,6 +187,13 @@ impl Preprocessor for ExternalPreprocessor { // handling rather than calling `std::process::Command` directly // (forbidden by the `ci/find_direct_subprocess.py` lint, see // FastLED/fbuild#141). + // + // FastLED/fbuild#820 (Phase B of #813): `run_command_with_stdin` + // is now `async`. The `Preprocessor` trait keeps a sync signature + // for backward compatibility, so we bridge here via the ambient + // tokio runtime + `block_in_place`. The shrink-libc probe is on + // the cold path (one call per build at most) so a single + // synchronous bridge is fine here. let compiler_str = self.compiler.to_string_lossy().into_owned(); let mut args: Vec<&str> = Vec::with_capacity(self.extra_args.len() + 5); args.push(compiler_str.as_str()); @@ -195,8 +202,26 @@ impl Preprocessor for ExternalPreprocessor { } args.extend_from_slice(&["-E", "-x", "c", "-"]); - let output = run_command_with_stdin(&args, source.as_bytes(), None, None, None) - .map_err(|e| io::Error::other(e.to_string()))?; + let output = match tokio::runtime::Handle::try_current() { + Ok(handle) => tokio::task::block_in_place(|| { + handle.block_on(async { + run_command_with_stdin(&args, source.as_bytes(), None, None, None).await + }) + }), + Err(_) => { + // No ambient runtime — spin up a single-thread runtime + // just for this call. Acceptable because the probe is + // invoked at most once per build. + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(io::Error::other)?; + rt.block_on(async { + run_command_with_stdin(&args, source.as_bytes(), None, None, None).await + }) + } + } + .map_err(|e| io::Error::other(e.to_string()))?; Ok(PreprocessResult { // `run_command_with_stdin` collapses signal-killed cases into a @@ -346,12 +371,19 @@ mod tests { /// dylint forbids raw spawns even for benign `--version` probes. fn find_host_cc() -> Option<&'static str> { use fbuild_core::subprocess::run_command; + // Bridge to async via a current-thread runtime since this test + // helper runs in a `#[test]` (sync) context that may not have an + // ambient tokio runtime. + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .ok()?; for candidate in ["cc", "gcc", "clang"] { let args = [candidate, "--version"]; - if matches!( - run_command(&args, None, None, Some(std::time::Duration::from_secs(5))), - Ok(o) if o.success() - ) { + let outcome = rt.block_on(async { + run_command(&args, None, None, Some(std::time::Duration::from_secs(5))).await + }); + if matches!(outcome, Ok(o) if o.success()) { return Some(candidate); } } diff --git a/crates/fbuild-build/src/silabs/mod.rs b/crates/fbuild-build/src/silabs/mod.rs index b5307908..bc39a560 100644 --- a/crates/fbuild-build/src/silabs/mod.rs +++ b/crates/fbuild-build/src/silabs/mod.rs @@ -1,4 +1,4 @@ -//! Silicon Labs platform build support (EFR32MG24 / SparkFun Thing Plus Matter, etc.) +//! Silicon Labs platform build support (EFR32MG24 / SparkFun Thing Plus Matter, etc.) pub mod mcu_config; pub mod orchestrator; @@ -12,15 +12,16 @@ pub use silabs_linker::SilabsLinker; /// Silicon Labs platform support. pub struct SilabsPlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for SilabsPlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("ARM toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/silabs/orchestrator.rs b/crates/fbuild-build/src/silabs/orchestrator.rs index 7913c8f1..d314c346 100644 --- a/crates/fbuild-build/src/silabs/orchestrator.rs +++ b/crates/fbuild-build/src/silabs/orchestrator.rs @@ -1,4 +1,4 @@ -//! Silicon Labs build orchestrator. +//! Silicon Labs build orchestrator. use std::collections::HashMap; use std::path::{Path, PathBuf}; @@ -14,18 +14,19 @@ use super::{SilabsCompiler, SilabsLinker}; /// Silicon Labs platform build orchestrator. pub struct SilabsOrchestrator; +#[async_trait::async_trait] impl BuildOrchestrator for SilabsOrchestrator { fn platform(&self) -> Platform { Platform::SiliconLabs } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); - let mut ctx = pipeline::BuildContext::new(params)?; + let mut ctx = pipeline::BuildContext::new(params).await?; let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-gcc toolchain at {}", toolchain_dir.display()); use fbuild_packages::Toolchain; @@ -33,10 +34,11 @@ impl BuildOrchestrator for SilabsOrchestrator { &toolchain.get_gcc_path(), "arm-none-eabi-gcc", &mut ctx.build_log, - ); + ) + .await; let framework = fbuild_packages::library::SilabsCores::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("Silicon Labs cores at {}", framework_dir.display()); let core_dir = framework.get_core_dir(&ctx.board.core); @@ -155,6 +157,7 @@ impl BuildOrchestrator for SilabsOrchestrator { "Silicon Labs", start, ) + .await } } diff --git a/crates/fbuild-build/src/silabs/silabs_compiler.rs b/crates/fbuild-build/src/silabs/silabs_compiler.rs index d2e702ca..25290a40 100644 --- a/crates/fbuild-build/src/silabs/silabs_compiler.rs +++ b/crates/fbuild-build/src/silabs/silabs_compiler.rs @@ -75,8 +75,9 @@ impl SilabsCompiler { } } +#[async_trait::async_trait] impl Compiler for SilabsCompiler { - fn compile_one( + async fn compile_one( &self, compiler_path: &Path, source: &Path, @@ -96,6 +97,7 @@ impl Compiler for SilabsCompiler { None, &[], ) + .await } fn gcc_path(&self) -> &Path { diff --git a/crates/fbuild-build/src/silabs/silabs_linker.rs b/crates/fbuild-build/src/silabs/silabs_linker.rs index 9e860976..9aa5937d 100644 --- a/crates/fbuild-build/src/silabs/silabs_linker.rs +++ b/crates/fbuild-build/src/silabs/silabs_linker.rs @@ -1,4 +1,4 @@ -//! Silicon Labs ARM linker implementation. +//! Silicon Labs ARM linker implementation. //! //! Links ARM Cortex-M33 object files into firmware.elf, converts to firmware.bin, //! and reports size using arm-none-eabi-size. @@ -60,12 +60,14 @@ impl SilabsLinker { } } +#[async_trait::async_trait] impl Linker for SilabsLinker { - fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { + async fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "arm-none-eabi-ar") + .await } - fn link( + async fn link( &self, objects: &[PathBuf], archives: &[PathBuf], @@ -125,7 +127,7 @@ impl Linker for SilabsLinker { tracing::info!("link: {}", args.join(" ")); } - // GCC LTO temp dir for MSYS-safe paths — see FastLED/fbuild#261. + // GCC LTO temp dir for MSYS-safe paths — see FastLED/fbuild#261. let lto_env = fbuild_core::subprocess::link_env_for_build(output_dir)?; let env_slice: Vec<(&str, &str)> = lto_env .iter() @@ -133,7 +135,7 @@ impl Linker for SilabsLinker { .collect(); let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, Some(&env_slice), None)?; + let result = run_command(&args_ref, None, Some(&env_slice), None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( @@ -145,7 +147,7 @@ impl Linker for SilabsLinker { Ok(elf_path) } - fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { + async fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { crate::linker::LinkerBase::objcopy_firmware( &self.objcopy_path, elf_path, @@ -154,6 +156,7 @@ impl Linker for SilabsLinker { &self.mcu_config.objcopy.remove_sections, "arm-none-eabi-objcopy", ) + .await } fn size_tool_path(&self) -> &Path { @@ -172,7 +175,7 @@ impl Linker for SilabsLinker { Some(&self.gcc_path) } - fn report_size(&self, elf_path: &Path) -> Result { + async fn report_size(&self, elf_path: &Path) -> Result { crate::linker::LinkerBase::report_size( &self.size_path, elf_path, @@ -180,6 +183,7 @@ impl Linker for SilabsLinker { self.max_ram, "arm-none-eabi-size", ) + .await } } diff --git a/crates/fbuild-build/src/stm32/mod.rs b/crates/fbuild-build/src/stm32/mod.rs index 1f2ae1c5..040ce442 100644 --- a/crates/fbuild-build/src/stm32/mod.rs +++ b/crates/fbuild-build/src/stm32/mod.rs @@ -1,4 +1,4 @@ -//! STM32 platform build support (STM32F1, STM32F4, STM32H7, etc.) +//! STM32 platform build support (STM32F1, STM32F4, STM32H7, etc.) pub mod mcu_config; pub mod orchestrator; @@ -8,15 +8,16 @@ pub use orchestrator::Stm32Orchestrator; /// STM32 platform support. pub struct Stm32PlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for Stm32PlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("ARM toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/stm32/orchestrator/arduino_mbed.rs b/crates/fbuild-build/src/stm32/orchestrator/arduino_mbed.rs index bd047769..348421cd 100644 --- a/crates/fbuild-build/src/stm32/orchestrator/arduino_mbed.rs +++ b/crates/fbuild-build/src/stm32/orchestrator/arduino_mbed.rs @@ -28,7 +28,7 @@ pub(super) fn is_arduino_mbed_stm32_variant(variant: &str) -> bool { ) } -pub(super) fn build_arduino_mbed_stm32( +pub(super) async fn build_arduino_mbed_stm32( params: &BuildParams, ctx: pipeline::BuildContext, toolchain: &fbuild_packages::toolchain::ArmToolchain, @@ -39,7 +39,7 @@ pub(super) fn build_arduino_mbed_stm32( crate::eh_frame_policy_compute::compute_eh_frame_policy(&ctx, params.profile, None); let framework = fbuild_packages::library::ArduinoMbedCore::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("Arduino mbed core at {}", framework_dir.display()); let core_dir = framework.get_core_dir("arduino"); @@ -105,7 +105,8 @@ pub(super) fn build_arduino_mbed_stm32( &ctx.board.variant, &ctx.build_dir, &variant_ldflags, - )?; + ) + .await?; let mcu_config = build_arduino_mbed_mcu_config( &framework, @@ -172,6 +173,7 @@ pub(super) fn build_arduino_mbed_stm32( "STM32", start, ) + .await } fn build_arduino_mbed_mcu_config( @@ -247,7 +249,7 @@ fn build_arduino_mbed_mcu_config( } } -fn preprocess_linker_script( +async fn preprocess_linker_script( gxx_path: PathBuf, variant_dir: &Path, variant_name: &str, @@ -269,7 +271,7 @@ fn preprocess_linker_script( args.push(output.to_string_lossy().to_string()); let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = fbuild_core::subprocess::run_command(&args_ref, None, None, None)?; + let result = fbuild_core::subprocess::run_command(&args_ref, None, None, None).await?; if !result.success() { return Err(fbuild_core::FbuildError::BuildFailed(format!( "failed to preprocess Arduino mbed linker script for {}:\n{}", diff --git a/crates/fbuild-build/src/stm32/orchestrator/mod.rs b/crates/fbuild-build/src/stm32/orchestrator/mod.rs index bebfffce..7b43f8bd 100644 --- a/crates/fbuild-build/src/stm32/orchestrator/mod.rs +++ b/crates/fbuild-build/src/stm32/orchestrator/mod.rs @@ -1,4 +1,4 @@ -//! STM32 build orchestrator — wires together config, packages, compiler, linker. +//! STM32 build orchestrator — wires together config, packages, compiler, linker. //! //! Build phases: //! 1. Parse platformio.ini @@ -13,10 +13,10 @@ //! 10. Convert to hex + report size //! //! Module layout (refactored to keep each .rs file under the 1000-LOC gate): -//! - `arduino_mbed` — Arduino mbed-core build path (GIGA, PORTENTA, ...) -//! - `framework_props` — STM32duino `boards.txt` parser -//! - `includes` — include-path/define helpers and small shared utilities -//! - `variant_files` — variant_*.{h,cpp} / PeripheralPins_*.c selection +//! - `arduino_mbed` — Arduino mbed-core build path (GIGA, PORTENTA, ...) +//! - `framework_props` — STM32duino `boards.txt` parser +//! - `includes` — include-path/define helpers and small shared utilities +//! - `variant_files` — variant_*.{h,cpp} / PeripheralPins_*.c selection //! //! All four submodules are private internals of the STM32 orchestrator. @@ -48,16 +48,17 @@ use self::variant_files::{keep_variant_source, select_variant_files}; /// STM32 platform build orchestrator. pub struct Stm32Orchestrator; +#[async_trait::async_trait] impl BuildOrchestrator for Stm32Orchestrator { fn platform(&self) -> Platform { Platform::Ststm32 } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = pipeline::BuildContext::new(params)?; + let mut ctx = pipeline::BuildContext::new(params).await?; // Compute eh_frame strip policy once per build (FastLED/fbuild#244). let eh_frame_policy = @@ -65,22 +66,23 @@ impl BuildOrchestrator for Stm32Orchestrator { // 3. Ensure ARM GCC toolchain let toolchain = fbuild_packages::toolchain::ArmToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("arm-gcc toolchain at {}", toolchain_dir.display()); pipeline::log_toolchain_version( &toolchain.get_gcc_path(), "arm-none-eabi-gcc", &mut ctx.build_log, - ); + ) + .await; if is_arduino_mbed_stm32_variant(&ctx.board.variant) { - return build_arduino_mbed_stm32(params, ctx, &toolchain, start); + return build_arduino_mbed_stm32(params, ctx, &toolchain, start).await; } // 4. Ensure STM32duino cores let framework = fbuild_packages::library::Stm32Cores::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("STM32 cores at {}", framework_dir.display()); // 5. Scan sources (core + variant) @@ -104,7 +106,7 @@ impl BuildOrchestrator for Stm32Orchestrator { ); let scanner = SourceScanner::new(&ctx.src_dir, &ctx.src_build_dir); - // Scan core + variant, but pass None for variant — we'll filter variant + // Scan core + variant, but pass None for variant — we'll filter variant // sources manually because the variant dir contains files for multiple // board variants (MALYAN, AFROFLIGHT, etc.) and startup files that // conflict with the generic one in cores/arduino/stm32/. @@ -116,7 +118,7 @@ impl BuildOrchestrator for Stm32Orchestrator { .filter(|p| keep_variant_source(p, &selected_variant)) .collect(); - // SrcWrapper is a core library in STM32duino — its sources must be + // SrcWrapper is a core library in STM32duino — its sources must be // compiled alongside the Arduino core (HAL wrappers, syscalls, etc.) // scan_core_sources is recursive, so one call covers all subdirs. let libs_dir = framework.get_libraries_dir(); @@ -137,7 +139,7 @@ impl BuildOrchestrator for Stm32Orchestrator { // through M7 (H7xx) but the toolchain triple is constant // (`arm-none-eabi`). The cache key already includes // `framework_install_path` + `framework_version`, so per-MCU drift - // is handled there — this string only needs to disambiguate stm32 + // is handled there — this string only needs to disambiguate stm32 // from teensy etc. so cross-platform key collisions are impossible. let framework_info = fbuild_packages::Package::get_info(&framework); let framework_library_sources = match library_select_kv_store() { @@ -207,7 +209,7 @@ impl BuildOrchestrator for Stm32Orchestrator { // JSON extra_flags may only have STM32F1, so ensure the full family define. defines.insert(family.to_string(), "1".to_string()); // STM32duino variant sources are guarded by ARDUINO_GENERIC_ defines. - // Derive from the MCU name: stm32f103c8t6 → ARDUINO_GENERIC_F103C8TX + // Derive from the MCU name: stm32f103c8t6 → ARDUINO_GENERIC_F103C8TX let generic_board = stm32_generic_board_define(&ctx.board.mcu); defines.insert(format!("ARDUINO_{generic_board}"), "1".to_string()); // STM32duino requires these defines for HAL/LL and variant header resolution. @@ -217,7 +219,7 @@ impl BuildOrchestrator for Stm32Orchestrator { "VARIANT_H".to_string(), format!("\\\"{}\\\"", selected_variant.header), ); - // UART HAL module is disabled by default in stm32yyxx_hal_conf.h — enable it + // UART HAL module is disabled by default in stm32yyxx_hal_conf.h — enable it // so WSerial.h can create the Serial instance. defines.insert("HAL_UART_MODULE_ENABLED".to_string(), "1".to_string()); defines.insert("HAL_PCD_MODULE_ENABLED".to_string(), "1".to_string()); @@ -247,9 +249,9 @@ impl BuildOrchestrator for Stm32Orchestrator { let system_dir = framework.get_system_dir(); add_stm32_system_includes(&system_dir, family, &mut include_dirs); - // CMSIS Core includes (core_cm3.h, core_cm4.h, etc.) — not bundled in STM32duino + // CMSIS Core includes (core_cm3.h, core_cm4.h, etc.) — not bundled in STM32duino let cmsis = fbuild_packages::library::CmsisFramework::new(¶ms.project_dir); - let _cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis)?; + let _cmsis_dir = fbuild_packages::Package::ensure_installed(&cmsis).await?; tracing::info!("CMSIS framework installed"); include_dirs.push(cmsis.get_core_include_dir()); @@ -322,6 +324,7 @@ impl BuildOrchestrator for Stm32Orchestrator { "STM32", start, ) + .await } } diff --git a/crates/fbuild-build/src/symbol_analyzer/mod.rs b/crates/fbuild-build/src/symbol_analyzer/mod.rs index 5aa080b2..292990b5 100644 --- a/crates/fbuild-build/src/symbol_analyzer/mod.rs +++ b/crates/fbuild-build/src/symbol_analyzer/mod.rs @@ -167,15 +167,16 @@ pub fn default_map_path(elf_path: &Path) -> Option { /// /// When c++filt can't decode a name it echoes it back unchanged, which /// is the desired fallback. Output stays parallel to the input. -pub fn demangle_batch(mangled: &[String], cppfilt_path: &Path) -> Result> { +pub async fn demangle_batch(mangled: &[String], cppfilt_path: &Path) -> Result> { if mangled.is_empty() { return Ok(Vec::new()); } let stdin_data = mangled.join("\n"); let cppfilt_s = cppfilt_path.to_string_lossy().to_string(); let args = [cppfilt_s.as_str()]; - let result = - run_command_with_stdin(&args, stdin_data.as_bytes(), None, None, None).map_err(|e| { + let result = run_command_with_stdin(&args, stdin_data.as_bytes(), None, None, None) + .await + .map_err(|e| { FbuildError::BuildFailed(format!( "failed to run c++filt at {}: {e}", cppfilt_path.display() @@ -212,7 +213,7 @@ pub struct AnalyzeConfig<'a> { /// Run nm + c++filt + map-file parse and return the fully-attributed /// per-symbol map. -pub fn analyze_elf(cfg: AnalyzeConfig<'_>) -> Result { +pub async fn analyze_elf(cfg: AnalyzeConfig<'_>) -> Result { use fbuild_core::subprocess::run_command; let nm_path_s = cfg.nm_path.to_string_lossy().to_string(); @@ -225,7 +226,7 @@ pub fn analyze_elf(cfg: AnalyzeConfig<'_>) -> Result { "-S", elf_s.as_str(), ]; - let result = run_command(&args, None, None, None)?; + let result = run_command(&args, None, None, None).await?; if !result.success() { return Err(FbuildError::BuildFailed(format!( "nm failed: {}", @@ -237,10 +238,13 @@ pub fn analyze_elf(cfg: AnalyzeConfig<'_>) -> Result { let mangled: Vec = nm_rows.iter().map(|r| r.3.clone()).collect(); let demangled = if let Some(cppfilt) = cfg.cppfilt_path { - demangle_batch(&mangled, cppfilt).unwrap_or_else(|e| { - tracing::warn!("c++filt unavailable ({e}); falling back to mangled names"); - mangled.clone() - }) + match demangle_batch(&mangled, cppfilt).await { + Ok(v) => v, + Err(e) => { + tracing::warn!("c++filt unavailable ({e}); falling back to mangled names"); + mangled.clone() + } + } } else { mangled.clone() }; @@ -274,12 +278,15 @@ pub fn analyze_elf(cfg: AnalyzeConfig<'_>) -> Result { let synth_demangled = if synth_mangled.is_empty() { Vec::new() } else if let Some(cppfilt) = cfg.cppfilt_path { - demangle_batch(&synth_mangled, cppfilt).unwrap_or_else(|e| { - tracing::warn!( - "c++filt unavailable for synthetic owners ({e}); falling back to mangled names" - ); - synth_mangled.clone() - }) + match demangle_batch(&synth_mangled, cppfilt).await { + Ok(v) => v, + Err(e) => { + tracing::warn!( + "c++filt unavailable for synthetic owners ({e}); falling back to mangled names" + ); + synth_mangled.clone() + } + } } else { synth_mangled.clone() }; @@ -327,7 +334,7 @@ pub fn analyze_elf(cfg: AnalyzeConfig<'_>) -> Result { // — we'd rather ship a report without forward edges than fail // the whole symbol-analysis post-link step. if let Some(objdump_path) = cfg.objdump_path { - match run_objdump_and_attribute(objdump_path, cfg.elf_path, &mut map) { + match run_objdump_and_attribute(objdump_path, cfg.elf_path, &mut map).await { Ok(edge_count) => { tracing::info!( "objdump: extracted {edge_count} forward edges from {}", @@ -351,7 +358,7 @@ pub fn analyze_elf(cfg: AnalyzeConfig<'_>) -> Result { /// `references_to` on every symbol in `map` that the parser found /// outgoing call edges for. Returns the total edge count surfaced /// (across all symbols) so the caller can log a one-liner. -fn run_objdump_and_attribute( +async fn run_objdump_and_attribute( objdump_path: &Path, elf_path: &Path, map: &mut FineGrainedSymbolMap, @@ -367,7 +374,7 @@ fn run_objdump_and_attribute( "--no-show-raw-insn", elf_s.as_str(), ]; - let result = run_command(&args, None, None, None)?; + let result = run_command(&args, None, None, None).await?; if !result.success() { return Err(FbuildError::BuildFailed(format!( "objdump exit={}: {}", diff --git a/crates/fbuild-build/src/teensy/mod.rs b/crates/fbuild-build/src/teensy/mod.rs index 1ca3e40f..0fe139d5 100644 --- a/crates/fbuild-build/src/teensy/mod.rs +++ b/crates/fbuild-build/src/teensy/mod.rs @@ -1,4 +1,4 @@ -//! Teensy platform build support (Teensy 4.0, 4.1) +//! Teensy platform build support (Teensy 4.0, 4.1) pub mod mcu_config; pub mod orchestrator; @@ -12,15 +12,16 @@ pub use teensy_linker::TeensyLinker; /// Teensy platform support. pub struct TeensyPlatformSupport; +#[async_trait::async_trait] impl crate::PlatformSupport for TeensyPlatformSupport { fn create_orchestrator(&self) -> Box { orchestrator::create() } - fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { + async fn install_deps(&self, project_dir: &std::path::Path) -> fbuild_core::Result<()> { use fbuild_packages::Package; let tc = fbuild_packages::toolchain::ArmToolchain::new(project_dir); - Package::ensure_installed(&tc)?; + Package::ensure_installed(&tc).await?; tracing::info!("ARM toolchain installed"); Ok(()) } diff --git a/crates/fbuild-build/src/teensy/orchestrator.rs b/crates/fbuild-build/src/teensy/orchestrator.rs index 49806055..30f6a82d 100644 --- a/crates/fbuild-build/src/teensy/orchestrator.rs +++ b/crates/fbuild-build/src/teensy/orchestrator.rs @@ -1,4 +1,4 @@ -//! Teensy build orchestrator — wires together config, packages, compiler, linker. +//! Teensy build orchestrator — wires together config, packages, compiler, linker. //! //! Build phases: //! 1. Parse platformio.ini @@ -62,17 +62,18 @@ fn profile_label(profile: fbuild_core::BuildProfile) -> &'static str { } } +#[async_trait::async_trait] impl BuildOrchestrator for TeensyOrchestrator { fn platform(&self) -> Platform { Platform::Teensy } - fn build(&self, params: &BuildParams) -> Result { + async fn build(&self, params: &BuildParams) -> Result { let start = Instant::now(); let compiler_cache: Option = None; // 1-2. Parse config, load board, setup build dirs, resolve src dir, collect flags - let mut ctx = pipeline::BuildContext::new(params)?; + let mut ctx = pipeline::BuildContext::new(params).await?; // Compute eh_frame strip policy once per build (FastLED/fbuild#244). // No sdkconfig on Teensy. @@ -87,7 +88,7 @@ impl BuildOrchestrator for TeensyOrchestrator { // 3. Ensure Teensy-compatible ARM GCC toolchain let toolchain = fbuild_packages::toolchain::TeensyArmToolchain::new(¶ms.project_dir); - let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain)?; + let toolchain_dir = fbuild_packages::Package::ensure_installed(&toolchain).await?; tracing::info!("Teensy ARM GCC toolchain at {}", toolchain_dir.display()); use fbuild_packages::Toolchain; @@ -95,11 +96,12 @@ impl BuildOrchestrator for TeensyOrchestrator { &toolchain.get_gcc_path(), "arm-none-eabi-gcc", &mut ctx.build_log, - ); + ) + .await; // 4. Ensure Teensy cores let framework = fbuild_packages::library::TeensyCores::new(¶ms.project_dir); - let framework_dir = fbuild_packages::Package::ensure_installed(&framework)?; + let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; tracing::info!("Teensy cores at {}", framework_dir.display()); let core_dir = framework.get_core_dir(&ctx.board.core); @@ -173,7 +175,7 @@ impl BuildOrchestrator for TeensyOrchestrator { let framework_libs = framework.get_framework_libraries(); // WHY: Teensy 3.x/4.x and TeensyLC all share teensyduino's - // arm-none-eabi toolchain — a single stable triple covers every + // arm-none-eabi toolchain — a single stable triple covers every // board this orchestrator handles. The triple feeds the cache key // so bumping it invalidates the entire teensy slice without // touching SCANNER_VERSION / LDF_MODE_VERSION. @@ -298,7 +300,7 @@ impl BuildOrchestrator for TeensyOrchestrator { TargetArchitecture::Arm, "Teensy", start, - )?; + ).await?; if build_result.success && !params.compiledb_only diff --git a/crates/fbuild-build/src/teensy/teensy_compiler.rs b/crates/fbuild-build/src/teensy/teensy_compiler.rs index 4b485975..dbaf0462 100644 --- a/crates/fbuild-build/src/teensy/teensy_compiler.rs +++ b/crates/fbuild-build/src/teensy/teensy_compiler.rs @@ -98,8 +98,9 @@ impl TeensyCompiler { } } +#[async_trait::async_trait] impl Compiler for TeensyCompiler { - fn compile_one( + async fn compile_one( &self, compiler_path: &Path, source: &Path, @@ -119,6 +120,7 @@ impl Compiler for TeensyCompiler { None, &[], ) + .await } fn gcc_path(&self) -> &Path { diff --git a/crates/fbuild-build/src/teensy/teensy_linker.rs b/crates/fbuild-build/src/teensy/teensy_linker.rs index 256591f6..5bc67f72 100644 --- a/crates/fbuild-build/src/teensy/teensy_linker.rs +++ b/crates/fbuild-build/src/teensy/teensy_linker.rs @@ -120,12 +120,14 @@ impl TeensyLinker { } } +#[async_trait::async_trait] impl Linker for TeensyLinker { - fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { + async fn archive(&self, objects: &[PathBuf], output: &Path) -> Result<()> { crate::linker::LinkerBase::archive(&self.ar_path, objects, output, "arm-none-eabi-ar") + .await } - fn link( + async fn link( &self, objects: &[PathBuf], archives: &[PathBuf], @@ -161,12 +163,13 @@ impl Linker for TeensyLinker { &rsp_content, &temp_dir, "teensy_link", - )?; + ) + .await?; let rsp_arg = format!("@{}", rsp_path.display()); - run_command(&[args[0].as_str(), &rsp_arg], None, Some(&env_slice), None)? + run_command(&[args[0].as_str(), &rsp_arg], None, Some(&env_slice), None).await? } else { let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - run_command(&args_ref, None, Some(&env_slice), None)? + run_command(&args_ref, None, Some(&env_slice), None).await? }; if !result.success() { @@ -179,7 +182,7 @@ impl Linker for TeensyLinker { Ok(elf_path) } - fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { + async fn convert_firmware(&self, elf_path: &Path, output_dir: &Path) -> Result { crate::linker::LinkerBase::objcopy_firmware( &self.objcopy_path, elf_path, @@ -188,6 +191,7 @@ impl Linker for TeensyLinker { &self.mcu_config.objcopy.remove_sections, "arm-none-eabi-objcopy", ) + .await } fn size_tool_path(&self) -> &Path { @@ -206,7 +210,7 @@ impl Linker for TeensyLinker { Some(&self.gcc_path) } - fn report_size(&self, elf_path: &Path) -> Result { + async fn report_size(&self, elf_path: &Path) -> Result { crate::linker::LinkerBase::report_size( &self.size_path, elf_path, @@ -214,6 +218,7 @@ impl Linker for TeensyLinker { self.max_ram, "arm-none-eabi-size", ) + .await } } diff --git a/crates/fbuild-build/src/zccache_embedded.rs b/crates/fbuild-build/src/zccache_embedded.rs index 7eaad973..c66e796b 100644 --- a/crates/fbuild-build/src/zccache_embedded.rs +++ b/crates/fbuild-build/src/zccache_embedded.rs @@ -172,30 +172,21 @@ impl FbuildZccacheService { &self.inner } - /// Synchronous per-compile dispatch (Phase 2 / FastLED/fbuild#791). + /// Async per-compile dispatch (FastLED/fbuild#820, Phase B of #813). /// - /// Builds a [`ZccacheCompileRequest`] from the wrapper-mode - /// inputs, blocks on the underlying async - /// [`ZccacheService::compile`] via the caller-supplied - /// `tokio::runtime::Handle`, and returns an - /// [`EmbeddedCompileOutcome`] shaped to drop into - /// [`crate::compiler::CompileResult`]. Designed to slot into - /// `compile_source` exactly where the wrapper-mode - /// `run_command(["zccache", "wrap", compiler, …])` runs today. - /// - /// `runtime` must be a multi-thread tokio runtime handle — the - /// daemon's `#[tokio::main]` default. Calling from inside a - /// current-thread runtime will panic per tokio's `Handle::block_on` - /// contract. + /// Builds a [`ZccacheCompileRequest`] and awaits the underlying + /// [`ZccacheService::compile`] directly. Replaces `compile_blocking` + /// on the hot path now that `compile_source` and the `Compiler` + /// trait are fully `async fn`. No `block_on` — the call happens on + /// whatever tokio task the orchestrator is already running on. /// /// `env` is forwarded as a `Vec<(String, String)>` — zccache's /// embedded API takes the same shape. fbuild does NOT inherit /// caller env here; the call site must pass exactly the env vars /// the compile needs (typically the empty vec — gcc reads no env /// fbuild cares about for caching). - pub fn compile_blocking( + pub async fn compile( &self, - runtime: &tokio::runtime::Handle, compiler: &Path, args: Vec, cwd: PathBuf, @@ -209,9 +200,10 @@ impl FbuildZccacheService { env, stdin: Vec::new(), }; - let inner = self.inner.clone(); - let resp = runtime - .block_on(async move { inner.compile(req).await }) + let resp = self + .inner + .compile(req) + .await .map_err(|e| EmbeddedServiceError::Compile(e.to_string()))?; Ok(EmbeddedCompileOutcome { exit_code: resp.exit_code, @@ -221,6 +213,36 @@ impl FbuildZccacheService { }) } + /// Deprecated sync wrapper around [`Self::compile`]. Retained + /// transiently for any out-of-tree caller that hasn't migrated to + /// the async path. `compile_source` no longer routes through this. + #[deprecated( + since = "0.2.0", + note = "FastLED/fbuild#820: use the async `compile` method directly; \ + the build pipeline is fully async as of #813 Phase B" + )] + pub fn compile_blocking( + &self, + runtime: &tokio::runtime::Handle, + compiler: &Path, + args: Vec, + cwd: PathBuf, + env: Vec<(String, String)>, + ) -> Result { + let svc = self.clone_handle(); + runtime.block_on(async move { svc.compile(compiler, args, cwd, env).await }) + } + + /// Cheap clone of the service handle (Arc bump) for the deprecated + /// `compile_blocking` shim. Kept private to discourage direct use. + fn clone_handle(&self) -> Self { + Self { + inner: Arc::clone(&self.inner), + identity: self.identity.clone(), + cache_root: self.cache_root.clone(), + } + } + /// Drain pending writes. Useful at end-of-session boundaries. pub async fn flush(&self) -> Result<(), EmbeddedServiceError> { self.inner diff --git a/crates/fbuild-build/tests/avr_build.rs b/crates/fbuild-build/tests/avr_build.rs index 319d0a27..d242a653 100644 --- a/crates/fbuild-build/tests/avr_build.rs +++ b/crates/fbuild-build/tests/avr_build.rs @@ -62,9 +62,9 @@ fn cache_paths_stem_hash() { /// This test requires: /// - Internet access (first run only, then cached) /// - ~/dev/fbuild/tests/uno_minimal/ to exist (Python fbuild repo) -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_uno_minimal() { +async fn build_uno_minimal() { let project_dir = home_dir().join("dev/fbuild/tests/uno_minimal"); if !project_dir.exists() { @@ -105,6 +105,7 @@ fn build_uno_minimal() { let orchestrator = fbuild_build::avr::orchestrator::AvrOrchestrator; let result = orchestrator .build(¶ms) + .await .expect("AVR build should succeed"); assert!(result.success, "build should report success"); @@ -159,9 +160,9 @@ fn build_uno_minimal() { } /// Compare our build output against the Python fbuild's cached output. -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn compare_with_python_output() { +async fn compare_with_python_output() { let project_dir = home_dir().join("dev/fbuild/tests/uno_minimal"); let python_hex = project_dir.join(".fbuild/build/uno/release/firmware.hex"); @@ -199,7 +200,10 @@ fn compare_with_python_output() { }; let orchestrator = fbuild_build::avr::orchestrator::AvrOrchestrator; - let result = orchestrator.build(¶ms).expect("build should succeed"); + let result = orchestrator + .build(¶ms) + .await + .expect("build should succeed"); let rust_hex = result.firmware_path.expect("should produce hex"); let python_content = fs::read_to_string(&python_hex).unwrap(); @@ -229,9 +233,9 @@ fn compare_with_python_output() { } /// Build a self-contained test project (no dependency on Python fbuild repo). -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_self_contained_blink() { +async fn build_self_contained_blink() { let tmp = tempfile::TempDir::new().unwrap(); let project_dir = tmp.path(); @@ -287,6 +291,7 @@ void loop() { let orchestrator = fbuild_build::avr::orchestrator::AvrOrchestrator; let result = orchestrator .build(¶ms) + .await .expect("self-contained build should succeed"); assert!(result.success); @@ -427,9 +432,9 @@ impl Drop for EnvVarGuard { /// /// Gated `#[ignore]` because it downloads avr-gcc + Arduino-AVR core (cached globally /// after first run, but still adds 30s+ to first invocation). -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn cache_survives_tar_extract_uno() { +async fn cache_survives_tar_extract_uno() { let tmp_a = tempfile::TempDir::new().unwrap(); let proj_a = tmp_a.path().join("proj"); fs::create_dir_all(&proj_a).unwrap(); @@ -443,6 +448,7 @@ fn cache_survives_tar_extract_uno() { proj_a.join(".fbuild/build/uno/release"), true, )) + .await .expect("cold AVR build should succeed"); assert!(cold_result.success, "cold build should report success"); assert!( @@ -475,6 +481,7 @@ fn cache_survives_tar_extract_uno() { proj_a.join(".fbuild/build/uno/release"), false, )) + .await .expect("same-project warm build should succeed"); assert!( same_project_warm @@ -520,6 +527,7 @@ fn cache_survives_tar_extract_uno() { proj_b.join(".fbuild/build/uno/release"), false, )) + .await .expect("warm AVR build (post tar-extract) should succeed"); assert!(warm_result.success, "warm build should report success"); assert!( diff --git a/crates/fbuild-build/tests/compile_many_stage2_perf.rs b/crates/fbuild-build/tests/compile_many_stage2_perf.rs index b3efcd8d..80c8924b 100644 --- a/crates/fbuild-build/tests/compile_many_stage2_perf.rs +++ b/crates/fbuild-build/tests/compile_many_stage2_perf.rs @@ -47,9 +47,9 @@ fn scaffold_uno_blink(project_dir: &Path) { fs::write(src_dir.join("blink.ino"), UNO_BLINK_INO).unwrap(); } -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn stage2_per_sketch_wall_is_a_fraction_of_stage1() { +async fn stage2_per_sketch_wall_is_a_fraction_of_stage1() { let tmp = tempfile::TempDir::new().unwrap(); let sketches: Vec = (0..4) .map(|i| { @@ -71,7 +71,9 @@ fn stage2_per_sketch_wall_is_a_fraction_of_stage1() { diag_stage2: true, }; - let result = compile_many(req).expect("compile_many should not error"); + let result = compile_many(req) + .await + .expect("compile_many should not error"); assert!( result.all_success, "every sketch should build: results={:?}", diff --git a/crates/fbuild-build/tests/compile_many_two_stage.rs b/crates/fbuild-build/tests/compile_many_two_stage.rs index 69c676f0..7d7b2ef8 100644 --- a/crates/fbuild-build/tests/compile_many_two_stage.rs +++ b/crates/fbuild-build/tests/compile_many_two_stage.rs @@ -10,14 +10,14 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Barrier, Mutex}; -use std::time::Duration; +use std::sync::{Arc, Mutex}; use fbuild_build::compile_many::{ compile_many_with, stage2_jobs_per_worker, CompileManyRequest, SketchBuildInputs, SketchBuilder, SketchResult, Stage, }; use fbuild_core::BuildProfile; +use tokio::sync::Barrier; /// Test sketch root with a minimal `platformio.ini`. Used as a /// `project_dir` parameter — the mock builder does not read it, but @@ -88,8 +88,9 @@ impl MockBuilder { } } +#[async_trait::async_trait] impl SketchBuilder for MockBuilder { - fn build(&self, inputs: SketchBuildInputs) -> SketchResult { + async fn build(&self, inputs: SketchBuildInputs) -> SketchResult { // Synthesize the canonical per-sketch firmware path the same // way the real orchestrator does. We deliberately use the // same naming convention so the uniqueness assertion is @@ -132,7 +133,7 @@ impl SketchBuilder for MockBuilder { // wired up: every worker blocks here until N peers arrive. if inputs.stage == Stage::Stage2Sketch { if let Some(ref b) = self.stage2_barrier { - b.wait(); + b.wait().await; self.stage2_wait_count.fetch_add(1, Ordering::Relaxed); } } @@ -173,16 +174,17 @@ fn make_request( /// AC: stage 1 runs exactly once across N sketches, stage 2 produces /// N-1 firmware artifacts, and per-sketch output paths are unique. -#[test] -fn stage1_runs_exactly_once_and_stage2_handles_the_rest() { +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stage1_runs_exactly_once_and_stage2_handles_the_rest() { let tmp = tempfile::tempdir().unwrap(); let sketches: Vec = (0..5) .map(|i| make_sketch(tmp.path(), &format!("sketch{i}"), "uno")) .collect(); let mock = MockBuilder::new(); - let result = - compile_many_with(make_request(sketches.clone(), 1, 4), &mock).expect("compile_many"); + let result = compile_many_with(make_request(sketches.clone(), 1, 4), &mock) + .await + .expect("compile_many"); assert!(result.all_success, "all mock builds should succeed"); assert_eq!(result.results.len(), 5); @@ -227,15 +229,17 @@ fn stage1_runs_exactly_once_and_stage2_handles_the_rest() { /// AC: stage-1 results are placed at index 0; stage-2 results follow /// input order regardless of completion order. -#[test] -fn results_are_returned_in_input_order() { +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn results_are_returned_in_input_order() { let tmp = tempfile::tempdir().unwrap(); let sketches: Vec = (0..6) .map(|i| make_sketch(tmp.path(), &format!("ordered{i}"), "uno")) .collect(); let mock = MockBuilder::new(); - let result = compile_many_with(make_request(sketches.clone(), 1, 3), &mock).expect("ok"); + let result = compile_many_with(make_request(sketches.clone(), 1, 3), &mock) + .await + .expect("ok"); for (i, r) in result.results.iter().enumerate() { assert_eq!(r.sketch, sketches[i], "result {i} should match input order"); @@ -250,8 +254,8 @@ fn results_are_returned_in_input_order() { /// concurrently — proven via a Barrier that would deadlock under serial /// execution. The barrier counts each stage-2 worker as it crosses; a /// successful test means N workers ran in parallel. -#[test] -fn stage2_workers_run_concurrently() { +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stage2_workers_run_concurrently() { let tmp = tempfile::tempdir().unwrap(); let n_stage2 = 4; let total = n_stage2 + 1; @@ -262,35 +266,27 @@ fn stage2_workers_run_concurrently() { // Barrier sized to exactly the stage-2 worker count. If // `compile_many` ran them serially, `wait()` would block forever // (only one worker at a time would arrive), and the test would - // hang. We add a wall-clock timeout below so a regression is - // a fast failure rather than a CI hang. + // hang. We wrap in `tokio::time::timeout` so a regression is a + // fast failure rather than a CI hang. let barrier = Arc::new(Barrier::new(n_stage2)); let mock = Arc::new(MockBuilder::with_barrier(Arc::clone(&barrier))); let req = make_request(sketches, 1, n_stage2); - let mock_for_thread = Arc::clone(&mock); - let handle = std::thread::spawn(move || compile_many_with(req, mock_for_thread.as_ref())); - - // Generous deadline; on a healthy machine the test completes in - // milliseconds. The deadline only fires if the barrier deadlocks - // because workers ran serially. - let start = std::time::Instant::now(); - let deadline = Duration::from_secs(10); - loop { - if handle.is_finished() { - break; - } - if start.elapsed() > deadline { - panic!( - "stage-2 deadlock: barrier expected {} concurrent workers but only {} arrived", - n_stage2, - mock.stage2_wait_count.load(Ordering::Relaxed) - ); - } - std::thread::sleep(Duration::from_millis(20)); - } + let mock_for_task = Arc::clone(&mock); + let result = tokio::time::timeout( + std::time::Duration::from_secs(10), + compile_many_with(req, mock_for_task.as_ref()), + ) + .await + .unwrap_or_else(|_| { + panic!( + "stage-2 deadlock: barrier expected {} concurrent workers but only {} arrived", + n_stage2, + mock.stage2_wait_count.load(Ordering::Relaxed) + ) + }) + .expect("compile_many"); - let result = handle.join().expect("worker thread").expect("compile_many"); assert!(result.all_success); assert_eq!(result.stage2_count, n_stage2); assert_eq!( @@ -303,11 +299,12 @@ fn stage2_workers_run_concurrently() { /// AC: stage-1 failure short-circuits stage 2 — no point fanning out /// when the framework build is broken, every stage-2 worker would just /// repeat the same error. -#[test] -fn stage1_failure_skips_stage2() { +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stage1_failure_skips_stage2() { struct FailingStage1; + #[async_trait::async_trait] impl SketchBuilder for FailingStage1 { - fn build(&self, inputs: SketchBuildInputs) -> SketchResult { + async fn build(&self, inputs: SketchBuildInputs) -> SketchResult { SketchResult { sketch: inputs.sketch.clone(), env_name: inputs.env_name, @@ -329,7 +326,9 @@ fn stage1_failure_skips_stage2() { let sketches: Vec = (0..3) .map(|i| make_sketch(tmp.path(), &format!("fail{i}"), "uno")) .collect(); - let result = compile_many_with(make_request(sketches, 1, 2), &FailingStage1).expect("ok"); + let result = compile_many_with(make_request(sketches, 1, 2), &FailingStage1) + .await + .expect("ok"); assert!(!result.all_success); assert_eq!(result.stage1_count, 1); assert_eq!( @@ -344,12 +343,14 @@ fn stage1_failure_skips_stage2() { } /// AC: a single sketch falls through stage 1 only — stage 2 is empty. -#[test] -fn single_sketch_runs_only_stage1() { +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn single_sketch_runs_only_stage1() { let tmp = tempfile::tempdir().unwrap(); let sketch = make_sketch(tmp.path(), "only", "uno"); let mock = MockBuilder::new(); - let result = compile_many_with(make_request(vec![sketch], 2, 4), &mock).expect("ok"); + let result = compile_many_with(make_request(vec![sketch], 2, 4), &mock) + .await + .expect("ok"); assert!(result.all_success); assert_eq!(result.stage1_count, 1); assert_eq!(result.stage2_count, 0); @@ -428,8 +429,8 @@ fn stage2_jobs_per_worker_splits_cores_across_workers() { ); } -#[test] -fn stage2_results_report_worker_and_seed_diagnostics() { +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn stage2_results_report_worker_and_seed_diagnostics() { let tmp = tempfile::tempdir().unwrap(); let sketches: Vec = (0..4) .map(|i| make_sketch(tmp.path(), &format!("diag{i}"), "uno")) @@ -438,7 +439,7 @@ fn stage2_results_report_worker_and_seed_diagnostics() { let mock = MockBuilder::new(); let mut req = make_request(sketches, 1, 2); req.diag_stage2 = true; - let result = compile_many_with(req, &mock).expect("compile_many"); + let result = compile_many_with(req, &mock).await.expect("compile_many"); let stage2: Vec<_> = result .results diff --git a/crates/fbuild-build/tests/eh_frame_strip_esp32.rs b/crates/fbuild-build/tests/eh_frame_strip_esp32.rs index c58b7acd..562258e7 100644 --- a/crates/fbuild-build/tests/eh_frame_strip_esp32.rs +++ b/crates/fbuild-build/tests/eh_frame_strip_esp32.rs @@ -71,9 +71,9 @@ void loop() { .unwrap(); } -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "downloads ESP32 toolchain (~hundreds of MB)"] -fn eh_frame_strip_drops_firmware_at_least_150kb() { +async fn eh_frame_strip_drops_firmware_at_least_150kb() { // Use two separate tempdirs so .fbuild/build/... paths don't collide. let preserve_tmp = tempfile::TempDir::new().unwrap(); let strip_tmp = tempfile::TempDir::new().unwrap(); @@ -95,6 +95,7 @@ fn eh_frame_strip_drops_firmware_at_least_150kb() { let preserve_params = make_params(&preserve_dir); let preserve_result = orchestrator .build(&preserve_params) + .await .expect("preserve build should succeed"); assert!( preserve_result.success, @@ -113,6 +114,7 @@ fn eh_frame_strip_drops_firmware_at_least_150kb() { let strip_params = make_params(&strip_dir); let strip_result = orchestrator .build(&strip_params) + .await .expect("strip build should succeed"); assert!(strip_result.success, "strip build should report success"); let strip_elf = strip_result diff --git a/crates/fbuild-build/tests/esp32_build.rs b/crates/fbuild-build/tests/esp32_build.rs index 9655c272..2336153f 100644 --- a/crates/fbuild-build/tests/esp32_build.rs +++ b/crates/fbuild-build/tests/esp32_build.rs @@ -25,9 +25,9 @@ fn home_dir() -> PathBuf { /// Build a self-contained ESP32 blink sketch. /// /// This test requires Internet access (first run only, then cached). -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_esp32dev_blink() { +async fn build_esp32dev_blink() { let tmp = tempfile::TempDir::new().unwrap(); let project_dir = tmp.path(); @@ -85,6 +85,7 @@ void loop() { let orchestrator = fbuild_build::esp32::orchestrator::Esp32Orchestrator; let result = orchestrator .build(¶ms) + .await .expect("ESP32 build should succeed"); assert!(result.success); @@ -117,9 +118,9 @@ void loop() { } /// Build a self-contained ESP32-C6 blink sketch (RISC-V). -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_esp32c6_blink() { +async fn build_esp32c6_blink() { let tmp = tempfile::TempDir::new().unwrap(); let project_dir = tmp.path(); @@ -175,6 +176,7 @@ void loop() { let orchestrator = fbuild_build::esp32::orchestrator::Esp32Orchestrator; let result = orchestrator .build(¶ms) + .await .expect("ESP32-C6 build should succeed"); assert!(result.success); @@ -199,9 +201,9 @@ void loop() { /// ESP32-C3 uses the rv32imc RISC-V ISA. This test validates the full build /// pipeline for the C3 variant, including toolchain selection and framework /// extraction. It requires Internet access (first run only, then cached). -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_esp32c3_blink() { +async fn build_esp32c3_blink() { let tmp = tempfile::TempDir::new().unwrap(); let project_dir = tmp.path(); @@ -258,6 +260,7 @@ void loop() { let orchestrator = fbuild_build::esp32::orchestrator::Esp32Orchestrator; let result = orchestrator .build(¶ms) + .await .expect("ESP32-C3 build should succeed"); assert!(result.success); @@ -280,9 +283,9 @@ void loop() { /// Build a self-contained ESP32-S3 blink sketch (Xtensa, native USB-CDC). /// /// This test requires Internet access (first run only, then cached). -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_esp32s3_blink() { +async fn build_esp32s3_blink() { let tmp = tempfile::TempDir::new().unwrap(); let project_dir = tmp.path(); @@ -342,6 +345,7 @@ void loop() { let orchestrator = fbuild_build::esp32::orchestrator::Esp32Orchestrator; let result = orchestrator .build(¶ms) + .await .expect("ESP32-S3 build should succeed"); assert!(result.success); @@ -376,9 +380,9 @@ void loop() { /// Build ESP32-S3 blink from the tests/platform/esp32s3 fixture with persistent output. /// /// Build output is stored at tests/platform/esp32s3/.fbuild/build/ for manual deployment. -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_esp32s3_fixture() { +async fn build_esp32s3_fixture() { let project_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() .unwrap() @@ -416,6 +420,7 @@ fn build_esp32s3_fixture() { let orchestrator = fbuild_build::esp32::orchestrator::Esp32Orchestrator; let result = orchestrator .build(¶ms) + .await .expect("ESP32-S3 fixture build should succeed"); assert!(result.success); @@ -442,9 +447,9 @@ fn build_esp32s3_fixture() { /// Requires ~/dev/fbuild/tests/NightDriverStrip/ to exist. /// NOTE: This will fail until library dependency resolution (Phase 4) is implemented, /// because NightDriverStrip depends on FastLED, ArduinoJson, etc. -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_nightdriverstrip_demo() { +async fn build_nightdriverstrip_demo() { let project_dir = home_dir().join("dev/fbuild/tests/NightDriverStrip"); if !project_dir.exists() { @@ -482,6 +487,7 @@ fn build_nightdriverstrip_demo() { let orchestrator = fbuild_build::esp32::orchestrator::Esp32Orchestrator; let result = orchestrator .build(¶ms) + .await .expect("NightDriverStrip demo build should succeed"); assert!(result.success, "build should report success"); @@ -516,9 +522,9 @@ fn build_nightdriverstrip_demo() { /// /// Requires a prior clean build to exist at ~/dev/NightDriverStrip/.fbuild/build/demo/. /// Measures how fast a no-op rebuild is (should be seconds, not minutes). -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn incremental_nightdriverstrip_no_changes() { +async fn incremental_nightdriverstrip_no_changes() { // Try both NightDriverStrip locations let project_dir = home_dir().join("dev/NightDriverStrip"); let env_name = if project_dir.exists() { @@ -530,13 +536,13 @@ fn incremental_nightdriverstrip_no_changes() { return; } // Use the test copy - return incremental_build_at(&alt, "demo"); + return incremental_build_at(&alt, "demo").await; }; - incremental_build_at(&project_dir, &env_name); + incremental_build_at(&project_dir, &env_name).await; } -fn incremental_build_at(project_dir: &std::path::Path, env_name: &str) { +async fn incremental_build_at(project_dir: &std::path::Path, env_name: &str) { // Verify there's an existing build let build_marker = project_dir .join(".fbuild/build") @@ -579,6 +585,7 @@ fn incremental_build_at(project_dir: &std::path::Path, env_name: &str) { let orchestrator = fbuild_build::esp32::orchestrator::Esp32Orchestrator; let result = orchestrator .build(¶ms) + .await .expect("incremental build should succeed"); assert!(result.success, "incremental build should succeed"); @@ -612,9 +619,9 @@ fn incremental_build_at(project_dir: &std::path::Path, env_name: &str) { /// /// Touches one .cpp file to simulate a single-file edit, then rebuilds. /// This measures the real incremental compile + relink time. -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn incremental_nightdriverstrip_one_file_changed() { +async fn incremental_nightdriverstrip_one_file_changed() { let project_dir = home_dir().join("dev/NightDriverStrip"); if !project_dir.exists() { eprintln!("SKIP: ~/dev/NightDriverStrip does not exist"); @@ -676,6 +683,7 @@ fn incremental_nightdriverstrip_one_file_changed() { let orchestrator = fbuild_build::esp32::orchestrator::Esp32Orchestrator; let result = orchestrator .build(¶ms) + .await .expect("incremental build should succeed"); assert!(result.success, "incremental build should succeed"); diff --git a/crates/fbuild-build/tests/lite_scons_acceptance.rs b/crates/fbuild-build/tests/lite_scons_acceptance.rs index dd6ab3e7..7a872d4f 100644 --- a/crates/fbuild-build/tests/lite_scons_acceptance.rs +++ b/crates/fbuild-build/tests/lite_scons_acceptance.rs @@ -68,10 +68,11 @@ fn write_project(extra_scripts: &str, scripts: &[(&str, &str)]) -> tempfile::Tem temp } -fn resolve_lite(project_dir: &Path) -> BuildOverlay { +async fn resolve_lite(project_dir: &Path) -> BuildOverlay { let config = fbuild_config::PlatformIOConfig::from_path(&project_dir.join("platformio.ini")) .expect("parse platformio.ini"); resolve_extra_script_overlay(project_dir, "demo", &config) + .await .expect("lite-SCons harness must succeed for the 5 spike patterns") } @@ -82,8 +83,8 @@ fn resolve_lite(project_dir: &Path) -> BuildOverlay { /// MockEnv treats Execute as a no-op; the lite harness actually invokes /// the callable, captures the executed-action record, and notices the /// new file via the generated-files manifest. Tests both halves. -#[test] -fn lite_scons_executes_generator_action_and_records_file() { +#[tokio::test] +async fn lite_scons_executes_generator_action_and_records_file() { if !python_available() { return; } @@ -112,7 +113,7 @@ env.Append(CPPDEFINES=[("BUILDINFO_PRESENT", "1")]) )], ); - let overlay = resolve_lite(temp.path()); + let overlay = resolve_lite(temp.path()).await; let records = overlay .lite_scons_records .as_ref() @@ -159,8 +160,8 @@ env.Append(CPPDEFINES=[("BUILDINFO_PRESENT", "1")]) /// Marlin-class hook: glob pattern + callback name captured so fbuild's /// native compile pipeline can call the middleware per matching source. -#[test] -fn lite_scons_records_build_middleware() { +#[tokio::test] +async fn lite_scons_records_build_middleware() { if !python_available() { return; } @@ -185,7 +186,7 @@ env.Append(CCFLAGS=["-Wno-unused-parameter"]) )], ); - let overlay = resolve_lite(temp.path()); + let overlay = resolve_lite(temp.path()).await; let records = overlay .lite_scons_records .as_ref() @@ -220,8 +221,8 @@ env.Append(CCFLAGS=["-Wno-unused-parameter"]) /// OTA-style merge_bin packager: target template MUST come back /// unresolved so fbuild can subst it (`$BUILD_DIR/$PROGNAME$PROGSUFFIX`) /// after link, when it knows the actual values. -#[test] -fn lite_scons_records_post_action_with_unresolved_target_template() { +#[tokio::test] +async fn lite_scons_records_post_action_with_unresolved_target_template() { if !python_available() { return; } @@ -245,7 +246,7 @@ env.AddPostAction("$BUILD_DIR/$PROGNAME$PROGSUFFIX", merge_firmware) )], ); - let overlay = resolve_lite(temp.path()); + let overlay = resolve_lite(temp.path()).await; let records = overlay .lite_scons_records .as_ref() @@ -275,8 +276,8 @@ env.AddPostAction("$BUILD_DIR/$PROGNAME$PROGSUFFIX", merge_firmware) /// SCons resolves SConscript paths relative to the *calling* script's /// directory, not the project root. Spike bug #3 — the first iteration /// looked in PROJECT_DIR and the child was missing. -#[test] -fn lite_scons_sconscript_recursion_resolves_caller_relative_and_lands_child_mutations() { +#[tokio::test] +async fn lite_scons_sconscript_recursion_resolves_caller_relative_and_lands_child_mutations() { if !python_available() { return; } @@ -303,7 +304,7 @@ env.SConscript("child.py") ) .expect("write child.py"); - let overlay = resolve_lite(temp.path()); + let overlay = resolve_lite(temp.path()).await; assert!( overlay @@ -327,8 +328,8 @@ env.SConscript("child.py") /// Real SCons handles both `-Ipath` and `-I path`. The spike's first /// iteration only handled the joined form — bug #2 of the 3 caught. -#[test] -fn lite_scons_parseflags_handles_joined_and_space_separated_forms() { +#[tokio::test] +async fn lite_scons_parseflags_handles_joined_and_space_separated_forms() { if !python_available() { return; } @@ -348,7 +349,7 @@ env.Append(**parsed) )], ); - let overlay = resolve_lite(temp.path()); + let overlay = resolve_lite(temp.path()).await; let common = &overlay.global_compile.common; let common_joined = common.join(" "); diff --git a/crates/fbuild-build/tests/nxplpc_build_flags.rs b/crates/fbuild-build/tests/nxplpc_build_flags.rs index 6b72ee83..10cb73d1 100644 --- a/crates/fbuild-build/tests/nxplpc_build_flags.rs +++ b/crates/fbuild-build/tests/nxplpc_build_flags.rs @@ -37,9 +37,9 @@ fn fixture_dir() -> std::path::PathBuf { .join("tests/platform/lpc845_build_flags") } -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "downloads ARM GCC + ArduinoCore-LPC8xx + builds firmware; CI-only"] -fn lpc845brk_propagates_build_flags_to_library_compile_587() { +async fn lpc845brk_propagates_build_flags_to_library_compile_587() { let fixture = fixture_dir(); assert!( fixture.join("platformio.ini").is_file(), @@ -75,6 +75,7 @@ fn lpc845brk_propagates_build_flags_to_library_compile_587() { let orchestrator = fbuild_build::nxplpc::orchestrator::NxpLpcOrchestrator; let result = orchestrator .build(¶ms) + .await .expect("#587 regression: lpc845brk build with check_flag library must succeed"); assert!( result.success, diff --git a/crates/fbuild-build/tests/nxplpc_core_compile_commands.rs b/crates/fbuild-build/tests/nxplpc_core_compile_commands.rs index b0e65906..531d1572 100644 --- a/crates/fbuild-build/tests/nxplpc_core_compile_commands.rs +++ b/crates/fbuild-build/tests/nxplpc_core_compile_commands.rs @@ -14,7 +14,7 @@ fn arduino_core_repo() -> Option { repo.join("platformio.ini").is_file().then_some(repo) } -fn build_core_repo(repo: &Path, env_name: &str) -> tempfile::TempDir { +async fn build_core_repo(repo: &Path, env_name: &str) -> tempfile::TempDir { let tmp = tempfile::TempDir::new().expect("tempdir"); let build_dir = tmp .path() @@ -46,19 +46,20 @@ fn build_core_repo(repo: &Path, env_name: &str) -> tempfile::TempDir { let orchestrator = fbuild_build::nxplpc::orchestrator::NxpLpcOrchestrator; let result = orchestrator .build(¶ms) + .await .expect("ArduinoCore-LPC8xx nxplpc build should succeed"); assert!(result.success); tmp } -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "requires local ~/dev/ArduinoCore-LPC8xx checkout and ARM toolchain package"] -fn arduino_core_lpc845brk_compile_commands_match_platform_txt() { +async fn arduino_core_lpc845brk_compile_commands_match_platform_txt() { let Some(repo) = arduino_core_repo() else { eprintln!("skipping: ~/dev/ArduinoCore-LPC8xx not found"); return; }; - let tmp = build_core_repo(&repo, "lpc845brk"); + let tmp = build_core_repo(&repo, "lpc845brk").await; let compile_db = tmp .path() .join(".fbuild/build/lpc845brk/release/compile_commands.json"); diff --git a/crates/fbuild-build/tests/stm32_acceptance.rs b/crates/fbuild-build/tests/stm32_acceptance.rs index ee47d5af..b7afba3b 100644 --- a/crates/fbuild-build/tests/stm32_acceptance.rs +++ b/crates/fbuild-build/tests/stm32_acceptance.rs @@ -32,9 +32,9 @@ use fbuild_build::{BuildOrchestrator, BuildParams}; use fbuild_core::BuildProfile; use fbuild_test_support::{CompileDb, ElfProbe}; -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "downloads STM32duino + builds firmware; CI-only"] -fn stm32f103c8_blink_with_spi_auto_discovers_library_205_ac4() { +async fn stm32f103c8_blink_with_spi_auto_discovers_library_205_ac4() { // Use a temporary project dir so we can write our own SPI-using sketch // independent of whatever ships in the fixture. let tmp = tempfile::TempDir::new().unwrap(); @@ -85,6 +85,7 @@ fn stm32f103c8_blink_with_spi_auto_discovers_library_205_ac4() { let orchestrator = fbuild_build::stm32::orchestrator::Stm32Orchestrator; let result = orchestrator .build(¶ms) + .await .expect("stm32f103c8 build with SPI must succeed"); assert!(result.success, "build did not report success"); diff --git a/crates/fbuild-build/tests/teensy30_acceptance.rs b/crates/fbuild-build/tests/teensy30_acceptance.rs index 728c82eb..92115759 100644 --- a/crates/fbuild-build/tests/teensy30_acceptance.rs +++ b/crates/fbuild-build/tests/teensy30_acceptance.rs @@ -39,9 +39,9 @@ use fbuild_build::{BuildOrchestrator, BuildParams}; use fbuild_core::BuildProfile; use fbuild_test_support::{CompileDb, ElfProbe}; -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "downloads Teensyduino + arm-gcc; CI-only"] -fn teensy30_analog_output_meets_205_ac2() { +async fn teensy30_analog_output_meets_205_ac2() { // Use a temporary project dir so the committed teensy30 fixture // at tests/platform/teensy30/ stays untouched and no scratch // build artifacts land in the repo. @@ -102,6 +102,7 @@ fn teensy30_analog_output_meets_205_ac2() { let result = fbuild_build::teensy::orchestrator::TeensyOrchestrator .build(¶ms) + .await .expect("teensy30 AnalogOutput build must succeed for AC#2 gate"); assert!(result.success, "build did not report success"); diff --git a/crates/fbuild-build/tests/teensy41_acceptance.rs b/crates/fbuild-build/tests/teensy41_acceptance.rs index 54b491c3..4214680b 100644 --- a/crates/fbuild-build/tests/teensy41_acceptance.rs +++ b/crates/fbuild-build/tests/teensy41_acceptance.rs @@ -37,9 +37,9 @@ use fbuild_library_select::resolve; use fbuild_packages::library::TeensyCores; use fbuild_packages::Package; -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "downloads Teensyduino + arm-gcc; CI-only"] -fn teensy41_cold_library_selection_meets_205_ac6() { +async fn teensy41_cold_library_selection_meets_205_ac6() { // Inline tempdir project — same root-cause-isolation pattern as // stm32_acceptance.rs / teensy30_acceptance.rs. AC#6 needs only the // sketch on disk; we don't run a full build, only the resolver. @@ -78,8 +78,9 @@ fn teensy41_cold_library_selection_meets_205_ac6() { // Materialize Teensyduino. Idempotent — cached across runs on the // CI runner once the package has been downloaded once. let teensy_cores = TeensyCores::new(project_dir); - let framework_dir = - Package::ensure_installed(&teensy_cores).expect("Teensyduino must install for AC#6 gate"); + let framework_dir = Package::ensure_installed(&teensy_cores) + .await + .expect("Teensyduino must install for AC#6 gate"); println!( "AC#6 teensy41 framework installed at {}", framework_dir.display() diff --git a/crates/fbuild-build/tests/teensy_build.rs b/crates/fbuild-build/tests/teensy_build.rs index ee022552..64e34643 100644 --- a/crates/fbuild-build/tests/teensy_build.rs +++ b/crates/fbuild-build/tests/teensy_build.rs @@ -28,9 +28,9 @@ fn copy_dir_recursive(src: &Path, dst: &Path) { /// Build a self-contained Teensy 4.1 blink sketch. /// /// This test requires Internet access (first run only, then cached). -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_teensy41_blink() { +async fn build_teensy41_blink() { let tmp = tempfile::TempDir::new().unwrap(); let project_dir = tmp.path(); @@ -86,6 +86,7 @@ void loop() { let orchestrator = fbuild_build::teensy::orchestrator::TeensyOrchestrator; let result = orchestrator .build(¶ms) + .await .expect("Teensy build should succeed"); assert!(result.success); @@ -104,9 +105,9 @@ void loop() { } /// Build a Teensy 4.1 sketch that includes Teensyduino framework libraries. -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_teensy41_spi_octo_headers() { +async fn build_teensy41_spi_octo_headers() { let tmp = tempfile::TempDir::new().unwrap(); let project_dir = tmp.path(); @@ -159,6 +160,7 @@ void loop() {} let orchestrator = fbuild_build::teensy::orchestrator::TeensyOrchestrator; let result = orchestrator .build(¶ms) + .await .expect("Teensy framework library headers should build"); assert!(result.success); @@ -166,9 +168,9 @@ void loop() {} } /// Build using Teensy test fixture from the repo. -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_teensy41_fixture() { +async fn build_teensy41_fixture() { // Use the repo's test fixture let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let project_dir = manifest_dir @@ -210,6 +212,7 @@ fn build_teensy41_fixture() { let orchestrator = fbuild_build::teensy::orchestrator::TeensyOrchestrator; let result = orchestrator .build(¶ms) + .await .expect("Teensy fixture build should succeed"); assert!(result.success); @@ -231,9 +234,9 @@ fn build_teensy41_fixture() { } /// Build a Teensy 3.0 fixture where a project-local lib/FastLED shadows the bundled framework. -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore] -fn build_teensy30_fixture_prefers_local_fastled() { +async fn build_teensy30_fixture_prefers_local_fastled() { let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); let fixture_dir = manifest_dir .parent() @@ -320,6 +323,7 @@ void loop() { let orchestrator = fbuild_build::teensy::orchestrator::TeensyOrchestrator; let result = orchestrator .build(¶ms) + .await .expect("Teensy 3.0 local FastLED shadow build should succeed"); assert!(result.success); diff --git a/crates/fbuild-build/tests/teensylc_acceptance.rs b/crates/fbuild-build/tests/teensylc_acceptance.rs index a61d8716..49202301 100644 --- a/crates/fbuild-build/tests/teensylc_acceptance.rs +++ b/crates/fbuild-build/tests/teensylc_acceptance.rs @@ -22,9 +22,9 @@ use fbuild_build::{BuildOrchestrator, BuildParams}; use fbuild_core::BuildProfile; use fbuild_test_support::{CompileDb, ElfProbe}; -#[test] +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "downloads Teensyduino + builds firmware; CI-only"] -fn teensylc_blink_meets_205_acceptance_criteria() { +async fn teensylc_blink_meets_205_acceptance_criteria() { let project_dir = repo_fixture("teensylc"); let build_dir = tempfile::TempDir::new().unwrap(); @@ -54,6 +54,7 @@ fn teensylc_blink_meets_205_acceptance_criteria() { let result = fbuild_build::teensy::orchestrator::TeensyOrchestrator .build(¶ms) + .await .expect("teensyLC build must succeed for acceptance gate"); assert!(result.success, "build did not report success"); diff --git a/crates/fbuild-build/tests/zccache_hit_across_workspace_rename.rs b/crates/fbuild-build/tests/zccache_hit_across_workspace_rename.rs deleted file mode 100644 index 132b6bb4..00000000 --- a/crates/fbuild-build/tests/zccache_hit_across_workspace_rename.rs +++ /dev/null @@ -1,321 +0,0 @@ -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::{env, fs}; - -use fbuild_build::compiler::compile_source; - -const FAKE_ZCCACHE: &str = r#" -use std::env; -use std::fs; -use std::io::Write; -use std::path::{Component, Path, PathBuf}; - -fn main() { - let args: Vec = env::args().skip(1).collect(); - if args.len() < 2 || args[0] != "wrap" { - eprintln!("usage: fake-zccache wrap "); - std::process::exit(2); - } - - let cwd = env::current_dir().unwrap(); - let expanded = expand_response_files(&args[2..]); - let source = find_source(&expanded, &cwd).expect("source file"); - let output = find_output(&expanded, &cwd).expect("output file"); - let includes = find_includes(&expanded, &cwd); - - let key = cache_key(&cwd, &source, &includes); - let key_hash = stable_hash(key.as_bytes()); - let cache_dir = PathBuf::from(env::var("FBUILD_FAKE_ZCCACHE_CACHE").unwrap()); - let log_path = PathBuf::from(env::var("FBUILD_FAKE_ZCCACHE_LOG").unwrap()); - fs::create_dir_all(&cache_dir).unwrap(); - if let Some(parent) = output.parent() { - fs::create_dir_all(parent).unwrap(); - } - - let cache_path = cache_dir.join(format!("{key_hash:016x}.o")); - let mut log = fs::OpenOptions::new() - .create(true) - .append(true) - .open(log_path) - .unwrap(); - - if cache_path.exists() { - fs::copy(&cache_path, &output).unwrap(); - writeln!(log, "hit cwd={} key={key_hash:016x}", cwd.display()).unwrap(); - } else { - let object = format!("object\n{}\n", key); - fs::write(&output, object.as_bytes()).unwrap(); - fs::copy(&output, &cache_path).unwrap(); - writeln!(log, "miss cwd={} key={key_hash:016x}", cwd.display()).unwrap(); - } -} - -fn expand_response_files(args: &[String]) -> Vec { - let mut expanded = Vec::new(); - for arg in args { - if let Some(path) = arg.strip_prefix('@') { - let text = fs::read_to_string(path).unwrap(); - expanded.extend(text.split_whitespace().map(unquote)); - } else { - expanded.push(arg.clone()); - } - } - expanded -} - -fn unquote(value: &str) -> String { - value - .trim_matches('"') - .trim_matches('\'') - .to_string() -} - -fn find_source(args: &[String], cwd: &Path) -> Option { - let mut after_c = false; - for arg in args { - if after_c { - return Some(resolve(arg, cwd)); - } - after_c = arg == "-c"; - } - args.iter() - .find(|arg| !arg.starts_with('-') && is_source(arg)) - .map(|arg| resolve(arg, cwd)) -} - -fn find_output(args: &[String], cwd: &Path) -> Option { - let mut i = 0; - while i < args.len() { - let arg = &args[i]; - if arg == "-o" { - return args.get(i + 1).map(|value| resolve(value, cwd)); - } - if let Some(value) = arg.strip_prefix("-o") { - return Some(resolve(value, cwd)); - } - i += 1; - } - None -} - -fn find_includes(args: &[String], cwd: &Path) -> Vec { - let mut includes = Vec::new(); - let mut i = 0; - while i < args.len() { - let arg = &args[i]; - if arg == "-I" { - if let Some(value) = args.get(i + 1) { - includes.push(resolve(value, cwd)); - } - i += 2; - continue; - } - if let Some(value) = arg.strip_prefix("-I") { - includes.push(resolve(value, cwd)); - } - i += 1; - } - includes -} - -fn cache_key(cwd: &Path, source: &Path, includes: &[PathBuf]) -> String { - let mut key = String::new(); - key.push_str("source="); - key.push_str(&key_path(source, cwd)); - key.push(':'); - key.push_str(&fs::read_to_string(source).unwrap()); - key.push('\n'); - - for include in includes { - key.push_str("include-dir="); - key.push_str(&key_path(include, cwd)); - key.push('\n'); - let header = include.join("demo.h"); - key.push_str("header="); - key.push_str(&key_path(&header, cwd)); - key.push(':'); - key.push_str(&fs::read_to_string(header).unwrap()); - key.push('\n'); - } - key -} - -fn key_path(path: &Path, cwd: &Path) -> String { - let absolute = if path.is_absolute() { - path.to_path_buf() - } else { - cwd.join(path) - }; - let comparable = absolute.strip_prefix(cwd).unwrap_or(&absolute); - comparable - .components() - .filter_map(|component| match component { - Component::Prefix(prefix) => Some(prefix.as_os_str().to_string_lossy().replace('\\', "/")), - Component::RootDir | Component::CurDir => None, - Component::ParentDir => Some("..".to_string()), - Component::Normal(value) => Some(value.to_string_lossy().replace('\\', "/")), - }) - .collect::>() - .join("/") -} - -fn resolve(value: &str, cwd: &Path) -> PathBuf { - let path = Path::new(value); - if path.is_absolute() { - path.to_path_buf() - } else { - cwd.join(path) - } -} - -fn is_source(value: &str) -> bool { - Path::new(value) - .extension() - .and_then(|ext| ext.to_str()) - .is_some_and(|ext| matches!(ext, "c" | "cc" | "cpp" | "cxx")) -} - -fn stable_hash(bytes: &[u8]) -> u64 { - let mut hash = 0xcbf29ce484222325u64; - for byte in bytes { - hash ^= u64::from(*byte); - hash = hash.wrapping_mul(0x100000001b3); - } - hash -} -"#; - -struct CurrentDirGuard { - original: PathBuf, -} - -impl CurrentDirGuard { - fn set_to(path: &Path) -> Self { - let original = env::current_dir().unwrap(); - env::set_current_dir(path).unwrap(); - Self { original } - } -} - -impl Drop for CurrentDirGuard { - fn drop(&mut self) { - let _ = env::set_current_dir(&self.original); - } -} - -#[test] -fn zccache_hit_across_workspace_rename() { - let tmp = tempfile::TempDir::new().unwrap(); - let fake_zccache = compile_fake_zccache(tmp.path()); - let fake_compiler = tmp - .path() - .join(format!("fake-compiler{}", env::consts::EXE_SUFFIX)); - let cache_dir = tmp.path().join("fake-cache"); - let log_path = tmp.path().join("fake-zccache.log"); - let ws_a = tmp.path().join("workspace-a"); - let ws_b = tmp.path().join("workspace-b"); - - create_workspace(&ws_a); - create_workspace(&ws_b); - let expected_ws_a = cwd_display_path(&ws_a); - let expected_ws_b = cwd_display_path(&ws_b); - - let _cwd = CurrentDirGuard::set_to(tmp.path()); - env::set_var("FBUILD_FAKE_ZCCACHE_CACHE", &cache_dir); - env::set_var("FBUILD_FAKE_ZCCACHE_LOG", &log_path); - - compile_workspace(&ws_a, &fake_compiler, &fake_zccache); - compile_workspace(&ws_b, &fake_compiler, &fake_zccache); - - let log = fs::read_to_string(&log_path).unwrap(); - let lines: Vec<&str> = log.lines().collect(); - assert_eq!(lines.len(), 2, "unexpected fake zccache log:\n{log}"); - assert!( - lines[0].starts_with("miss "), - "first compile should populate the cache:\n{log}" - ); - assert!( - lines[1].starts_with("hit "), - "renamed workspace should reuse the cache entry:\n{log}" - ); - assert!( - lines[0].contains(&format!("cwd={expected_ws_a}")), - "first wrapper CWD should be workspace root:\n{log}" - ); - assert!( - lines[1].contains(&format!("cwd={expected_ws_b}")), - "second wrapper CWD should be workspace root:\n{log}" - ); -} - -fn cwd_display_path(path: &Path) -> String { - let path = path.canonicalize().unwrap_or_else(|_| path.to_path_buf()); - let display = path.display().to_string(); - display - .strip_prefix(r"\\?\") - .unwrap_or(&display) - .to_string() -} - -fn compile_fake_zccache(root: &Path) -> PathBuf { - let source = root.join("fake_zccache.rs"); - let exe = root.join(format!("fake-zccache{}", env::consts::EXE_SUFFIX)); - fs::write(&source, FAKE_ZCCACHE).unwrap(); - - let rustc = env::var_os("RUSTC").unwrap_or_else(|| "rustc".into()); - // allow-direct-spawn: test helper compiling a throwaway rustc binary. - let status = Command::new(rustc) - .arg(&source) - .arg("-o") - .arg(&exe) - .status() - .expect("failed to spawn rustc for fake zccache"); - assert!(status.success(), "failed to compile fake zccache helper"); - exe -} - -fn create_workspace(root: &Path) { - fs::create_dir_all(root.join("src")).unwrap(); - fs::create_dir_all(root.join("include")).unwrap(); - fs::create_dir_all(root.join(".fbuild").join("build")).unwrap(); - fs::write( - root.join("include").join("demo.h"), - "#pragma once\ninline int demo() { return 7; }\n", - ) - .unwrap(); - fs::write( - root.join("src").join("main.cpp"), - "#include \"demo.h\"\nint main() { return demo(); }\n", - ) - .unwrap(); -} - -fn compile_workspace(root: &Path, compiler: &Path, zccache: &Path) { - let source = root.join("src").join("main.cpp"); - let output = root.join(".fbuild").join("build").join("main.o"); - let flags = vec![ - "-I".to_string(), - root.join("include").to_string_lossy().to_string(), - ]; - - let result = compile_source( - compiler, - &source, - &output, - &flags, - &[], - &root.join(".fbuild").join("build").join("tmp"), - "zccache-rename", - false, - Some(zccache), - &[], - ) - .unwrap(); - - assert!( - result.success, - "compile failed: stdout={} stderr={}", - result.stdout, result.stderr - ); - assert!(output.exists(), "expected object at {}", output.display()); -} diff --git a/crates/fbuild-cli/src/cli/bloat_lookup.rs b/crates/fbuild-cli/src/cli/bloat_lookup.rs index 818aee4e..aedff170 100644 --- a/crates/fbuild-cli/src/cli/bloat_lookup.rs +++ b/crates/fbuild-cli/src/cli/bloat_lookup.rs @@ -20,7 +20,7 @@ use fbuild_core::{FbuildError, Result}; use super::symbols_cmd::resolve_tool_paths_public; #[allow(clippy::too_many_arguments)] -pub fn run_bloat_lookup( +pub async fn run_bloat_lookup( input: String, symbol: Option, symbol_mangled: Option, @@ -69,7 +69,7 @@ pub fn run_bloat_lookup( cppfilt_path: cppfilt_path.as_deref(), objdump_path: objdump_path.as_deref(), }; - let report = analyze_elf(cfg)?; + let report = analyze_elf(cfg).await?; let query_str = symbol.clone().or_else(|| symbol_mangled.clone()).unwrap(); let query = match (&symbol, &symbol_mangled) { diff --git a/crates/fbuild-cli/src/cli/build.rs b/crates/fbuild-cli/src/cli/build.rs index 3c47c2f9..750d3f3b 100644 --- a/crates/fbuild-cli/src/cli/build.rs +++ b/crates/fbuild-cli/src/cli/build.rs @@ -2,7 +2,7 @@ use crate::daemon_client::{self, BuildRequest, DaemonClient}; -pub fn open_in_browser(url: &str) -> fbuild_core::Result<()> { +pub async fn open_in_browser(url: &str) -> fbuild_core::Result<()> { let args: Vec<&str> = if cfg!(target_os = "windows") { vec!["cmd", "/c", "start", "", url] } else if cfg!(target_os = "macos") { @@ -11,6 +11,7 @@ pub fn open_in_browser(url: &str) -> fbuild_core::Result<()> { vec!["xdg-open", url] }; let output = fbuild_core::subprocess::run_command(&args, None, None, None) + .await .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to launch browser: {}", e)))?; if output.success() { diff --git a/crates/fbuild-cli/src/cli/compile_many.rs b/crates/fbuild-cli/src/cli/compile_many.rs index fee0be0e..45b92678 100644 --- a/crates/fbuild-cli/src/cli/compile_many.rs +++ b/crates/fbuild-cli/src/cli/compile_many.rs @@ -151,13 +151,9 @@ pub async fn run_compile_many(args: CompileManyArgs) -> fbuild_core::Result<()> effective_sketch, ); - // `compile_many` is fully synchronous (CPU-bound). Run it on a - // blocking pool so we don't tie up the tokio runtime thread. - let result = tokio::task::spawn_blocking(move || compile_many(req)) - .await - .map_err(|e| { - fbuild_core::FbuildError::Other(format!("compile-many task panicked: {e}")) - })??; + // `compile_many` is async (driving per-stage tokio fanout). Await it + // directly — the runtime is already multi-threaded. + let result = compile_many(req).await?; // Per-sketch result map suitable for the bench summary. println!(); diff --git a/crates/fbuild-cli/src/cli/daemon_cmd.rs b/crates/fbuild-cli/src/cli/daemon_cmd.rs index c3321763..fe20ea81 100644 --- a/crates/fbuild-cli/src/cli/daemon_cmd.rs +++ b/crates/fbuild-cli/src/cli/daemon_cmd.rs @@ -96,7 +96,7 @@ pub async fn run_daemon(action: DaemonAction) -> fbuild_core::Result<()> { run_daemon_kill(&client, pid, force).await?; } DaemonAction::KillAll { force } => { - run_daemon_kill_all(force)?; + run_daemon_kill_all(force).await?; } DaemonAction::Locks => { run_daemon_locks(&client).await?; @@ -292,7 +292,7 @@ pub async fn run_daemon_list(client: &DaemonClient) -> fbuild_core::Result<()> { } // Also scan for orphan processes - let pids = find_daemon_pids()?; + let pids = find_daemon_pids().await?; if pids.len() > 1 { println!("\nwarning: multiple fbuild-daemon processes detected:"); for pid in &pids { @@ -493,7 +493,7 @@ pub async fn run_daemon_kill( read_pid_from_file()? }; - kill_process(target_pid, force)?; + kill_process(target_pid, force).await?; println!("killed daemon (PID {})", target_pid); let _ = std::fs::remove_file(fbuild_paths::get_daemon_pid_file()); Ok(()) @@ -517,8 +517,8 @@ pub fn read_pid_from_file() -> fbuild_core::Result { } } -pub fn run_daemon_kill_all(force: bool) -> fbuild_core::Result<()> { - let pids = find_daemon_pids()?; +pub async fn run_daemon_kill_all(force: bool) -> fbuild_core::Result<()> { + let pids = find_daemon_pids().await?; if pids.is_empty() { println!("no fbuild-daemon processes found"); return Ok(()); @@ -526,7 +526,7 @@ pub fn run_daemon_kill_all(force: bool) -> fbuild_core::Result<()> { let mut killed = 0; for pid in &pids { - match kill_process(*pid, force) { + match kill_process(*pid, force).await { Ok(()) => { println!("killed daemon (PID {})", pid); killed += 1; @@ -542,7 +542,7 @@ pub fn run_daemon_kill_all(force: bool) -> fbuild_core::Result<()> { Ok(()) } -pub fn kill_process(pid: u32, force: bool) -> fbuild_core::Result<()> { +pub async fn kill_process(pid: u32, force: bool) -> fbuild_core::Result<()> { let pid_str = pid.to_string(); let argv: Vec<&str> = if cfg!(windows) { if force { @@ -555,9 +555,11 @@ pub fn kill_process(pid: u32, force: bool) -> fbuild_core::Result<()> { vec!["kill", signal, &pid_str] }; - let output = fbuild_core::subprocess::run_command(&argv, None, None, None).map_err(|e| { - fbuild_core::FbuildError::Other(format!("failed to execute kill command: {}", e)) - })?; + let output = fbuild_core::subprocess::run_command(&argv, None, None, None) + .await + .map_err(|e| { + fbuild_core::FbuildError::Other(format!("failed to execute kill command: {}", e)) + })?; if !output.success() { return Err(fbuild_core::FbuildError::Other(format!( @@ -568,7 +570,7 @@ pub fn kill_process(pid: u32, force: bool) -> fbuild_core::Result<()> { Ok(()) } -pub fn find_daemon_pids() -> fbuild_core::Result> { +pub async fn find_daemon_pids() -> fbuild_core::Result> { if cfg!(windows) { let output = fbuild_core::subprocess::run_command( &[ @@ -583,6 +585,7 @@ pub fn find_daemon_pids() -> fbuild_core::Result> { None, None, ) + .await .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to run tasklist: {}", e)))?; let mut pids = Vec::new(); for line in output.stdout.lines() { @@ -605,6 +608,7 @@ pub fn find_daemon_pids() -> fbuild_core::Result> { None, None, ) + .await .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to run pgrep: {}", e)))?; let pids: Vec = output .stdout diff --git a/crates/fbuild-cli/src/cli/deploy.rs b/crates/fbuild-cli/src/cli/deploy.rs index efc35deb..1a84fd40 100644 --- a/crates/fbuild-cli/src/cli/deploy.rs +++ b/crates/fbuild-cli/src/cli/deploy.rs @@ -222,7 +222,7 @@ pub async fn run_deploy( // Open browser for avr8js only when daemon returned a launch URL (non-headless mode) if deploy_route == CliDeployRoute::Emulator(CliEmulatorKind::Avr8js) { if let Some(url) = resp.launch_url.as_deref() { - if let Err(e) = open_in_browser(url) { + if let Err(e) = open_in_browser(url).await { eprintln!("warning: failed to open browser: {}", e); eprintln!("open this URL manually: {}", url); } diff --git a/crates/fbuild-cli/src/cli/dispatch.rs b/crates/fbuild-cli/src/cli/dispatch.rs index e9fb9ab1..715754fe 100644 --- a/crates/fbuild-cli/src/cli/dispatch.rs +++ b/crates/fbuild-cli/src/cli/dispatch.rs @@ -78,38 +78,28 @@ pub async fn async_main() { graph_fan_out, graph_collapse_archive, graph_exclude_archive, - }) => run_symbols( - input, - map, - nm, - cppfilt, - build_info, - json, - output_dir, - top, - no_graph, - graph_top, - graph_min_bytes, - graph_depth, - graph_fan_out, - graph_collapse_archive, - graph_exclude_archive, - ), - Some(Commands::Bloat { cmd }) => match cmd { - BloatCmd::Graph { + }) => { + run_symbols( input, - symbol, map, nm, cppfilt, build_info, - output, - depth, - fan_out, - max_depth, - collapse_archive, - exclude_archive, - } => run_bloat_graph( + json, + output_dir, + top, + no_graph, + graph_top, + graph_min_bytes, + graph_depth, + graph_fan_out, + graph_collapse_archive, + graph_exclude_archive, + ) + .await + } + Some(Commands::Bloat { cmd }) => match cmd { + BloatCmd::Graph { input, symbol, map, @@ -122,7 +112,23 @@ pub async fn async_main() { max_depth, collapse_archive, exclude_archive, - ), + } => { + run_bloat_graph( + input, + symbol, + map, + nm, + cppfilt, + build_info, + output, + depth, + fan_out, + max_depth, + collapse_archive, + exclude_archive, + ) + .await + } BloatCmd::Lookup { input, symbol, @@ -132,16 +138,19 @@ pub async fn async_main() { nm, cppfilt, build_info, - } => run_bloat_lookup( - input, - symbol, - symbol_mangled, - json, - map, - nm, - cppfilt, - build_info, - ), + } => { + run_bloat_lookup( + input, + symbol, + symbol_mangled, + json, + map, + nm, + cppfilt, + build_info, + ) + .await + } }, Some(Commands::Build { project_dir, @@ -163,7 +172,7 @@ pub async fn async_main() { }) => { let project_dir = resolve_project_dir(project_dir, &top_level_project_dir); if platformio { - pio_build(&project_dir, environment.as_deref(), clean, verbose) + pio_build(&project_dir, environment.as_deref(), clean, verbose).await } else { run_build( project_dir, @@ -216,6 +225,7 @@ pub async fn async_main() { clean, verbose, ) + .await } else { let monitor_after = monitor.is_some(); let parsed = monitor @@ -268,6 +278,7 @@ pub async fn async_main() { port.as_deref(), baud_rate, ) + .await } else { run_monitor( project_dir, @@ -485,6 +496,7 @@ pub async fn async_main() { cli.clean, cli.verbose, ) + .await } else { let monitor_after = true; let parsed = cli diff --git a/crates/fbuild-cli/src/cli/graph_cmd.rs b/crates/fbuild-cli/src/cli/graph_cmd.rs index 19852cbe..ae695f22 100644 --- a/crates/fbuild-cli/src/cli/graph_cmd.rs +++ b/crates/fbuild-cli/src/cli/graph_cmd.rs @@ -21,7 +21,7 @@ use fbuild_core::{FbuildError, Result}; use super::symbols_cmd::resolve_tool_paths_public; #[allow(clippy::too_many_arguments)] -pub fn run_bloat_graph( +pub async fn run_bloat_graph( input: String, symbol: String, map: Option, @@ -68,7 +68,7 @@ pub fn run_bloat_graph( cppfilt_path: cppfilt_path.as_deref(), objdump_path: objdump_path.as_deref(), }; - let report = analyze_elf(cfg)?; + let report = analyze_elf(cfg).await?; let graph_config = parse_graph_config( &depth, diff --git a/crates/fbuild-cli/src/cli/pio.rs b/crates/fbuild-cli/src/cli/pio.rs index dc48aa7b..cb152f97 100644 --- a/crates/fbuild-cli/src/cli/pio.rs +++ b/crates/fbuild-cli/src/cli/pio.rs @@ -5,10 +5,12 @@ //! fbuild daemon or the upstream `pio` CLI for A/B comparisons. /// Find the `pio` binary. Checks PATH first, then the fbuild cache. -pub fn find_pio() -> fbuild_core::Result { +pub async fn find_pio() -> fbuild_core::Result { // Check PATH let locator = if cfg!(windows) { "where" } else { "which" }; - if let Ok(output) = fbuild_core::subprocess::run_command(&[locator, "pio"], None, None, None) { + if let Ok(output) = + fbuild_core::subprocess::run_command(&[locator, "pio"], None, None, None).await + { if output.success() { let path = output .stdout @@ -45,12 +47,13 @@ pub fn find_pio() -> fbuild_core::Result { } /// Run a PlatformIO command with real-time output streaming. -pub fn run_pio_command(args: &[&str]) -> fbuild_core::Result<()> { - let pio = find_pio()?; +pub async fn run_pio_command(args: &[&str]) -> fbuild_core::Result<()> { + let pio = find_pio().await?; let pio_str = pio.to_string_lossy(); let mut argv: Vec<&str> = vec![pio_str.as_ref()]; argv.extend_from_slice(args); let code = fbuild_core::subprocess::run_command_passthrough(&argv, None, None, None) + .await .map_err(|e| fbuild_core::FbuildError::Other(format!("failed to run pio: {}", e)))?; if code != 0 { @@ -59,7 +62,7 @@ pub fn run_pio_command(args: &[&str]) -> fbuild_core::Result<()> { Ok(()) } -pub fn pio_build( +pub async fn pio_build( project_dir: &str, environment: Option<&str>, clean: bool, @@ -70,7 +73,7 @@ pub fn pio_build( if let Some(env) = environment { args.extend(["-e", env]); } - let _ = run_pio_command(&args); + let _ = run_pio_command(&args).await; } let mut args = vec!["run", "-d", project_dir]; if let Some(env) = environment { @@ -79,10 +82,10 @@ pub fn pio_build( if verbose { args.push("-v"); } - run_pio_command(&args) + run_pio_command(&args).await } -pub fn pio_deploy( +pub async fn pio_deploy( project_dir: &str, environment: Option<&str>, port: Option<&str>, @@ -94,7 +97,7 @@ pub fn pio_deploy( if let Some(env) = environment { args.extend(["-e", env]); } - let _ = run_pio_command(&args); + let _ = run_pio_command(&args).await; } let mut args = vec!["run", "--target", "upload", "-d", project_dir]; if let Some(env) = environment { @@ -106,10 +109,10 @@ pub fn pio_deploy( if verbose { args.push("-v"); } - run_pio_command(&args) + run_pio_command(&args).await } -pub fn pio_monitor( +pub async fn pio_monitor( project_dir: &str, environment: Option<&str>, port: Option<&str>, @@ -127,5 +130,5 @@ pub fn pio_monitor( baud_str = b.to_string(); args.extend(["--baud", &baud_str]); } - run_pio_command(&args) + run_pio_command(&args).await } diff --git a/crates/fbuild-cli/src/cli/symbols_cmd.rs b/crates/fbuild-cli/src/cli/symbols_cmd.rs index eea434c6..7d0c237a 100644 --- a/crates/fbuild-cli/src/cli/symbols_cmd.rs +++ b/crates/fbuild-cli/src/cli/symbols_cmd.rs @@ -27,7 +27,7 @@ use fbuild_core::{FbuildError, Result}; use super::graph_cmd::parse_graph_config; #[allow(clippy::too_many_arguments)] -pub fn run_symbols( +pub async fn run_symbols( input: String, map: Option, nm: Option, @@ -86,7 +86,7 @@ pub fn run_symbols( objdump_path: tool_paths.objdump.as_deref(), }; - let report = analyze_elf(cfg)?; + let report = analyze_elf(cfg).await?; let mut wrote_anything = false; diff --git a/crates/fbuild-core/Cargo.toml b/crates/fbuild-core/Cargo.toml index a429a409..17943a6f 100644 --- a/crates/fbuild-core/Cargo.toml +++ b/crates/fbuild-core/Cargo.toml @@ -32,7 +32,13 @@ prost = { workspace = true } running-process = { workspace = true } # Async helpers for the tokio integration in `containment::tokio_spawn` — # we need the `process` feature so `tokio::process::Command` is in scope. +# Also drives the async `subprocess::run_command*` surface introduced +# in the full-async runtime conversion (#813). tokio = { workspace = true } +# Used by future async trait surfaces folded under this crate. Added in +# the #813 Phase A foundation so downstream crates can `use async_trait` +# without each one redeclaring the workspace dep. +async-trait = { workspace = true } # `libc::setpgid` / `libc::prctl` for the Unix `pre_exec` hook used by # `containment::tokio_spawn::configure`. See FastLED/fbuild#32. diff --git a/crates/fbuild-core/src/README.md b/crates/fbuild-core/src/README.md index 1594ed62..2035d5ac 100644 --- a/crates/fbuild-core/src/README.md +++ b/crates/fbuild-core/src/README.md @@ -5,6 +5,6 @@ - **`lib.rs`** -- Crate root; defines `FbuildError`, `Result`, `BuildProfile`, `Platform`, `OperationType`, `DaemonState`, `SizeInfo`; re-exports `BuildLog` - **`build_log.rs`** -- `BuildLog` struct that accumulates output lines and optionally streams them through an `mpsc::Sender` - **`compiler_flags.rs`** -- `prepare_flags_for_exec()` to strip backslash-escaped quotes from GCC define flags on non-Windows platforms -- **`response_file.rs`** -- `write_response_file()` for GCC `@file` syntax on Windows; `replace_path_backslashes()` and `windows_temp_dir()` helpers +- **`response_file.rs`** -- `async fn write_response_file()` (and `write_response_file_blocking` for sync escape) for GCC `@file` syntax on Windows; `replace_path_backslashes()` and `windows_temp_dir()` helpers - **`shell_split.rs`** -- `split()` function for quote-aware tokenization that preserves backslashes as literal characters -- **`subprocess.rs`** -- `run_command()` with optional timeout, `ToolOutput` result type, Windows `CREATE_NO_WINDOW` flag, and MSYS environment variable stripping +- **`subprocess.rs`** -- `async fn run_command()` / `run_command_with_stdin()` / `run_command_passthrough()` with optional timeout, plus `_blocking` sync escape hatches; `ToolOutput` result type, Windows `CREATE_NO_WINDOW` flag, MSYS environment variable stripping, and tokio-driven Job-Object / `PR_SET_PDEATHSIG` containment (#813) diff --git a/crates/fbuild-core/src/containment.rs b/crates/fbuild-core/src/containment.rs index 4a89a311..4c8432b9 100644 --- a/crates/fbuild-core/src/containment.rs +++ b/crates/fbuild-core/src/containment.rs @@ -35,8 +35,10 @@ //! through: //! //! * [`fbuild_core::subprocess::run_command`](super::subprocess::run_command) -//! — the central blocking helper used by compilers, linkers, esptool, -//! avrdude, addr2line, and most emulator setup code. +//! — the central async helper used by compilers, linkers, esptool, +//! avrdude, addr2line, and most emulator setup code. (#813 Phase A +//! converted this from sync to async — it spawns via +//! [`tokio_spawn::spawn_contained`] under the hood.) //! * [`spawn_contained`] — direct `std::process::Command` spawns that //! don't go through `run_command` (e.g. zccache daemon startup). //! * [`tokio_spawn::spawn_contained`] — long-running async spawns in diff --git a/crates/fbuild-core/src/response_file.rs b/crates/fbuild-core/src/response_file.rs index 5f4f9ba3..c4b1ad12 100644 --- a/crates/fbuild-core/src/response_file.rs +++ b/crates/fbuild-core/src/response_file.rs @@ -3,6 +3,12 @@ //! Handles writing `@file` response files for GCC/G++ on Windows where //! command-line length limits (32KB CreateProcess) and MSYS2 path translation //! issues require special handling. +//! +//! Async surface (#813): `write_response_file` is now an `async fn` — +//! it's called once per compile/link, often interleaved with other I/O, +//! so going through `tokio::fs` keeps the runtime free to schedule +//! other work. `write_response_file_blocking` exists as the escape +//! hatch for the (rare) sync call sites. use crate::Result; use std::path::{Path, PathBuf}; @@ -117,32 +123,21 @@ pub fn replace_path_backslashes(s: &str) -> String { result } -/// Write flags to a temporary GCC response file (`@file` syntax). -/// -/// Returns the path to the response file. -/// -/// Flags containing either `\"` or bare `"` in define values are wrapped in -/// single quotes with `\"` converted to plain `"`. GCC's response file -/// parser preserves literal `"` inside single-quoted arguments. -pub fn write_response_file(flags: &[String], temp_dir: &Path, prefix: &str) -> Result { - std::fs::create_dir_all(temp_dir).map_err(|e| { - crate::FbuildError::BuildFailed(format!( - "failed to create temp dir {}: {}", - temp_dir.display(), - e - )) - })?; - let _ = cleanup_stale_response_files(temp_dir, RESPONSE_FILE_STALE_AFTER, SystemTime::now()); - - // GCC treats backslashes in response files as escape characters (\n = newline, - // \f = formfeed, etc.). Convert to forward slashes for Windows path compatibility, - // but preserve \" sequences which are intentional escape sequences (e.g., in +/// Build the deterministic on-disk path + content for a response file +/// without performing any I/O. Shared by both the async and blocking +/// public surfaces. +fn render_response_file(flags: &[String], temp_dir: &Path, prefix: &str) -> (PathBuf, String) { + // GCC treats backslashes in response files as escape characters + // (\n = newline, \f = formfeed, etc.). Convert to forward slashes + // for Windows path compatibility, but preserve \" sequences which + // are intentional escape sequences (e.g., // -DMBEDTLS_CONFIG_FILE=\"mbedtls/esp_config.h\"). // - // Flags containing quoted define values need single-quote wrapping. Some - // define sources use escaped quotes (-DFOO=\"bar\"), while data-driven MCU - // configs can contain bare quotes (-DFOO="bar"). Normalize both forms to - // the response-file spelling GCC preserves as a string literal. + // Flags containing quoted define values need single-quote wrapping. + // Some define sources use escaped quotes (-DFOO=\"bar\"), while + // data-driven MCU configs can contain bare quotes (-DFOO="bar"). + // Normalize both forms to the response-file spelling GCC preserves + // as a string literal. let content = flags .iter() .map(|f| { @@ -173,6 +168,64 @@ pub fn write_response_file(flags: &[String], temp_dir: &Path, prefix: &str) -> R ) }; let path = temp_dir.join(format!("fbuild_{}_{}.rsp", prefix, hash)); + (path, content) +} + +/// Write flags to a temporary GCC response file (`@file` syntax) using +/// async filesystem I/O. +/// +/// Returns the path to the response file. +/// +/// Flags containing either `\"` or bare `"` in define values are wrapped in +/// single quotes with `\"` converted to plain `"`. GCC's response file +/// parser preserves literal `"` inside single-quoted arguments. +pub async fn write_response_file( + flags: &[String], + temp_dir: &Path, + prefix: &str, +) -> Result { + tokio::fs::create_dir_all(temp_dir).await.map_err(|e| { + crate::FbuildError::BuildFailed(format!( + "failed to create temp dir {}: {}", + temp_dir.display(), + e + )) + })?; + let _ = cleanup_stale_response_files(temp_dir, RESPONSE_FILE_STALE_AFTER, SystemTime::now()); + + let (path, content) = render_response_file(flags, temp_dir, prefix); + + if tokio::fs::try_exists(&path).await.unwrap_or(false) { + return Ok(path); + } + + tokio::fs::write(&path, content).await.map_err(|e| { + crate::FbuildError::BuildFailed(format!( + "failed to write response file {}: {}", + path.display(), + e + )) + })?; + Ok(path) +} + +/// Sync escape hatch for [`write_response_file`]. Useful in tests and +/// sync diagnostic subcommands. +pub fn write_response_file_blocking( + flags: &[String], + temp_dir: &Path, + prefix: &str, +) -> Result { + std::fs::create_dir_all(temp_dir).map_err(|e| { + crate::FbuildError::BuildFailed(format!( + "failed to create temp dir {}: {}", + temp_dir.display(), + e + )) + })?; + let _ = cleanup_stale_response_files(temp_dir, RESPONSE_FILE_STALE_AFTER, SystemTime::now()); + + let (path, content) = render_response_file(flags, temp_dir, prefix); if path.exists() { return Ok(path); @@ -216,52 +269,76 @@ mod tests { ); } - #[test] - fn test_write_response_file_reuses_same_path_for_same_content() { + #[tokio::test] + async fn test_write_response_file_reuses_same_path_for_same_content() { let tmp = tempfile::TempDir::new().unwrap(); let flags = vec!["-O2".to_string(), "-c".to_string(), "src.cpp".to_string()]; - let first = write_response_file(&flags, tmp.path(), "stable").unwrap(); - let second = write_response_file(&flags, tmp.path(), "stable").unwrap(); + let first = write_response_file(&flags, tmp.path(), "stable") + .await + .unwrap(); + let second = write_response_file(&flags, tmp.path(), "stable") + .await + .unwrap(); assert_eq!(first, second); } - #[test] - fn test_write_response_file_changes_path_when_content_changes() { + #[tokio::test] + async fn test_write_response_file_changes_path_when_content_changes() { let tmp = tempfile::TempDir::new().unwrap(); - let first = write_response_file(&["-O2".to_string()], tmp.path(), "stable").unwrap(); - let second = write_response_file(&["-O3".to_string()], tmp.path(), "stable").unwrap(); + let first = write_response_file(&["-O2".to_string()], tmp.path(), "stable") + .await + .unwrap(); + let second = write_response_file(&["-O3".to_string()], tmp.path(), "stable") + .await + .unwrap(); assert_ne!(first, second); } - #[test] - fn test_write_response_file_wraps_escaped_quote_defines() { + #[tokio::test] + async fn test_write_response_file_wraps_escaped_quote_defines() { let tmp = tempfile::TempDir::new().unwrap(); let rsp = write_response_file( &[r#"-DARDUINO_BOARD=\"ESP32_DEV\""#.to_string()], tmp.path(), "define", ) + .await .unwrap(); - let content = std::fs::read_to_string(rsp).unwrap(); + let content = tokio::fs::read_to_string(rsp).await.unwrap(); assert_eq!(content, r#"'-DARDUINO_BOARD="ESP32_DEV"'"#); } - #[test] - fn test_write_response_file_wraps_bare_quote_defines() { + #[tokio::test] + async fn test_write_response_file_wraps_bare_quote_defines() { let tmp = tempfile::TempDir::new().unwrap(); let rsp = write_response_file( &[r#"-DARDUINO_BSP_VERSION="1.6.1""#.to_string()], tmp.path(), "define", ) + .await .unwrap(); - let content = std::fs::read_to_string(rsp).unwrap(); + let content = tokio::fs::read_to_string(rsp).await.unwrap(); assert_eq!(content, r#"'-DARDUINO_BSP_VERSION="1.6.1"'"#); } + #[test] + fn test_write_response_file_blocking_matches_async_path() { + // The sync escape hatch must produce byte-identical output to + // the async path so callers can mix them without surprises. + let tmp = tempfile::TempDir::new().unwrap(); + let flags = vec!["-O2".to_string(), "-DFOO=bar".to_string()]; + let blocking = write_response_file_blocking(&flags, tmp.path(), "stable").unwrap(); + let content = std::fs::read_to_string(&blocking).unwrap(); + let (expected_path, expected_content) = + render_response_file(&flags, tmp.path(), "stable"); + assert_eq!(blocking, expected_path); + assert_eq!(content, expected_content); + } + #[test] fn test_response_files_root_uses_fbuild_owned_tmp_dir() { let home = Path::new("/home/user"); diff --git a/crates/fbuild-core/src/subprocess.rs b/crates/fbuild-core/src/subprocess.rs index 57c7b0b1..dd0c1fb4 100644 --- a/crates/fbuild-core/src/subprocess.rs +++ b/crates/fbuild-core/src/subprocess.rs @@ -1,10 +1,17 @@ -//! Subprocess runner backed by `running-process`. +//! Subprocess runner — async-first (#813). //! -//! Every synchronous spawn in fbuild flows through this module. We use -//! [`running_process::NativeProcess`] so that stdout and stderr are -//! drained concurrently from the moment the child starts — the manual -//! drain loop that preceded this module deadlocked the moment a compiler -//! filled its stderr pipe (see FastLED/fbuild#141). +//! Every subprocess spawn in fbuild flows through this module. The +//! primary surface is now `async fn`-shaped (`run_command`, +//! `run_command_with_stdin`, `run_command_passthrough`); a small set of +//! `_blocking` shims exist as the escape hatch for the handful of sync +//! call sites (CLI diagnostic subcommands, tests, etc.). +//! +//! Internally we spawn via [`tokio::process::Command`] routed through +//! [`crate::containment::tokio_spawn::spawn_contained`] so the daemon's +//! Job Object / `PR_SET_PDEATHSIG` containment still kills every child +//! when the daemon goes down. When no global containment group has been +//! installed (CLI binary, unit tests) the contained helper falls back +//! to a plain spawn — same coverage as the pre-async implementation. //! //! On Windows we still: //! * prepend the executable's directory to PATH so GCC's `cc1plus` can @@ -12,23 +19,24 @@ //! * strip MSYS/MSYS2 env vars that would otherwise poison native //! Windows toolchain binaries. //! -//! On Windows, containment is applied post-spawn by -//! `crate::containment::windows_job::assign` when the daemon has -//! installed the global containment group; CLI binaries and unit tests -//! run uncontained just as before. (Earlier code used a `containment:` -//! field on `ProcessConfig` from a pre-release `running-process-core`; -//! the published `running-process` 4.0 API does not expose that field — -//! see #32.) +//! The legacy "what running-process gives us" output shape is preserved +//! byte-for-byte: stdout/stderr are returned as `String`s composed of +//! lossy-UTF-8 lines joined by `\n` with a trailing newline when +//! non-empty (see [`join_lines`]). use std::path::Path; +use std::process::Stdio; use std::time::Duration; -use running_process::{ - CommandSpec, NativeProcess, ProcessConfig, ProcessError, StderrMode, StdinMode, -}; +use tokio::io::AsyncWriteExt; +use tokio::process::{Child, Command as TokioCommand}; +use crate::containment::tokio_spawn; use crate::{FbuildError, Result}; +#[cfg(windows)] +const CREATE_NO_WINDOW: u32 = 0x08000000; + /// Build the env overlay that GCC link steps should pass to /// [`run_command`] so that `lto-wrapper`'s temp files (the `*.ltrans*.o` /// pieces it shuffles between partitions) land inside a fbuild-owned, @@ -78,69 +86,88 @@ impl ToolOutput { } /// Run an external command and capture its output. -pub fn run_command( +/// +/// Async-first: callers in an async context should `.await` this. +/// Use [`run_command_blocking`] from sync contexts (CLI diagnostic +/// subcommands, tests). +pub async fn run_command( args: &[&str], cwd: Option<&Path>, env: Option<&[(&str, &str)]>, timeout: Option, ) -> Result { - let config = build_config(args, cwd, env, /*capture=*/ true, StdinMode::Null)?; - run_captured(config, args, timeout) + if args.is_empty() { + return Err(FbuildError::Other("empty command".to_string())); + } + let mut cmd = build_command(args, cwd, env, /*capture=*/ true, /*stdin_piped=*/ false)?; + let child = tokio_spawn::spawn_contained(&mut cmd).map_err(|e| spawn_err(args, e))?; + wait_and_capture(child, args, timeout).await } /// Run an external command, feed `stdin_bytes` to its stdin, and /// capture stdout+stderr. Used by tools that operate on a payload /// piped through a filter (e.g. `c++filt`, `clang-format`). /// -/// Routing through `NativeProcess` is critical for large stdin -/// payloads: the running-process reader thread drains stdout in -/// background while we write stdin, avoiding the Windows pipe-buffer -/// deadlock that hits when ~3k mangled symbols (~250 KB) saturate -/// the 4-8 KB stdout pipe before stdin EOF. -pub fn run_command_with_stdin( +/// Concurrency-safe: stdin write, stdout drain, and stderr drain all +/// run on the tokio runtime concurrently — no risk of the Windows +/// pipe-buffer deadlock that hits when a multi-hundred-KB symbol +/// payload saturates the stdout pipe before stdin EOF. +pub async fn run_command_with_stdin( args: &[&str], stdin_bytes: &[u8], cwd: Option<&Path>, env: Option<&[(&str, &str)]>, timeout: Option, ) -> Result { - let config = build_config(args, cwd, env, /*capture=*/ true, StdinMode::Piped)?; - let process = NativeProcess::new(config); - process.start().map_err(|e| spawn_err(args, e))?; - // Writes the data then closes stdin (signals EOF). The reader - // threads spawned by `start()` keep stdout/stderr drained - // throughout the write. - if !stdin_bytes.is_empty() { - process - .write_stdin(stdin_bytes) - .map_err(|e| FbuildError::Other(format!("stdin write to {:?} failed: {}", args, e)))?; - } else { - // Empty payload: close stdin immediately so the child sees EOF. - let _ = process.close_stdin(); + if args.is_empty() { + return Err(FbuildError::Other("empty command".to_string())); } - let exit_code = match process.wait(timeout) { - Ok(code) => code, - Err(ProcessError::Timeout) => { - let _ = process.kill(); - return Err(FbuildError::Timeout(format!( - "command timed out after {}s", - timeout.map(|d| d.as_secs()).unwrap_or(0) - ))); - } - Err(e) => { - return Err(FbuildError::Other(format!( - "command {:?} failed: {}", - args, e - ))) + let mut cmd = build_command(args, cwd, env, /*capture=*/ true, /*stdin_piped=*/ true)?; + let mut child = tokio_spawn::spawn_contained(&mut cmd).map_err(|e| spawn_err(args, e))?; + + // Take the stdin handle and concurrently write the payload while + // tokio drains stdout/stderr in the background. Dropping `stdin` + // closes the pipe (signals EOF) before we wait for the exit. + if let Some(mut stdin) = child.stdin.take() { + let bytes = stdin_bytes.to_vec(); + let args_owned: Vec = args.iter().map(|s| (*s).to_string()).collect(); + // Spawn the writer as a sibling task so the read side of the + // pipe can drain concurrently when we `wait_with_output` below. + let write_task = tokio::spawn(async move { + if !bytes.is_empty() { + stdin.write_all(&bytes).await?; + } + stdin.shutdown().await?; + drop(stdin); + Ok::<_, std::io::Error>(args_owned) + }); + + let output = wait_and_capture(child, args, timeout).await; + + // Surface a stdin-write error only if the command itself + // succeeded — otherwise the command's own error is more useful. + match write_task.await { + Ok(Ok(_)) => output, + Ok(Err(e)) => match output { + Ok(_) => Err(FbuildError::Other(format!( + "stdin write to {:?} failed: {}", + args, e + ))), + Err(orig) => Err(orig), + }, + Err(join_err) => match output { + Ok(_) => Err(FbuildError::Other(format!( + "stdin writer task for {:?} panicked: {}", + args, join_err + ))), + Err(orig) => Err(orig), + }, } - }; - let stdout = join_lines(process.captured_stdout()); - let stderr = join_lines(process.captured_stderr()); - Ok(ToolOutput { - stdout, - stderr, - exit_code, - }) + } else { + // No stdin handle (extremely unlikely with `stdin(Stdio::piped())`) + // — just wait for completion. + wait_and_capture(child, args, timeout).await + } } /// Run an external command with inherited stdin/stdout/stderr (no @@ -148,130 +175,242 @@ pub fn run_command_with_stdin( /// delegation where users expect the tool's live output. /// /// Returns the exit code. -pub fn run_command_passthrough( +pub async fn run_command_passthrough( args: &[&str], cwd: Option<&Path>, env: Option<&[(&str, &str)]>, timeout: Option, ) -> Result { - let config = build_config(args, cwd, env, /*capture=*/ false, StdinMode::Inherit)?; - let process = NativeProcess::new(config); - process.start().map_err(|e| spawn_err(args, e))?; - match process.wait(timeout) { - Ok(code) => Ok(code), - Err(ProcessError::Timeout) => { - let _ = process.kill(); - Err(FbuildError::Timeout(format!( + if args.is_empty() { + return Err(FbuildError::Other("empty command".to_string())); + } + let mut cmd = build_command(args, cwd, env, /*capture=*/ false, /*stdin_piped=*/ false)?; + let mut child = tokio_spawn::spawn_contained(&mut cmd).map_err(|e| spawn_err(args, e))?; + let status = match wait_with_timeout(&mut child, timeout).await? { + Some(status) => status, + None => { + let _ = child.kill().await; + return Err(FbuildError::Timeout(format!( "command timed out after {}s", timeout.map(|d| d.as_secs()).unwrap_or(0) - ))) + ))); } - Err(e) => Err(FbuildError::Other(format!( - "command {:?} failed: {}", - args, e - ))), - } + }; + Ok(exit_code_from(status)) } -fn run_captured( - config: ProcessConfig, +// --------------------------------------------------------------------------- +// Sync bridges for one-shot callers (diagnostic CLI subcommands, tests). +// --------------------------------------------------------------------------- + +/// Blocking variant of [`run_command`] for sync call sites. +/// +/// Constructs a fresh single-threaded tokio runtime to drive the async +/// path. Only use from contexts that are *not* already inside a tokio +/// reactor — calling this from inside an async function will panic. +pub fn run_command_blocking( args: &[&str], + cwd: Option<&Path>, + env: Option<&[(&str, &str)]>, timeout: Option, ) -> Result { - let process = NativeProcess::new(config); - process.start().map_err(|e| spawn_err(args, e))?; - let exit_code = match process.wait(timeout) { - Ok(code) => code, - Err(ProcessError::Timeout) => { - let _ = process.kill(); - return Err(FbuildError::Timeout(format!( - "command timed out after {}s", - timeout.map(|d| d.as_secs()).unwrap_or(0) - ))); - } - Err(e) => { - return Err(FbuildError::Other(format!( - "command {:?} failed: {}", - args, e - ))) - } - }; + block_on(run_command(args, cwd, env, timeout)) +} + +/// Blocking variant of [`run_command_with_stdin`]. +pub fn run_command_with_stdin_blocking( + args: &[&str], + stdin_bytes: &[u8], + cwd: Option<&Path>, + env: Option<&[(&str, &str)]>, + timeout: Option, +) -> Result { + block_on(run_command_with_stdin(args, stdin_bytes, cwd, env, timeout)) +} + +/// Blocking variant of [`run_command_passthrough`]. +pub fn run_command_passthrough_blocking( + args: &[&str], + cwd: Option<&Path>, + env: Option<&[(&str, &str)]>, + timeout: Option, +) -> Result { + block_on(run_command_passthrough(args, cwd, env, timeout)) +} + +fn block_on(fut: F) -> F::Output { + // Use a fresh current-thread runtime. We deliberately do not try to + // pick up an ambient `Handle` here — callers of `_blocking` are sync + // contexts where re-entering a tokio runtime would panic anyway, and + // a fresh single-threaded runtime is the cheap, safe escape hatch. + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .expect("failed to build single-threaded tokio runtime for blocking subprocess call"); + rt.block_on(fut) +} - let stdout = join_lines(process.captured_stdout()); - let stderr = join_lines(process.captured_stderr()); +// --------------------------------------------------------------------------- +// Shared async plumbing +// --------------------------------------------------------------------------- +async fn wait_and_capture( + child: Child, + args: &[&str], + timeout: Option, +) -> Result { + let wait_fut = child.wait_with_output(); + let output = match timeout { + Some(d) => match tokio::time::timeout(d, wait_fut).await { + Ok(res) => res.map_err(|e| FbuildError::Other(format!("command {:?} failed: {}", args, e)))?, + Err(_) => { + return Err(FbuildError::Timeout(format!( + "command timed out after {}s", + d.as_secs() + ))) + } + }, + None => wait_fut + .await + .map_err(|e| FbuildError::Other(format!("command {:?} failed: {}", args, e)))?, + }; + let exit_code = exit_code_from(output.status); Ok(ToolOutput { - stdout, - stderr, + stdout: bytes_to_lines_string(&output.stdout), + stderr: bytes_to_lines_string(&output.stderr), exit_code, }) } -fn build_config( +async fn wait_with_timeout( + child: &mut Child, + timeout: Option, +) -> Result> { + match timeout { + Some(d) => match tokio::time::timeout(d, child.wait()).await { + Ok(res) => res + .map(Some) + .map_err(|e| FbuildError::Other(format!("wait failed: {}", e))), + Err(_) => Ok(None), + }, + None => child + .wait() + .await + .map(Some) + .map_err(|e| FbuildError::Other(format!("wait failed: {}", e))), + } +} + +fn exit_code_from(status: std::process::ExitStatus) -> i32 { + status.code().unwrap_or_else(|| { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt; + // Surface signal as -signo to match running-process semantics. + status.signal().map(|s| -s).unwrap_or(-1) + } + #[cfg(not(unix))] + { + let _ = status; + -1 + } + }) +} + +fn build_command( args: &[&str], cwd: Option<&Path>, env: Option<&[(&str, &str)]>, capture: bool, - stdin_mode: StdinMode, -) -> Result { - if args.is_empty() { - return Err(FbuildError::Other("empty command".to_string())); + stdin_piped: bool, +) -> Result { + let mut cmd = TokioCommand::new(args[0]); + if args.len() > 1 { + cmd.args(&args[1..]); + } + if let Some(dir) = cwd { + cmd.current_dir(dir); } - let argv: Vec = args.iter().map(|s| (*s).to_string()).collect(); + // Build the env overlay (Windows PATH rewriting / MSYS stripping, + // and any explicit overlay vars). When `compute_env` returns `None` + // the child inherits the parent env verbatim. + if let Some(env_vec) = compute_env(args[0], env) { + cmd.env_clear(); + for (k, v) in env_vec { + cmd.env(k, v); + } + } - // Build the environment the child will see. Windows needs PATH - // rewriting (prepend exe dir) and optional MSYS-var stripping; Unix - // only needs overlay vars applied. When no changes are required - // leave `env = None` so the child inherits the parent environment - // verbatim (matching the pre-migration behaviour). - let env_vec = compute_env(args[0], env); + // Stdio wiring. + if capture { + cmd.stdout(Stdio::piped()); + cmd.stderr(Stdio::piped()); + } else { + cmd.stdout(Stdio::inherit()); + cmd.stderr(Stdio::inherit()); + } + if stdin_piped { + cmd.stdin(Stdio::piped()); + } else if capture { + // Same shape as the pre-async StdinMode::Null: detach stdin so + // captured children don't accidentally read from the parent + // terminal. + cmd.stdin(Stdio::null()); + } else { + cmd.stdin(Stdio::inherit()); + } + // Hide the console window for child processes on Windows. Matches + // the pre-async `CREATE_NO_WINDOW` flag. #[cfg(windows)] - let creationflags = { - const CREATE_NO_WINDOW: u32 = 0x08000000; - Some(CREATE_NO_WINDOW) - }; - #[cfg(not(windows))] - let creationflags: Option = None; - - Ok(ProcessConfig { - command: CommandSpec::Argv(argv), - cwd: cwd.map(|p| p.to_path_buf()), - env: env_vec, - capture, - stderr_mode: StderrMode::Pipe, - creationflags, - create_process_group: false, - stdin_mode, - nice: None, - }) + { + use std::os::windows::process::CommandExt; + cmd.as_std_mut().creation_flags(CREATE_NO_WINDOW); + } + + Ok(cmd) +} + +fn spawn_err(args: &[&str], e: std::io::Error) -> FbuildError { + FbuildError::Other(format!("failed to spawn {:?}: {}", args, e)) } -fn join_lines(lines: Vec>) -> String { - // NativeProcess returns one Vec per line (CR/LF stripped). Join - // with '\n' and add a trailing newline when non-empty so the result - // matches the shape of the previous `String::from_utf8_lossy(&raw)` - // output closely enough for downstream parsers (which mostly call - // `.trim()` or `.lines()`). - if lines.is_empty() { +/// Format captured bytes the same way the old running-process path did: +/// split into lossy-UTF-8 lines (stripping CR/LF), join with '\n', and +/// add a trailing newline when non-empty. Downstream parsers rely on +/// this exact shape (most call `.trim()` / `.lines()` but a few search +/// for substrings — see #141). +fn bytes_to_lines_string(raw: &[u8]) -> String { + if raw.is_empty() { return String::new(); } - let mut out = String::new(); - for (idx, line) in lines.iter().enumerate() { - if idx > 0 { + let lossy = String::from_utf8_lossy(raw); + let mut out = String::with_capacity(lossy.len()); + let mut first = true; + for line in lossy.split('\n') { + // The split by '\n' already drops trailing '\n'; strip a + // trailing '\r' so CRLF input is collapsed to '\n'. + let line = line.strip_suffix('\r').unwrap_or(line); + if first { + first = false; + } else { out.push('\n'); } - out.push_str(&String::from_utf8_lossy(line)); + out.push_str(line); + } + // The old `join_lines` always pushed a trailing newline when the + // input was non-empty. Preserve that. + if !out.is_empty() && !out.ends_with('\n') { + out.push('\n'); + } else if out.is_empty() { + // Edge case: all input was a single empty line — match the old + // behaviour which would return empty. + return String::new(); } - out.push('\n'); out } -fn spawn_err(args: &[&str], e: ProcessError) -> FbuildError { - FbuildError::Other(format!("failed to spawn {:?}: {}", args, e)) -} - /// Build the env vector to pass to the child. /// /// * On Unix: when `overlay` is Some, merge it into the current @@ -404,30 +543,42 @@ fn strip_msys_env(env_map: &mut std::collections::BTreeMap) { mod tests { use super::*; - #[test] - fn run_echo() { + #[tokio::test] + async fn run_echo() { let args = if cfg!(windows) { vec!["cmd", "/C", "echo hello"] } else { vec!["echo", "hello"] }; - let result = run_command(&args, None, None, None).unwrap(); + let result = run_command(&args, None, None, None).await.unwrap(); assert!(result.success()); assert!(result.stdout.trim().contains("hello")); } - #[test] - fn run_nonexistent_command() { - let result = run_command(&["nonexistent_command_xyz"], None, None, None); + #[tokio::test] + async fn run_nonexistent_command() { + let result = run_command(&["nonexistent_command_xyz"], None, None, None).await; assert!(result.is_err()); } - #[test] - fn run_empty_args() { - let result = run_command(&[], None, None, None); + #[tokio::test] + async fn run_empty_args() { + let result = run_command(&[], None, None, None).await; assert!(result.is_err()); } + #[test] + fn run_command_blocking_works_from_sync_context() { + let args = if cfg!(windows) { + vec!["cmd", "/C", "echo blocking"] + } else { + vec!["echo", "blocking"] + }; + let result = run_command_blocking(&args, None, None, None).unwrap(); + assert!(result.success()); + assert!(result.stdout.trim().contains("blocking")); + } + #[test] fn link_env_for_build_creates_dir_and_returns_posix_paths() { let tmp = tempfile::tempdir().expect("tempdir"); @@ -489,16 +640,36 @@ mod tests { assert!(!output.success()); } - #[test] - fn run_captures_stderr() { + #[tokio::test] + async fn run_captures_stderr() { // Verify that stderr is captured independently from stdout. let args = if cfg!(windows) { vec!["cmd", "/C", "echo err 1>&2"] } else { vec!["sh", "-c", "echo err 1>&2"] }; - let result = run_command(&args, None, None, None).unwrap(); + let result = run_command(&args, None, None, None).await.unwrap(); assert!(result.success()); assert!(result.stderr.contains("err"), "got: {:?}", result); } + + #[tokio::test] + async fn run_command_with_stdin_pipes_payload() { + // Round-trip: feed stdin → expect it back on stdout. `cat` on + // unix, `findstr` on windows (matches everything via /R ".*"). + let args = if cfg!(windows) { + vec!["findstr", "/R", ".*"] + } else { + vec!["cat"] + }; + let result = run_command_with_stdin(&args, b"hello world\n", None, None, None) + .await + .unwrap(); + assert!(result.success(), "got: {:?}", result); + assert!( + result.stdout.contains("hello world"), + "stdout was {:?}", + result.stdout + ); + } } diff --git a/crates/fbuild-daemon/src/handlers/emulator/avr8js_deploy.rs b/crates/fbuild-daemon/src/handlers/emulator/avr8js_deploy.rs index ca02e754..62052620 100644 --- a/crates/fbuild-daemon/src/handlers/emulator/avr8js_deploy.rs +++ b/crates/fbuild-daemon/src/handlers/emulator/avr8js_deploy.rs @@ -193,7 +193,7 @@ pub async fn deploy_avr8js( if monitor_after { // Headless path: run avr8js in Node.js subprocess, capture UART on stdout - let node_path = match find_node() { + let node_path = match find_node().await { Ok(p) => p, Err(e) => { return ( @@ -202,7 +202,7 @@ pub async fn deploy_avr8js( ); } }; - let avr8js_cache = match ensure_avr8js_npm() { + let avr8js_cache = match ensure_avr8js_npm().await { Ok(p) => p, Err(e) => { return ( diff --git a/crates/fbuild-daemon/src/handlers/emulator/avr8js_npm.rs b/crates/fbuild-daemon/src/handlers/emulator/avr8js_npm.rs index 0cf2a0b4..bc8ae7f9 100644 --- a/crates/fbuild-daemon/src/handlers/emulator/avr8js_npm.rs +++ b/crates/fbuild-daemon/src/handlers/emulator/avr8js_npm.rs @@ -5,13 +5,13 @@ use std::path::{Path, PathBuf}; -pub(crate) fn find_node() -> fbuild_core::Result { +pub(crate) async fn find_node() -> fbuild_core::Result { let node = if cfg!(windows) { "node.exe" } else { "node" }; // Route through fbuild-core's `run_command` so the probe spawn is // captured by the daemon's containment group (issue #32). The probe // is short-lived (`node --version`) but a missing binary should // still bubble up the same way. - match fbuild_core::subprocess::run_command(&[node, "--version"], None, None, None) { + match fbuild_core::subprocess::run_command(&[node, "--version"], None, None, None).await { Ok(output) if output.success() => Ok(PathBuf::from(node)), _ => Err(fbuild_core::FbuildError::DeployFailed( "Node.js is required for headless avr8js emulation but 'node' was not found on PATH. \ @@ -109,16 +109,16 @@ pub(crate) fn prepare_avr8js_cache_for_install( Avr8jsCachePrep::NothingToClean } -pub(crate) fn ensure_avr8js_npm() -> fbuild_core::Result { +pub(crate) async fn ensure_avr8js_npm() -> fbuild_core::Result { let cache_dir = fbuild_paths::get_cache_root().join("avr8js-node"); - ensure_avr8js_npm_in(&cache_dir, refresh_emu_cache_requested())?; + ensure_avr8js_npm_in(&cache_dir, refresh_emu_cache_requested()).await?; Ok(cache_dir) } /// Populate `cache_dir` with a fresh `node_modules/avr8js` install, wiping /// a corrupt or partial install as needed. Split out from `ensure_avr8js_npm` /// so unit tests can inject a temporary cache dir without touching env vars. -pub(crate) fn ensure_avr8js_npm_in( +pub(crate) async fn ensure_avr8js_npm_in( cache_dir: &Path, force_refresh: bool, ) -> fbuild_core::Result<()> { @@ -153,6 +153,7 @@ pub(crate) fn ensure_avr8js_npm_in( None, None, ) + .await .map_err(|e| { fbuild_core::FbuildError::DeployFailed(format!( "failed to launch 'npm' (for `npm install avr8js@0.21.0 --prefix {}`): {}. \ diff --git a/crates/fbuild-daemon/src/handlers/emulator/qemu_deploy.rs b/crates/fbuild-daemon/src/handlers/emulator/qemu_deploy.rs index 61019a8a..335407b6 100644 --- a/crates/fbuild-daemon/src/handlers/emulator/qemu_deploy.rs +++ b/crates/fbuild-daemon/src/handlers/emulator/qemu_deploy.rs @@ -51,7 +51,7 @@ pub(crate) fn is_qemu_supported_esp32_mcu(mcu: &str) -> bool { /// Picks `qemu-system-xtensa` for ESP32/ESP32-S3 and `qemu-system-riscv32` /// for ESP32-C3/C6/H2. Returns the resolved binary path (downloading into /// the managed fbuild cache if required). -pub(crate) fn resolve_esp_qemu_for_mcu( +pub(crate) async fn resolve_esp_qemu_for_mcu( project_dir: &Path, mcu: &str, ) -> fbuild_core::Result { @@ -62,7 +62,7 @@ pub(crate) fn resolve_esp_qemu_for_mcu( )) })?; let pkg = fbuild_packages::toolchain::EspQemu::new(project_dir, arch)?; - pkg.resolve_executable() + pkg.resolve_executable().await } /// Fail fast if the board's flash mode is incompatible with QEMU (DIO only). @@ -201,7 +201,7 @@ pub async fn deploy_qemu( } }; - let qemu = match resolve_esp_qemu_for_mcu(&project_dir, &board.mcu) { + let qemu = match resolve_esp_qemu_for_mcu(&project_dir, &board.mcu).await { Ok(path) => path, Err(e) => { return ( @@ -254,11 +254,14 @@ pub async fn deploy_qemu( &flash_image, board.qemu_esp32_psram_config(), ); - let addr2line_path = elf_path.as_ref().and_then(|_| { - resolve_esp32_toolchain_gcc_path(&project_dir, &mcu_config) - .ok() - .and_then(|gcc| fbuild_serial::crash_decoder::derive_addr2line_path(&gcc)) - }); + let addr2line_path = if elf_path.is_some() { + match resolve_esp32_toolchain_gcc_path(&project_dir, &mcu_config).await { + Ok(gcc) => fbuild_serial::crash_decoder::derive_addr2line_path(&gcc), + Err(_) => None, + } + } else { + None + }; let timeout_secs = monitor_timeout.or(Some(qemu_timeout_secs as f64)); let qemu_result = match run_qemu_process( diff --git a/crates/fbuild-daemon/src/handlers/emulator/runners.rs b/crates/fbuild-daemon/src/handlers/emulator/runners.rs index 69140451..b671f6e4 100644 --- a/crates/fbuild-daemon/src/handlers/emulator/runners.rs +++ b/crates/fbuild-daemon/src/handlers/emulator/runners.rs @@ -80,7 +80,7 @@ impl EmulatorRunner for QemuRunner { mcu_config.default_flash_size(), )?; - let qemu = resolve_esp_qemu_for_mcu(&self.project_dir, &self.board.mcu)?; + let qemu = resolve_esp_qemu_for_mcu(&self.project_dir, &self.board.mcu).await?; let session_dir = qemu_session_dir(&self.project_dir, &self.env_name); std::fs::create_dir_all(&session_dir)?; @@ -108,11 +108,14 @@ impl EmulatorRunner for QemuRunner { &flash_image, self.board.qemu_esp32_psram_config(), ); - let addr2line_path = config.elf_path.as_ref().and_then(|_| { - resolve_esp32_toolchain_gcc_path(&self.project_dir, &mcu_config) - .ok() - .and_then(|gcc| fbuild_serial::crash_decoder::derive_addr2line_path(&gcc)) - }); + let addr2line_path = if config.elf_path.is_some() { + match resolve_esp32_toolchain_gcc_path(&self.project_dir, &mcu_config).await { + Ok(gcc) => fbuild_serial::crash_decoder::derive_addr2line_path(&gcc), + Err(_) => None, + } + } else { + None + }; let command_line = format!("{} {}", qemu.display(), args.join(" ")); @@ -170,8 +173,8 @@ impl EmulatorRunner for Avr8jsRunner { { return Err(fbuild_core::FbuildError::DeployFailed(msg)); } - let node_path = find_node()?; - let avr8js_cache = ensure_avr8js_npm()?; + let node_path = find_node().await?; + let avr8js_cache = ensure_avr8js_npm().await?; // headless.mjs uses `import ... from "avr8js"` (a bare ESM // specifier). Node's ESM resolver does NOT honor NODE_PATH — @@ -235,7 +238,7 @@ impl EmulatorRunner for Avr8jsRunner { /// - Linux: `apt install simavr` or build from source /// - macOS: `brew install simavr` /// - Windows: build from source (MSYS2/MinGW) — limited support -fn find_simavr() -> fbuild_core::Result { +async fn find_simavr() -> fbuild_core::Result { let simavr = if cfg!(windows) { "simavr.exe" } else { @@ -244,7 +247,7 @@ fn find_simavr() -> fbuild_core::Result { // Try running simavr to verify it exists; route through containment // (issue #32). This is a short-lived probe so the containment // difference is purely consistency. - match fbuild_core::subprocess::run_command(&[simavr, "--help"], None, None, None) { + match fbuild_core::subprocess::run_command(&[simavr, "--help"], None, None, None).await { Ok(_) => Ok(PathBuf::from(simavr)), Err(_) => { let install_hint = if cfg!(target_os = "linux") { @@ -291,7 +294,7 @@ impl EmulatorRunner for SimavrRunner { { return Err(fbuild_core::FbuildError::DeployFailed(msg)); } - let simavr_path = find_simavr()?; + let simavr_path = find_simavr().await?; // simavr requires an ELF file let elf_path = config.elf_path.as_ref().ok_or_else(|| { diff --git a/crates/fbuild-daemon/src/handlers/emulator/select.rs b/crates/fbuild-daemon/src/handlers/emulator/select.rs index fd87a81b..ff2cbe57 100644 --- a/crates/fbuild-daemon/src/handlers/emulator/select.rs +++ b/crates/fbuild-daemon/src/handlers/emulator/select.rs @@ -307,15 +307,14 @@ pub async fn test_emu( }; let p = platform; - tokio::task::spawn_blocking(move || { - let orchestrator = fbuild_build::get_orchestrator(p)?; - orchestrator.build(¶ms) - }) - .await + match fbuild_build::get_orchestrator(p) { + Ok(orchestrator) => orchestrator.build(¶ms).await, + Err(e) => Err(e), + } }; let (firmware_path, elf_path) = match build_result { - Ok(Ok(r)) if r.success => { + Ok(r) if r.success => { let fw = r.firmware_path.clone().unwrap_or_else(|| { r.elf_path .clone() @@ -323,7 +322,7 @@ pub async fn test_emu( }); (fw, r.elf_path) } - Ok(Ok(r)) => { + Ok(r) => { return ( StatusCode::INTERNAL_SERVER_ERROR, Json(OperationResponse::fail( @@ -332,21 +331,12 @@ pub async fn test_emu( )), ); } - Ok(Err(e)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("build error: {}", e), - )), - ); - } Err(e) => { return ( StatusCode::INTERNAL_SERVER_ERROR, Json(OperationResponse::fail( request_id, - format!("build task panicked: {}", e), + format!("build error: {}", e), )), ); } diff --git a/crates/fbuild-daemon/src/handlers/emulator/shared.rs b/crates/fbuild-daemon/src/handlers/emulator/shared.rs index d98a56b1..2950b424 100644 --- a/crates/fbuild-daemon/src/handlers/emulator/shared.rs +++ b/crates/fbuild-daemon/src/handlers/emulator/shared.rs @@ -103,12 +103,12 @@ pub(crate) fn build_linux_macos_qemu_hint(err: &str) -> String { } } -pub(crate) fn resolve_esp32_toolchain_gcc_path( +pub(crate) async fn resolve_esp32_toolchain_gcc_path( project_dir: &Path, mcu_config: &fbuild_build::esp32::mcu_config::Esp32McuConfig, ) -> fbuild_core::Result { let platform = fbuild_packages::library::Esp32Platform::new(project_dir); - Package::ensure_installed(&platform)?; + Package::ensure_installed(&platform).await?; let is_riscv = mcu_config.is_riscv(); let prefix = mcu_config.toolchain_prefix(); @@ -122,11 +122,13 @@ pub(crate) fn resolve_esp32_toolchain_gcc_path( Ok(metadata_url) => { let cache = fbuild_packages::Cache::new(project_dir); let cache_dir = cache.toolchains_dir().join(toolchain_name); - match fbuild_packages::toolchain::esp32_metadata::resolve_toolchain_url_sync( + match fbuild_packages::toolchain::esp32_metadata::resolve_toolchain_url( &metadata_url, toolchain_name, &cache_dir, - ) { + ) + .await + { Ok(resolved) => fbuild_packages::toolchain::Esp32Toolchain::from_resolved( project_dir, &resolved.url, @@ -142,7 +144,7 @@ pub(crate) fn resolve_esp32_toolchain_gcc_path( Err(_) => fbuild_packages::toolchain::Esp32Toolchain::new(project_dir, is_riscv, &prefix), }; - let _ = Package::ensure_installed(&toolchain)?; + let _ = Package::ensure_installed(&toolchain).await?; Ok(toolchain.get_gcc_path()) } @@ -268,7 +270,7 @@ pub(crate) async fn run_qemu_process( break; } - if let Some(decoded_lines) = crash_decoder.process_line(&line.line) { + if let Some(decoded_lines) = crash_decoder.process_line(&line.line).await { for decoded in decoded_lines { synthetic_buf.push_str(&decoded); synthetic_buf.push('\n'); diff --git a/crates/fbuild-daemon/src/handlers/emulator/tests_npm_cache.rs b/crates/fbuild-daemon/src/handlers/emulator/tests_npm_cache.rs index 2953d7ad..af3ad7d0 100644 --- a/crates/fbuild-daemon/src/handlers/emulator/tests_npm_cache.rs +++ b/crates/fbuild-daemon/src/handlers/emulator/tests_npm_cache.rs @@ -12,12 +12,15 @@ use super::avr8js_npm::{ /// Serialises tests that mutate process-wide env vars (PATH). Without /// this, parallel cargo-test workers would clobber each other's PATH. -fn env_lock() -> std::sync::MutexGuard<'static, ()> { - use std::sync::{Mutex, OnceLock}; +/// +/// Uses `tokio::sync::Mutex` instead of `std::sync::Mutex` so the guard +/// can be safely held across the `.await` points inside the async tests +/// (clippy `await_holding_lock`). +async fn env_lock() -> tokio::sync::MutexGuard<'static, ()> { + use std::sync::OnceLock; + use tokio::sync::Mutex; static LOCK: OnceLock> = OnceLock::new(); - LOCK.get_or_init(|| Mutex::new(())) - .lock() - .unwrap_or_else(|p| p.into_inner()) + LOCK.get_or_init(|| Mutex::new(())).lock().await } #[test] @@ -102,9 +105,9 @@ fn prepare_avr8js_cache_nothing_to_clean_on_fresh_path() { ); } -#[test] -fn refresh_emu_cache_requested_recognises_truthy_values() { - let _guard = env_lock(); +#[tokio::test] +async fn refresh_emu_cache_requested_recognises_truthy_values() { + let _guard = env_lock().await; for (val, expected) in [ ("1", true), ("true", true), @@ -132,9 +135,9 @@ fn refresh_emu_cache_requested_recognises_truthy_values() { /// When npm isn't on PATH, `ensure_avr8js_npm_in` must return an /// `FbuildError::DeployFailed` that names both `npm` and the cache dir. /// This is the fix for issue #86's silent `ERR_MODULE_NOT_FOUND`. -#[test] -fn ensure_avr8js_npm_in_reports_clear_error_without_npm() { - let _guard = env_lock(); +#[tokio::test] +async fn ensure_avr8js_npm_in_reports_clear_error_without_npm() { + let _guard = env_lock().await; let saved_path = std::env::var_os("PATH"); // PATHEXT matters on Windows for command resolution of .cmd files. let saved_pathext = std::env::var_os("PATHEXT"); @@ -148,7 +151,7 @@ fn ensure_avr8js_npm_in_reports_clear_error_without_npm() { let tmp = tempfile::TempDir::new().unwrap(); let cache = tmp.path().join("avr8js-node"); - let result = ensure_avr8js_npm_in(&cache, false); + let result = ensure_avr8js_npm_in(&cache, false).await; // Restore BEFORE asserting so a panic doesn't leak PATH="" to sibling tests. if let Some(p) = saved_path { @@ -185,9 +188,9 @@ fn ensure_avr8js_npm_in_reports_clear_error_without_npm() { /// When the cache dir contains a corrupt partial install, the reinstall /// path must fire (detected here by asserting the partial tree is wiped /// even when the downstream npm call subsequently fails). -#[test] -fn ensure_avr8js_npm_in_wipes_corrupt_before_reinstall() { - let _guard = env_lock(); +#[tokio::test] +async fn ensure_avr8js_npm_in_wipes_corrupt_before_reinstall() { + let _guard = env_lock().await; let saved_path = std::env::var_os("PATH"); // Force the npm spawn to fail so we isolate the wipe behaviour. @@ -202,7 +205,7 @@ fn ensure_avr8js_npm_in_wipes_corrupt_before_reinstall() { std::fs::write(module_dir.join("garbage"), b"partial").unwrap(); assert!(!avr8js_cache_is_intact(&cache)); - let result = ensure_avr8js_npm_in(&cache, false); + let result = ensure_avr8js_npm_in(&cache, false).await; if let Some(p) = saved_path { std::env::set_var("PATH", p); diff --git a/crates/fbuild-daemon/src/handlers/emulator/tests_process.rs b/crates/fbuild-daemon/src/handlers/emulator/tests_process.rs index e5d10ec0..90c70375 100644 --- a/crates/fbuild-daemon/src/handlers/emulator/tests_process.rs +++ b/crates/fbuild-daemon/src/handlers/emulator/tests_process.rs @@ -93,9 +93,9 @@ async fn run_qemu_process_surfaces_crash_decoder_output() { } } -#[test] +#[tokio::test] #[ignore] -fn run_real_esp32s3_fixture_in_qemu() { +async fn run_real_esp32s3_fixture_in_qemu() { use fbuild_build::{BuildOrchestrator, BuildParams}; use fbuild_core::BuildProfile; @@ -150,6 +150,7 @@ fn run_real_esp32s3_fixture_in_qemu() { let orchestrator = fbuild_build::esp32::orchestrator::Esp32Orchestrator; let build_result = orchestrator .build(¶ms) + .await .expect("ESP32-S3 fixture build should succeed"); assert!(build_result.success); @@ -180,36 +181,39 @@ fn run_real_esp32s3_fixture_in_qemu() { ) .unwrap(); - let qemu = fbuild_packages::toolchain::EspQemuXtensa::new(&project_dir) - .and_then(|pkg| pkg.resolve_executable()) + let pkg = fbuild_packages::toolchain::EspQemuXtensa::new(&project_dir) + .expect("EspQemuXtensa::new should succeed for ignored integration test"); + let qemu = pkg + .resolve_executable() + .await .expect("native QEMU should resolve for ignored integration test"); let args = fbuild_deploy::esp32::build_qemu_args( &board.mcu, &flash_image, board.qemu_esp32_psram_config(), ); - let addr2line_path = resolve_esp32_toolchain_gcc_path(&project_dir, &mcu_config) - .ok() - .and_then(|gcc| fbuild_serial::crash_decoder::derive_addr2line_path(&gcc)); - - let rt = tokio::runtime::Runtime::new().unwrap(); - let result = rt - .block_on(run_qemu_process( - &qemu, - &args, - RunQemuOptions { - elf_path, - addr2line_path, - timeout_secs: Some(15.0), - halt_on_error: None, - halt_on_success: Some("Hello from ESP32-S3!"), - expect: Some("Hello from ESP32-S3!"), - show_timestamp: false, - verbose: true, - process_label: "QEMU", - }, - )) - .unwrap(); + let addr2line_path = match resolve_esp32_toolchain_gcc_path(&project_dir, &mcu_config).await { + Ok(gcc) => fbuild_serial::crash_decoder::derive_addr2line_path(&gcc), + Err(_) => None, + }; + + let result = run_qemu_process( + &qemu, + &args, + RunQemuOptions { + elf_path, + addr2line_path, + timeout_secs: Some(15.0), + halt_on_error: None, + halt_on_success: Some("Hello from ESP32-S3!"), + expect: Some("Hello from ESP32-S3!"), + show_timestamp: false, + verbose: true, + process_label: "QEMU", + }, + ) + .await + .unwrap(); assert!(result.stdout.contains("Hello from ESP32-S3!")); match result.outcome { diff --git a/crates/fbuild-daemon/src/handlers/operations/build.rs b/crates/fbuild-daemon/src/handlers/operations/build.rs index 1e3e2de7..b5a72693 100644 --- a/crates/fbuild-daemon/src/handlers/operations/build.rs +++ b/crates/fbuild-daemon/src/handlers/operations/build.rs @@ -322,11 +322,15 @@ pub async fn build( } }); - // Run build + // Run build. fbuild#813 / #815: the orchestrator is now async, + // so we spawn it directly on the runtime. The status-heartbeat + // loop still needs an awaitable handle, so wrap the build + // future in `tokio::spawn` and poll it via timeout-driven + // selection. let build_wallclock_start = std::time::Instant::now(); - let mut build_task = tokio::task::spawn_blocking(move || { + let mut build_task = tokio::spawn(async move { let orchestrator = fbuild_build::get_orchestrator(platform)?; - orchestrator.build(¶ms) + orchestrator.build(¶ms).await }); let build_result = loop { match tokio::time::timeout(STREAM_STATUS_INTERVAL, &mut build_task).await { @@ -517,14 +521,14 @@ pub async fn build( bloat_analysis: req.bloat_analysis, }; - let result = tokio::task::spawn_blocking(move || { - let orchestrator = fbuild_build::get_orchestrator(platform)?; - orchestrator.build(¶ms) - }) - .await; + // fbuild#813 / #815: orchestrator.build is async, call directly. + let result = match fbuild_build::get_orchestrator(platform) { + Ok(orch) => orch.build(¶ms).await, + Err(e) => Err(e), + }; match result { - Ok(Ok(build_result)) => { + Ok(build_result) => { let exported = if build_result.success { if let Some(ref out_dir) = resolved_output_dir { Some(export_artifacts_bundle( @@ -605,19 +609,11 @@ pub async fn build( ) .into_response() } - Ok(Err(e)) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("build error: {}", e), - )), - ) - .into_response(), Err(e) => ( StatusCode::INTERNAL_SERVER_ERROR, Json(OperationResponse::fail( request_id, - format!("build task panicked: {}", e), + format!("build error: {}", e), )), ) .into_response(), diff --git a/crates/fbuild-daemon/src/handlers/operations/deploy.rs b/crates/fbuild-daemon/src/handlers/operations/deploy.rs index 7447d35b..602e5de0 100644 --- a/crates/fbuild-daemon/src/handlers/operations/deploy.rs +++ b/crates/fbuild-daemon/src/handlers/operations/deploy.rs @@ -232,17 +232,17 @@ pub async fn deploy( bloat_analysis: false, }; - let build_result = { - let p = platform; - tokio::task::spawn_blocking(move || { - let orchestrator = fbuild_build::get_orchestrator(p)?; - orchestrator.build(¶ms) - }) - .await + // fbuild#813: orchestrator.build is now async — call directly, + // no spawn_blocking. The daemon runtime is multi-threaded so + // any heavy CPU work the orchestrator schedules (compile jobs) + // still runs in parallel via the executor. + let build_result = match fbuild_build::get_orchestrator(platform) { + Ok(orch) => orch.build(¶ms).await, + Err(e) => Err(e), }; match build_result { - Ok(Ok(r)) if r.success => { + Ok(r) if r.success => { let fw = r.firmware_path.clone().unwrap_or_else(|| { r.elf_path .clone() @@ -250,7 +250,7 @@ pub async fn deploy( }); (fw, r.elf_path) } - Ok(Ok(r)) => { + Ok(r) => { return ( StatusCode::INTERNAL_SERVER_ERROR, Json(OperationResponse::fail( @@ -259,21 +259,12 @@ pub async fn deploy( )), ); } - Ok(Err(e)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("build error: {}", e), - )), - ); - } Err(e) => { return ( StatusCode::INTERNAL_SERVER_ERROR, Json(OperationResponse::fail( request_id, - format!("build task panicked: {}", e), + format!("build error: {}", e), )), ); } @@ -415,7 +406,17 @@ pub async fn deploy( ctx.refresh_devices_if_stale_and_broadcast_serial_moves(std::time::Duration::from_secs(2)) .await; } - let deploy_result = tokio::task::spawn_blocking(move || -> fbuild_core::Result<(Option>, fbuild_deploy::DeploymentResult)> { + // fbuild#813 / #819: the Deployer trait is async; previously this + // entire block ran inside `spawn_blocking` because deploy() was sync. + // Now we run the dispatch + deploy inline on the daemon runtime — + // the long-running subprocess waits (esptool / avrdude / etc.) are + // tokio-aware via fbuild_core::subprocess::run_command, and the + // remaining espflash/native paths self-offload to spawn_blocking + // inside the Esp32Deployer. + let deploy_result: fbuild_core::Result<( + Option>, + fbuild_deploy::DeploymentResult, + )> = async { // Populated by the Espressif32 arm with (image_hash, port). // The tail of the closure consults it after `deployer.deploy` // returns to record or invalidate the daemon's trusted-hash @@ -575,7 +576,7 @@ pub async fn deploy( // that the verify call didn't understand. let mut selective_regions: Option> = None; if let Some(port) = deploy_port.as_deref() { - match deployer.try_verify_deployment(&deploy_fw, port) { + match deployer.try_verify_deployment(&deploy_fw, port).await { Ok(fbuild_deploy::esp32::VerifyOutcome::Match { stdout, stderr }) => { tracing::info!( port, @@ -629,7 +630,7 @@ pub async fn deploy( } } if let (Some(regions), Some(port)) = (selective_regions, deploy_port.as_deref()) { - let result = deployer.deploy_regions(&deploy_fw, port, ®ions); + let result = deployer.deploy_regions(&deploy_fw, port, ®ions).await; // Record/invalidate the trusted hash based on // the selective-flash outcome so the next warm // redeploy can short-circuit via trust-skip. @@ -715,12 +716,14 @@ pub async fn deploy( fbuild_core::Platform::NxpLpc => fbuild_deploy::lpc::dispatch_box(&board_id, &deploy_board_overrides, deploy_project.as_path(), baud_override), _ => return Err(fbuild_core::FbuildError::DeployFailed(format!("deployer for {:?} not yet implemented", platform))), }; - let result = deployer.deploy( - &deploy_project, - &deploy_env, - &deploy_fw, - deploy_port.as_deref(), - ); + let result = deployer + .deploy( + &deploy_project, + &deploy_env, + &deploy_fw, + deploy_port.as_deref(), + ) + .await; // Session-trusted verify-skip: record (or invalidate) the // image hash the daemon associates with this port. Scoped // to the Espressif32 arm above — other platforms leave @@ -744,15 +747,15 @@ pub async fn deploy( // Return the deployer so the async caller can invoke // `post_deploy_recovery` after `clear_preemption().await` (#605). result.map(|r| (Some(deployer), r)) - }) + } .await; // Split the deployer out so it can drive recovery while `deploy_result` - // retains its original shape for the downstream match. `None` covers - // verify-skip early returns, unsupported platforms, and join errors. + // retains its original shape for the downstream match. The async-block + // refactor (fbuild#813 / #819) replaces the join-error arm — no + // spawn_blocking, so the only failure mode is the deploy error itself. let (deployer_for_recovery, deploy_result) = match deploy_result { - Ok(Ok((d, r))) => (d, Ok(Ok(r))), - Ok(Err(e)) => (None, Ok(Err(e))), + Ok((d, r)) => (d, Ok(r)), Err(e) => (None, Err(e)), }; @@ -762,27 +765,27 @@ pub async fn deploy( // <4 s warm-trust-skip budget. let deploy_skipped_bus_work = matches!( &deploy_result, - Ok(Ok(r)) if r.success && matches!(r.outcome, fbuild_deploy::DeployOutcome::VerifySkip) + Ok(r) if r.success && matches!(r.outcome, fbuild_deploy::DeployOutcome::VerifySkip) ); if let Some(ref p) = deploy_port_str { ctx.serial_manager.clear_preemption(p).await; if !deploy_skipped_bus_work { if let Some(deployer) = deployer_for_recovery { let port_name = p.clone(); - let _ = tokio::task::spawn_blocking(move || { - if let Err(e) = deployer.post_deploy_recovery(&port_name) { - tracing::warn!("post_deploy_recovery failed for {}: {}", port_name, e); - } - }) - .await; + // post_deploy_recovery is now async (the default impl + // polls via tokio sleeps + spawn_blocking serialport + // probes); call it directly on the runtime. + if let Err(e) = deployer.post_deploy_recovery(&port_name).await { + tracing::warn!("post_deploy_recovery failed for {}: {}", port_name, e); + } } } } let (deploy_success, deploy_stdout, mut deploy_stderr, deploy_outcome, deploy_post_port) = match deploy_result { - Ok(Ok(r)) if r.success => (true, Some(r.stdout), Some(r.stderr), r.outcome, r.port), - Ok(Ok(r)) => { + Ok(r) if r.success => (true, Some(r.stdout), Some(r.stderr), r.outcome, r.port), + Ok(r) => { return ( StatusCode::INTERNAL_SERVER_ERROR, Json(OperationResponse { @@ -798,21 +801,12 @@ pub async fn deploy( }), ); } - Ok(Err(e)) => { - return ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("deploy error: {}", e), - )), - ); - } Err(e) => { return ( StatusCode::INTERNAL_SERVER_ERROR, Json(OperationResponse::fail( request_id, - format!("deploy task panicked: {}", e), + format!("deploy error: {}", e), )), ); } diff --git a/crates/fbuild-daemon/src/handlers/operations/install_deps.rs b/crates/fbuild-daemon/src/handlers/operations/install_deps.rs index 0a99f46a..380ee220 100644 --- a/crates/fbuild-daemon/src/handlers/operations/install_deps.rs +++ b/crates/fbuild-daemon/src/handlers/operations/install_deps.rs @@ -93,31 +93,21 @@ pub async fn install_deps( // Install dependencies via the package manager let env_label = env_name.clone(); - let result = tokio::task::spawn_blocking(move || { - fbuild_build::install_platform_deps(platform, &project_dir) - }) - .await; + let result = fbuild_build::install_platform_deps(platform, &project_dir).await; match result { - Ok(Ok(())) => ( + Ok(()) => ( StatusCode::OK, Json(OperationResponse::ok( request_id, format!("Dependencies installed for environment '{}'", env_label), )), ), - Ok(Err(e)) => ( - StatusCode::INTERNAL_SERVER_ERROR, - Json(OperationResponse::fail( - request_id, - format!("install-deps error: {}", e), - )), - ), Err(e) => ( StatusCode::INTERNAL_SERVER_ERROR, Json(OperationResponse::fail( request_id, - format!("install-deps task panicked: {}", e), + format!("install-deps error: {}", e), )), ), } diff --git a/crates/fbuild-daemon/src/handlers/websockets.rs b/crates/fbuild-daemon/src/handlers/websockets.rs index c35c0d6b..24907895 100644 --- a/crates/fbuild-daemon/src/handlers/websockets.rs +++ b/crates/fbuild-daemon/src/handlers/websockets.rs @@ -261,7 +261,7 @@ async fn handle_serial_ws(mut socket: WebSocket, ctx: Arc) { let mut lines: Vec = Vec::with_capacity(2); lines.push(line.clone()); if let Some(decoded) = - ctx.serial_manager.process_crash_line(&port_owned, &line) + ctx.serial_manager.process_crash_line(&port_owned, &line).await { lines.extend(decoded); } diff --git a/crates/fbuild-deploy/src/avr.rs b/crates/fbuild-deploy/src/avr.rs index 7419f4b5..f37b39b7 100644 --- a/crates/fbuild-deploy/src/avr.rs +++ b/crates/fbuild-deploy/src/avr.rs @@ -82,8 +82,9 @@ impl AvrDeployer { } } +#[async_trait::async_trait] impl Deployer for AvrDeployer { - fn deploy( + async fn deploy( &self, _project_dir: &Path, _env_name: &str, @@ -125,7 +126,8 @@ impl Deployer for AvrDeployer { None, None, Some(std::time::Duration::from_secs(self.timeout_secs)), - )?; + ) + .await?; if result.success() { Ok(DeploymentResult { @@ -188,11 +190,13 @@ mod tests { assert_eq!(deployer.baud_rate, "57600"); } - #[test] - fn test_deploy_requires_port() { + #[tokio::test] + async fn test_deploy_requires_port() { let deployer = AvrDeployer::new("atmega328p", "arduino", "115200", 60, None, false); let tmp = tempfile::TempDir::new().unwrap(); - let result = deployer.deploy(tmp.path(), "uno", Path::new("firmware.hex"), None); + let result = deployer + .deploy(tmp.path(), "uno", Path::new("firmware.hex"), None) + .await; assert!(result.is_err()); assert!(result .unwrap_err() diff --git a/crates/fbuild-deploy/src/esp32/deployer.rs b/crates/fbuild-deploy/src/esp32/deployer.rs index 3f33cdb9..d9da63ac 100644 --- a/crates/fbuild-deploy/src/esp32/deployer.rs +++ b/crates/fbuild-deploy/src/esp32/deployer.rs @@ -68,15 +68,12 @@ pub struct Esp32Deployer { } #[cfg(feature = "espflash-native")] -pub(super) fn native_write_or_fallback( +pub(super) fn native_write_or_fallback_outcome( port: &str, label: &str, - native: F, -) -> Option -where - F: FnOnce() -> Result, -{ - match native() { + outcome: Result, +) -> Option { + match outcome { Ok(result) if result.success => Some(result), Ok(result) => { tracing::warn!( @@ -291,10 +288,14 @@ impl Esp32Deployer { /// matching the post-flash behavior — so callers can treat a `true` /// return as "device is now running the requested firmware" without /// any extra reset. - pub fn try_verify_deployment(&self, firmware_path: &Path, port: &str) -> Result { + pub async fn try_verify_deployment( + &self, + firmware_path: &Path, + port: &str, + ) -> Result { #[cfg(feature = "espflash-native")] if self.use_native_verify { - match self.try_verify_deployment_native(firmware_path, port) { + match self.try_verify_deployment_native(firmware_path, port).await { Ok(outcome) => return Ok(outcome), Err(e) => { tracing::warn!( @@ -306,10 +307,10 @@ impl Esp32Deployer { } } - self.try_verify_deployment_esptool(firmware_path, port) + self.try_verify_deployment_esptool(firmware_path, port).await } - fn try_verify_deployment_esptool( + async fn try_verify_deployment_esptool( &self, firmware_path: &Path, port: &str, @@ -335,7 +336,8 @@ impl Esp32Deployer { // image. 30s gives plenty of headroom for slow USB-CDC stacks // and stub flasher upload retries. Some(std::time::Duration::from_secs(30)), - )?; + ) + .await?; if result.success() { Ok(VerifyOutcome::Match { @@ -378,7 +380,7 @@ impl Esp32Deployer { /// swap between the two behind the `use_native_verify` flag without /// any result-handling changes. #[cfg(feature = "espflash-native")] - fn try_verify_deployment_native( + async fn try_verify_deployment_native( &self, firmware_path: &Path, port: &str, @@ -416,17 +418,34 @@ impl Esp32Deployer { self.chip ); - crate::esp32_native::try_verify_deployment_native( - &self.chip, - port, - baud, - &self.before_reset, - &self.after_reset, - ®ions, - boot_off, - parts_off, - fw_off, - ) + // espflash's Flasher / Connection types are blocking (sync + // `serialport` under the hood). Offload to a blocking thread so + // the daemon's tokio runtime keeps draining other I/O while the + // ~6-10s verify happens. + let chip = self.chip.clone(); + let port = port.to_string(); + let before_reset = self.before_reset.clone(); + let after_reset = self.after_reset.clone(); + tokio::task::spawn_blocking(move || { + crate::esp32_native::try_verify_deployment_native( + &chip, + &port, + baud, + &before_reset, + &after_reset, + ®ions, + boot_off, + parts_off, + fw_off, + ) + }) + .await + .map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!( + "native verify: blocking task panicked: {}", + e + )) + })? } /// Native `write-flash` via the [`espflash`] crate (issue #66). @@ -438,7 +457,7 @@ impl Esp32Deployer { /// daemon's existing log broadcaster). Same [`DeploymentResult`] /// shape as the esptool path so callers swap behind a single flag. #[cfg(feature = "espflash-native")] - pub(super) fn try_deploy_native( + pub(super) async fn try_deploy_native( &self, firmware_path: &Path, port: &str, @@ -469,15 +488,30 @@ impl Esp32Deployer { self.chip ); - crate::esp32_native::try_write_deployment_native( - &self.chip, - port, - baud, - &self.before_reset, - &self.after_reset, - ®ions, - /* selective */ false, - ) + // espflash write is blocking (sync serialport + protocol); + // offload to a blocking thread so the runtime keeps moving. + let chip = self.chip.clone(); + let port = port.to_string(); + let before_reset = self.before_reset.clone(); + let after_reset = self.after_reset.clone(); + tokio::task::spawn_blocking(move || { + crate::esp32_native::try_write_deployment_native( + &chip, + &port, + baud, + &before_reset, + &after_reset, + ®ions, + /* selective */ false, + ) + }) + .await + .map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!( + "native write: blocking task panicked: {}", + e + )) + })? } /// Native `write-flash` for a caller-chosen subset of regions @@ -485,7 +519,7 @@ impl Esp32Deployer { /// regions that actually differ — skipping the ~1s /// bootloader/partitions rewrite when only firmware changed. #[cfg(feature = "espflash-native")] - pub(super) fn try_deploy_regions_native( + pub(super) async fn try_deploy_regions_native( &self, firmware_path: &Path, port: &str, @@ -519,15 +553,28 @@ impl Esp32Deployer { self.chip ); - crate::esp32_native::try_write_deployment_native( - &self.chip, - port, - baud, - &self.before_reset, - &self.after_reset, - &write_regions, - /* selective */ true, - ) + let chip = self.chip.clone(); + let port = port.to_string(); + let before_reset = self.before_reset.clone(); + let after_reset = self.after_reset.clone(); + tokio::task::spawn_blocking(move || { + crate::esp32_native::try_write_deployment_native( + &chip, + &port, + baud, + &before_reset, + &after_reset, + &write_regions, + /* selective */ true, + ) + }) + .await + .map_err(|e| { + fbuild_core::FbuildError::DeployFailed(format!( + "native write (selective): blocking task panicked: {}", + e + )) + })? } #[cfg(feature = "espflash-native")] @@ -612,7 +659,7 @@ impl Esp32Deployer { /// Returns an error when `regions` is empty; esptool rejects a /// write-flash call with no offset/file pair and the message would be /// opaque. - pub fn deploy_regions( + pub async fn deploy_regions( &self, firmware_path: &Path, port: &str, @@ -647,9 +694,10 @@ impl Esp32Deployer { #[cfg(feature = "espflash-native")] if self.use_native_write { - if let Some(result) = native_write_or_fallback(port, "selective write-flash", || { - self.try_deploy_regions_native(firmware_path, port, regions) - }) { + let native = self.try_deploy_regions_native(firmware_path, port, regions).await; + if let Some(result) = + native_write_or_fallback_outcome(port, "selective write-flash", native) + { return Ok(result); } } @@ -673,7 +721,8 @@ impl Esp32Deployer { None, None, Some(std::time::Duration::from_secs(120)), - )?; + ) + .await?; if result.success() { Ok(DeploymentResult { @@ -706,8 +755,9 @@ impl Esp32Deployer { } } +#[async_trait::async_trait] impl Deployer for Esp32Deployer { - fn deploy( + async fn deploy( &self, _project_dir: &Path, _env_name: &str, @@ -722,9 +772,8 @@ impl Deployer for Esp32Deployer { #[cfg(feature = "espflash-native")] if self.use_native_write { - if let Some(result) = native_write_or_fallback(port, "write-flash", || { - self.try_deploy_native(firmware_path, port) - }) { + let native = self.try_deploy_native(firmware_path, port).await; + if let Some(result) = native_write_or_fallback_outcome(port, "write-flash", native) { return Ok(result); } } @@ -748,7 +797,8 @@ impl Deployer for Esp32Deployer { None, None, Some(std::time::Duration::from_secs(120)), - )?; + ) + .await?; if result.success() { Ok(DeploymentResult { diff --git a/crates/fbuild-deploy/src/esp32/tests.rs b/crates/fbuild-deploy/src/esp32/tests.rs index 16a0bc75..686db70b 100644 --- a/crates/fbuild-deploy/src/esp32/tests.rs +++ b/crates/fbuild-deploy/src/esp32/tests.rs @@ -292,14 +292,16 @@ fn native_write_is_disabled_for_known_stalling_chips() { assert!(c3.use_native_write); } -#[test] -fn test_deploy_requires_port() { +#[tokio::test] +async fn test_deploy_requires_port() { let params = test_esptool_params(); let deployer = Esp32Deployer::new( "esp32c6", "460800", "0x0", "0x8000", "0x10000", ¶ms, false, ); let tmp = tempfile::TempDir::new().unwrap(); - let result = deployer.deploy(tmp.path(), "esp32c6", Path::new("firmware.bin"), None); + let result = deployer + .deploy(tmp.path(), "esp32c6", Path::new("firmware.bin"), None) + .await; assert!(result.is_err()); assert!(result .unwrap_err() @@ -508,8 +510,8 @@ fn build_write_flash_args_default_includes_all_regions() { /// with a clear error rather than silently emitting a write-flash /// call with no offset/file pair (which would produce an opaque /// esptool usage error). Addresses CodeRabbit review on PR #71. -#[test] -fn deploy_regions_errors_when_requested_region_file_missing() { +#[tokio::test] +async fn deploy_regions_errors_when_requested_region_file_missing() { let params = test_esptool_params(); let deployer = Esp32Deployer::new( "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, @@ -520,6 +522,7 @@ fn deploy_regions_errors_when_requested_region_file_missing() { // Note: no bootloader.bin written. let err = deployer .deploy_regions(&fw, "COM13", &[FlashRegion::Bootloader]) + .await .unwrap_err(); assert!( err.to_string().contains("bootloader.bin"), @@ -530,14 +533,15 @@ fn deploy_regions_errors_when_requested_region_file_missing() { /// Empty region slice -> usage error; we surface it rather than let /// esptool barf. -#[test] -fn deploy_regions_rejects_empty_slice() { +#[tokio::test] +async fn deploy_regions_rejects_empty_slice() { let params = test_esptool_params(); let deployer = Esp32Deployer::new( "esp32s3", "921600", "0x0", "0x8000", "0x10000", ¶ms, false, ); let err = deployer .deploy_regions(Path::new("firmware.bin"), "COM13", &[]) + .await .unwrap_err(); assert!(err.to_string().contains("no regions")); } @@ -589,7 +593,7 @@ fn verify_outcome_is_match_helper() { /// 2. Asserts that verify against the pre-flashed image returns `Match` /// in under 15 seconds. /// 3. Asserts that a tampered image (1 byte flipped) returns `Mismatch`. -fn run_verify_deployment_test( +async fn run_verify_deployment_test( chip: &str, bootloader_offset: &str, port_env: &str, @@ -649,6 +653,7 @@ fn run_verify_deployment_test( let start = std::time::Instant::now(); let outcome = deployer .try_verify_deployment(&reference, &port) + .await .unwrap_or_else(|e| panic!("verify must not fail against attached {}: {}", chip, e)); let elapsed = start.elapsed(); assert!( @@ -683,6 +688,7 @@ fn run_verify_deployment_test( let outcome = deployer .try_verify_deployment(&tampered, &port) + .await .unwrap_or_else(|e| { panic!( "[{}] verify must not fail with tampered firmware: {}", @@ -704,10 +710,10 @@ fn run_verify_deployment_test( /// ESP32_PORT=COM5 ESP32_FIRMWARE=C:\path\to\firmware.bin \ /// soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32 -- --ignored --nocapture /// ``` -#[test] +#[tokio::test] #[ignore = "requires real ESP32 board — set ESP32_PORT and ESP32_FIRMWARE"] -fn try_verify_deployment_real_esp32() { - run_verify_deployment_test("esp32", "0x1000", "ESP32_PORT", "ESP32_FIRMWARE"); +async fn try_verify_deployment_real_esp32() { + run_verify_deployment_test("esp32", "0x1000", "ESP32_PORT", "ESP32_FIRMWARE").await; } /// ESP32-S2 (Xtensa single-core, bootloader at 0x1000). @@ -716,10 +722,10 @@ fn try_verify_deployment_real_esp32() { /// ESP32S2_PORT=COM6 ESP32S2_FIRMWARE=C:\path\to\firmware.bin \ /// soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32s2 -- --ignored --nocapture /// ``` -#[test] +#[tokio::test] #[ignore = "requires real ESP32-S2 board — set ESP32S2_PORT and ESP32S2_FIRMWARE"] -fn try_verify_deployment_real_esp32s2() { - run_verify_deployment_test("esp32s2", "0x1000", "ESP32S2_PORT", "ESP32S2_FIRMWARE"); +async fn try_verify_deployment_real_esp32s2() { + run_verify_deployment_test("esp32s2", "0x1000", "ESP32S2_PORT", "ESP32S2_FIRMWARE").await; } /// ESP32-S3 (Xtensa dual-core, bootloader at 0x0). @@ -731,10 +737,10 @@ fn try_verify_deployment_real_esp32s2() { /// ESP32S3_PORT=COM13 ESP32S3_FIRMWARE=C:\Users\niteris\dev\fastled\.pio\build\esp32s3\firmware.bin \ /// soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32s3 -- --ignored --nocapture /// ``` -#[test] +#[tokio::test] #[ignore = "requires real ESP32-S3 board — set ESP32S3_PORT and ESP32S3_FIRMWARE"] -fn try_verify_deployment_real_esp32s3() { - run_verify_deployment_test("esp32s3", "0x0", "ESP32S3_PORT", "ESP32S3_FIRMWARE"); +async fn try_verify_deployment_real_esp32s3() { + run_verify_deployment_test("esp32s3", "0x0", "ESP32S3_PORT", "ESP32S3_FIRMWARE").await; } /// ESP32-C2 (RISC-V single-core, bootloader at 0x0). @@ -743,10 +749,10 @@ fn try_verify_deployment_real_esp32s3() { /// ESP32C2_PORT=COM7 ESP32C2_FIRMWARE=C:\path\to\firmware.bin \ /// soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32c2 -- --ignored --nocapture /// ``` -#[test] +#[tokio::test] #[ignore = "requires real ESP32-C2 board — set ESP32C2_PORT and ESP32C2_FIRMWARE"] -fn try_verify_deployment_real_esp32c2() { - run_verify_deployment_test("esp32c2", "0x0", "ESP32C2_PORT", "ESP32C2_FIRMWARE"); +async fn try_verify_deployment_real_esp32c2() { + run_verify_deployment_test("esp32c2", "0x0", "ESP32C2_PORT", "ESP32C2_FIRMWARE").await; } /// ESP32-C3 (RISC-V single-core, bootloader at 0x0). @@ -755,10 +761,10 @@ fn try_verify_deployment_real_esp32c2() { /// ESP32C3_PORT=COM8 ESP32C3_FIRMWARE=C:\path\to\firmware.bin \ /// soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32c3 -- --ignored --nocapture /// ``` -#[test] +#[tokio::test] #[ignore = "requires real ESP32-C3 board — set ESP32C3_PORT and ESP32C3_FIRMWARE"] -fn try_verify_deployment_real_esp32c3() { - run_verify_deployment_test("esp32c3", "0x0", "ESP32C3_PORT", "ESP32C3_FIRMWARE"); +async fn try_verify_deployment_real_esp32c3() { + run_verify_deployment_test("esp32c3", "0x0", "ESP32C3_PORT", "ESP32C3_FIRMWARE").await; } /// ESP32-C6 (RISC-V single-core, bootloader at 0x0). @@ -767,10 +773,10 @@ fn try_verify_deployment_real_esp32c3() { /// ESP32C6_PORT=COM9 ESP32C6_FIRMWARE=C:\path\to\firmware.bin \ /// soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32c6 -- --ignored --nocapture /// ``` -#[test] +#[tokio::test] #[ignore = "requires real ESP32-C6 board — set ESP32C6_PORT and ESP32C6_FIRMWARE"] -fn try_verify_deployment_real_esp32c6() { - run_verify_deployment_test("esp32c6", "0x0", "ESP32C6_PORT", "ESP32C6_FIRMWARE"); +async fn try_verify_deployment_real_esp32c6() { + run_verify_deployment_test("esp32c6", "0x0", "ESP32C6_PORT", "ESP32C6_FIRMWARE").await; } /// ESP32-H2 (RISC-V single-core, bootloader at 0x0). @@ -779,10 +785,10 @@ fn try_verify_deployment_real_esp32c6() { /// ESP32H2_PORT=COM10 ESP32H2_FIRMWARE=C:\path\to\firmware.bin \ /// soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32h2 -- --ignored --nocapture /// ``` -#[test] +#[tokio::test] #[ignore = "requires real ESP32-H2 board — set ESP32H2_PORT and ESP32H2_FIRMWARE"] -fn try_verify_deployment_real_esp32h2() { - run_verify_deployment_test("esp32h2", "0x0", "ESP32H2_PORT", "ESP32H2_FIRMWARE"); +async fn try_verify_deployment_real_esp32h2() { + run_verify_deployment_test("esp32h2", "0x0", "ESP32H2_PORT", "ESP32H2_FIRMWARE").await; } /// ESP32-P4 (RISC-V dual-core, OPI flash, bootloader at 0x2000). @@ -794,8 +800,8 @@ fn try_verify_deployment_real_esp32h2() { /// ESP32P4_PORT=COM11 ESP32P4_FIRMWARE=C:\path\to\firmware.bin \ /// soldr cargo test -p fbuild-deploy esp32::tests::try_verify_deployment_real_esp32p4 -- --ignored --nocapture /// ``` -#[test] +#[tokio::test] #[ignore = "requires real ESP32-P4 board — set ESP32P4_PORT and ESP32P4_FIRMWARE"] -fn try_verify_deployment_real_esp32p4() { - run_verify_deployment_test("esp32p4", "0x2000", "ESP32P4_PORT", "ESP32P4_FIRMWARE"); +async fn try_verify_deployment_real_esp32p4() { + run_verify_deployment_test("esp32p4", "0x2000", "ESP32P4_PORT", "ESP32P4_FIRMWARE").await; } diff --git a/crates/fbuild-deploy/src/lib.rs b/crates/fbuild-deploy/src/lib.rs index 7d222e31..70804461 100644 --- a/crates/fbuild-deploy/src/lib.rs +++ b/crates/fbuild-deploy/src/lib.rs @@ -92,8 +92,17 @@ pub struct DeploymentResult { } /// Trait for platform-specific deployers. +/// +/// Async per fbuild#813 / #819. The orchestrator calls into platform-specific +/// `deploy` and `post_deploy_recovery` from inside the daemon's tokio runtime; +/// implementations may `.await` subprocess I/O directly. CPU-bound or +/// inherently-sync work (e.g. the native espflash `Flasher` from the +/// `espflash` crate, or the `serialport` crate's blocking open path) must +/// still be wrapped in `tokio::task::spawn_blocking` inside the +/// implementation. +#[async_trait::async_trait] pub trait Deployer: Send + Sync { - fn deploy( + async fn deploy( &self, project_dir: &Path, env_name: &str, @@ -118,17 +127,26 @@ pub trait Deployer: Send + Sync { /// Phase 1 follow-up. /// // TODO(#605 Phase 1): LPC + CMSIS-DAP wedge-recovery override. - fn post_deploy_recovery(&self, port: &str) -> Result<()> { + async fn post_deploy_recovery(&self, port: &str) -> Result<()> { let deadline = std::time::Instant::now() + std::time::Duration::from_secs(3); + let port = port.to_string(); while std::time::Instant::now() < deadline { - if serialport::new(port, 115200) - .timeout(std::time::Duration::from_millis(50)) - .open() - .is_ok() - { + // serialport::new(...).open() is blocking; offload it. Each + // probe is short (50ms timeout) so spawn_blocking churn is + // bounded to ~30 calls over the 3s budget. + let port_for_probe = port.clone(); + let opened = tokio::task::spawn_blocking(move || { + serialport::new(&port_for_probe, 115200) + .timeout(std::time::Duration::from_millis(50)) + .open() + .is_ok() + }) + .await + .unwrap_or(false); + if opened { return Ok(()); } - std::thread::sleep(std::time::Duration::from_millis(100)); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; } tracing::warn!("USB re-enumeration: port {} not available after 3s", port); Ok(()) @@ -201,8 +219,9 @@ mod post_deploy_recovery_tests { port_seen: Arc>>, } + #[async_trait::async_trait] impl Deployer for ObservableDeployer { - fn deploy( + async fn deploy( &self, _project_dir: &Path, _env_name: &str, @@ -212,7 +231,7 @@ mod post_deploy_recovery_tests { unreachable!("deploy not exercised by post_deploy_recovery tests") } - fn post_deploy_recovery(&self, port: &str) -> Result<()> { + async fn post_deploy_recovery(&self, port: &str) -> Result<()> { self.called.store(true, Ordering::SeqCst); *self.port_seen.lock().unwrap() = Some(port.to_string()); Ok(()) @@ -226,8 +245,9 @@ mod post_deploy_recovery_tests { deploy_calls: Arc, } + #[async_trait::async_trait] impl Deployer for DefaultRecoveryDeployer { - fn deploy( + async fn deploy( &self, _project_dir: &Path, _env_name: &str, @@ -246,8 +266,8 @@ mod post_deploy_recovery_tests { } } - #[test] - fn override_is_dispatched_through_box_dyn() { + #[tokio::test] + async fn override_is_dispatched_through_box_dyn() { let called = Arc::new(AtomicBool::new(false)); let port_seen = Arc::new(std::sync::Mutex::new(None)); let dep: Box = Box::new(ObservableDeployer { @@ -256,6 +276,7 @@ mod post_deploy_recovery_tests { }); dep.post_deploy_recovery("COM-fake") + .await .expect("override returns Ok"); assert!(called.load(Ordering::SeqCst), "override must run"); @@ -266,8 +287,8 @@ mod post_deploy_recovery_tests { ); } - #[test] - fn default_impl_returns_ok_for_nonexistent_port_within_budget() { + #[tokio::test] + async fn default_impl_returns_ok_for_nonexistent_port_within_budget() { // The default impl polls the port for up to 3 seconds and then // returns Ok regardless. Using a port name that cannot possibly // exist exercises the slow path. Budget is 4s — the impl itself @@ -276,7 +297,7 @@ mod post_deploy_recovery_tests { deploy_calls: Arc::new(AtomicUsize::new(0)), }; let start = std::time::Instant::now(); - let result = dep.post_deploy_recovery("a-port-that-does-not-exist-zzz"); + let result = dep.post_deploy_recovery("a-port-that-does-not-exist-zzz").await; let elapsed = start.elapsed(); assert!(result.is_ok(), "default impl returns Ok even on timeout"); assert!( diff --git a/crates/fbuild-deploy/src/lpc.rs b/crates/fbuild-deploy/src/lpc.rs index ec7e492d..02476c58 100644 --- a/crates/fbuild-deploy/src/lpc.rs +++ b/crates/fbuild-deploy/src/lpc.rs @@ -162,8 +162,9 @@ pub fn dispatch_box( Box::new(deployer) } +#[async_trait::async_trait] impl Deployer for LpcDeployer { - fn deploy( + async fn deploy( &self, _project_dir: &Path, _env_name: &str, @@ -219,7 +220,8 @@ impl Deployer for LpcDeployer { None, None, Some(std::time::Duration::from_secs(self.timeout_secs)), - )?; + ) + .await?; if result.success() { Ok(DeploymentResult { @@ -288,11 +290,13 @@ mod tests { assert_eq!(deployer.baud_rate, "57600"); } - #[test] - fn test_deploy_requires_port() { + #[tokio::test] + async fn test_deploy_requires_port() { let deployer = LpcDeployer::new("115200", 12_000, 60, None, false); let tmp = tempfile::TempDir::new().unwrap(); - let result = deployer.deploy(tmp.path(), "lpc845", Path::new("firmware.hex"), None); + let result = deployer + .deploy(tmp.path(), "lpc845", Path::new("firmware.hex"), None) + .await; assert!(result.is_err()); assert!(result .unwrap_err() diff --git a/crates/fbuild-deploy/src/teensy/flash.rs b/crates/fbuild-deploy/src/teensy/flash.rs index 943eb0a1..471b9df6 100644 --- a/crates/fbuild-deploy/src/teensy/flash.rs +++ b/crates/fbuild-deploy/src/teensy/flash.rs @@ -87,7 +87,7 @@ impl FlashRunOutcome { /// after a baud-134 trigger; every subsequent retry uses the smaller /// `subsequent_attempt_timeout` since by then HalfKay has either already been /// observed or the device is wedged in a way another retry won't fix. -pub fn run_with_retry( +pub async fn run_with_retry( cfg: &FlashConfig, retries: u32, backoff_ms: u64, @@ -119,7 +119,7 @@ pub fn run_with_retry( } else { subsequent_attempt_timeout }; - let result = run_command(&args_ref, None, None, Some(attempt_timeout))?; + let result = run_command(&args_ref, None, None, Some(attempt_timeout)).await?; let success = result.success(); let exit_code = result.exit_code; let stdout = result.stdout; @@ -164,7 +164,7 @@ pub fn run_with_retry( last_err, backoff_ms ); - std::thread::sleep(Duration::from_millis(backoff_ms)); + tokio::time::sleep(Duration::from_millis(backoff_ms)).await; } } diff --git a/crates/fbuild-deploy/src/teensy/mod.rs b/crates/fbuild-deploy/src/teensy/mod.rs index bdb13e86..5ef913e7 100644 --- a/crates/fbuild-deploy/src/teensy/mod.rs +++ b/crates/fbuild-deploy/src/teensy/mod.rs @@ -145,8 +145,9 @@ fn resolve_trigger_port(explicit: Option<&str>) -> Option { port_discovery::first_pjrc_cdc_port() } +#[async_trait::async_trait] impl Deployer for TeensyDeployer { - fn deploy( + async fn deploy( &self, _project_dir: &Path, _env_name: &str, @@ -178,13 +179,34 @@ impl Deployer for TeensyDeployer { if self.baud_134_trigger { if let Some(ref tp) = trigger_port { if port_discovery::is_pjrc_cdc(tp) { - match soft_reboot::baud_134_trigger(tp, self.verbose) { - Ok(true) => { + // serialport open + DTR/RTS + sleep is blocking; + // offload so the runtime stays responsive. + let tp_owned = tp.clone(); + let verbose = self.verbose; + let trigger_result = + tokio::task::spawn_blocking(move || -> Result<(bool, String)> { + let triggered = soft_reboot::baud_134_trigger(&tp_owned, verbose)?; + Ok((triggered, tp_owned)) + }) + .await + .unwrap_or_else(|e| { + Err(fbuild_core::FbuildError::DeployFailed(format!( + "baud-134 trigger task panicked: {}", + e + ))) + }); + match trigger_result { + Ok((true, port_name)) => { // Confirm the device left CDC class (entered HalfKay). - let _ = - halfkay_probe::wait_for_disappearance(tp, Duration::from_secs(3)); + let _ = tokio::task::spawn_blocking(move || { + halfkay_probe::wait_for_disappearance( + &port_name, + Duration::from_secs(3), + ) + }) + .await; } - Ok(false) => { + Ok((false, _)) => { // Already gone — treat as already-HalfKay. } Err(e) => { @@ -238,7 +260,8 @@ impl Deployer for TeensyDeployer { // before falling through to the structured diagnostic. Duration::from_secs(self.flash_timeout_secs), self.verbose, - )?; + ) + .await?; if !flash_outcome.success { let attempt_count = flash_outcome.attempts.len(); @@ -258,10 +281,18 @@ impl Deployer for TeensyDeployer { } // ---- 4. Post-flash CDC ACM port discovery ---------------------- - let new_port = match port_discovery::wait_for_new_cdc_port( - &pre_snapshot, - Duration::from_secs(self.post_flash_port_discovery_secs), - ) { + // wait_for_new_cdc_port polls the OS port list with 100ms sleeps; + // offload to a blocking thread so the runtime is free during the + // up-to-5s discovery window. + let discovery_pre = pre_snapshot.clone(); + let discovery_window = Duration::from_secs(self.post_flash_port_discovery_secs); + let discovery_outcome = + tokio::task::spawn_blocking(move || { + port_discovery::wait_for_new_cdc_port(&discovery_pre, discovery_window) + }) + .await + .unwrap_or(port_discovery::NewPortOutcome::TimedOut); + let new_port = match discovery_outcome { port_discovery::NewPortOutcome::Found(name) => Some(name), port_discovery::NewPortOutcome::TimedOut => { // Re-enumeration may have reused the same port name (the @@ -277,11 +308,13 @@ impl Deployer for TeensyDeployer { let mut message_suffix = String::new(); if let Some(ref np) = new_port { if self.first_byte_timeout_secs > 0 { - let outcome = first_byte_probe::probe( - np, - 115_200, - Duration::from_secs(self.first_byte_timeout_secs), - ); + let np_owned = np.clone(); + let timeout = Duration::from_secs(self.first_byte_timeout_secs); + let outcome = tokio::task::spawn_blocking(move || { + first_byte_probe::probe(&np_owned, 115_200, timeout) + }) + .await + .unwrap_or(first_byte_probe::FirstByteOutcome::Disabled); match outcome { first_byte_probe::FirstByteOutcome::SawByte { elapsed_ms } => { message_suffix.push_str(&format!("; first byte at {} ms", elapsed_ms)); diff --git a/crates/fbuild-packages/Cargo.toml b/crates/fbuild-packages/Cargo.toml index ee871472..7333c197 100644 --- a/crates/fbuild-packages/Cargo.toml +++ b/crates/fbuild-packages/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true fbuild-core = { path = "../fbuild-core" } fbuild-paths = { path = "../fbuild-paths" } fbuild-config = { path = "../fbuild-config" } +async-trait = { workspace = true } tokio = { workspace = true } reqwest = { workspace = true } serde = { workspace = true } diff --git a/crates/fbuild-packages/src/disk_cache/budget.rs b/crates/fbuild-packages/src/disk_cache/budget.rs index 0cda75c0..80c43cc3 100644 --- a/crates/fbuild-packages/src/disk_cache/budget.rs +++ b/crates/fbuild-packages/src/disk_cache/budget.rs @@ -134,7 +134,7 @@ fn get_total_disk_space_windows(path: &Path) -> u64 { &drive[..1], &drive[..1] ); - fbuild_core::subprocess::run_command( + fbuild_core::subprocess::run_command_blocking( &["powershell", "-NoProfile", "-Command", &ps_cmd], None, None, @@ -151,8 +151,12 @@ fn get_total_disk_space_unix(path: &Path) -> u64 { // `-P` gives POSIX format, `-k` forces 1024-byte blocks on all // platforms. Route through the containment group (issue #32). let path_str = path.to_string_lossy(); - let output = - fbuild_core::subprocess::run_command(&["df", "-P", "-k", &path_str], None, None, None); + let output = fbuild_core::subprocess::run_command_blocking( + &["df", "-P", "-k", &path_str], + None, + None, + None, + ); output .ok() diff --git a/crates/fbuild-packages/src/downloader.rs b/crates/fbuild-packages/src/downloader.rs index bef976da..d1ab682f 100644 --- a/crates/fbuild-packages/src/downloader.rs +++ b/crates/fbuild-packages/src/downloader.rs @@ -10,6 +10,8 @@ use std::time::{Duration, Instant}; use fbuild_core::{FbuildError, Result}; use sha2::{Digest, Sha256}; +use crate::http; + /// Number of GET attempts before giving up on a transient failure. /// One initial attempt + two retries. Worst-case wall time at the /// default backoff schedule is ~4 s of sleep before the third @@ -110,7 +112,7 @@ async fn get_with_retry(url: &str) -> Result> { let mut attempt: u32 = 0; loop { attempt += 1; - match reqwest::get(url).await { + match http::client().get(url).send().await { Ok(response) => { let status = response.status(); if !status.is_success() { @@ -175,7 +177,7 @@ async fn open_with_retry(url: &str) -> Result { let mut attempt: u32 = 0; loop { attempt += 1; - match reqwest::get(url).await { + match http::client().get(url).send().await { Ok(response) => { let status = response.status(); if status.is_success() { diff --git a/crates/fbuild-packages/src/http.rs b/crates/fbuild-packages/src/http.rs new file mode 100644 index 00000000..6512d1ca --- /dev/null +++ b/crates/fbuild-packages/src/http.rs @@ -0,0 +1,42 @@ +//! Shared async `reqwest::Client` with configured timeouts. +//! +//! FastLED/fbuild#813 (async migration) + #805 (timeout audit): +//! every HTTP call in fbuild-packages goes through this client. +//! Tokio-console sees the I/O because the underlying tokio +//! reactor is the daemon's. Timeouts default to safe values; per- +//! call overrides via `client_with_timeout(...)`. + +use std::sync::OnceLock; +use std::time::Duration; + +use reqwest::Client; + +/// Default per-request timeout. Long enough for slow CDN downloads +/// of multi-MB toolchain archives, short enough that a wedged +/// server doesn't pin a fbuild-daemon worker indefinitely. +const DEFAULT_TIMEOUT: Duration = Duration::from_secs(300); // 5 min +const DEFAULT_CONNECT_TIMEOUT: Duration = Duration::from_secs(30); + +static CLIENT: OnceLock = OnceLock::new(); + +/// Get the shared client. Lazily built on first call. +pub fn client() -> &'static Client { + CLIENT.get_or_init(|| { + Client::builder() + .timeout(DEFAULT_TIMEOUT) + .connect_timeout(DEFAULT_CONNECT_TIMEOUT) + .build() + .expect("reqwest client builder should never fail with these settings") + }) +} + +/// Build a client with a custom total timeout (for callers that +/// need a tighter deadline, e.g. a registry ping that should +/// fail fast). +pub fn client_with_timeout(total: Duration) -> Client { + Client::builder() + .timeout(total) + .connect_timeout(DEFAULT_CONNECT_TIMEOUT) + .build() + .expect("reqwest client builder should never fail with valid settings") +} diff --git a/crates/fbuild-packages/src/lib.rs b/crates/fbuild-packages/src/lib.rs index fe3d8ab1..1dea4eee 100644 --- a/crates/fbuild-packages/src/lib.rs +++ b/crates/fbuild-packages/src/lib.rs @@ -10,6 +10,7 @@ pub mod cache; pub mod disk_cache; pub mod downloader; pub mod extractor; +pub mod http; pub mod library; pub mod lnk; pub mod toolchain; @@ -21,10 +22,10 @@ pub use disk_cache::DiskCache; pub use lnk::{ExtractMode, LnkFile}; use std::collections::{HashMap, HashSet}; -use std::future::Future; use std::path::{Path, PathBuf}; use std::sync::{Mutex, OnceLock}; +use async_trait::async_trait; use fbuild_core::install_status::{self, InstallPhase, InstallRole}; static PACKAGE_TOUCHES: OnceLock>> = OnceLock::new(); @@ -53,24 +54,15 @@ fn dir_size(path: &Path) -> u64 { .sum() } -pub(crate) fn block_on_package_future(future: F) -> fbuild_core::Result -where - F: Future>, -{ - if let Ok(handle) = tokio::runtime::Handle::try_current() { - tokio::task::block_in_place(|| handle.block_on(future)) - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!("failed to create tokio runtime: {}", e)) - })?; - rt.block_on(future) - } -} - /// Base trait for all installable packages. +/// +/// FastLED/fbuild#813: `ensure_installed` is async so it composes with the +/// daemon's tokio reactor and tokio-console sees every package install as +/// part of the task graph. Use `#[async_trait]` on impls. +#[async_trait] pub trait Package: Send + Sync { /// Ensure the package is installed, downloading if necessary. - fn ensure_installed(&self) -> fbuild_core::Result; + async fn ensure_installed(&self) -> fbuild_core::Result; /// Check if the package is already installed. fn is_installed(&self) -> bool; @@ -341,7 +333,7 @@ impl PackageBase { /// 5. Rename staging to final path (atomic commit) pub async fn staged_install(&self, validate: F) -> fbuild_core::Result where - F: FnOnce(&Path) -> fbuild_core::Result<()>, + F: FnOnce(&Path) -> fbuild_core::Result<()> + Send, { let install_path = self.install_path(); @@ -531,8 +523,9 @@ mod toolchain_gcc_ar_tests { ar_path: PathBuf, } + #[async_trait] impl Package for TestToolchain { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { Ok(PathBuf::new()) } fn is_installed(&self) -> bool { diff --git a/crates/fbuild-packages/src/library/apollo3_core.rs b/crates/fbuild-packages/src/library/apollo3_core.rs index 95a9842b..feae7c5c 100644 --- a/crates/fbuild-packages/src/library/apollo3_core.rs +++ b/crates/fbuild-packages/src/library/apollo3_core.rs @@ -120,25 +120,14 @@ impl Apollo3Cores { } } +#[async_trait::async_trait] impl crate::Package for Apollo3Cores { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_core_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/arduino_api.rs b/crates/fbuild-packages/src/library/arduino_api.rs index 3255a75d..abe10b8e 100644 --- a/crates/fbuild-packages/src/library/arduino_api.rs +++ b/crates/fbuild-packages/src/library/arduino_api.rs @@ -13,6 +13,8 @@ use std::path::Path; use fbuild_core::Result; +use crate::http; + /// Version of ArduinoCore-API to use. const ARDUINO_API_VERSION: &str = "1.5.2"; const ARDUINO_API_URL: &str = @@ -26,7 +28,7 @@ const ARDUINO_API_URL: &str = /// /// # Arguments /// * `core_dir` - The framework's `cores/arduino/` directory (or equivalent) -pub fn ensure_arduino_api(core_dir: &Path) -> Result<()> { +pub async fn ensure_arduino_api(core_dir: &Path) -> Result<()> { let api_marker = core_dir.join("api").join("ArduinoAPI.h"); if api_marker.exists() { tracing::debug!( @@ -47,8 +49,8 @@ pub fn ensure_arduino_api(core_dir: &Path) -> Result<()> { fbuild_core::FbuildError::PackageError(format!("failed to create temp dir: {}", e)) })?; - // Use blocking reqwest since we may or may not be in an async context - let response = reqwest::blocking::get(ARDUINO_API_URL).map_err(|e| { + // Async HTTP via the shared client (FastLED/fbuild#813). + let response = http::client().get(ARDUINO_API_URL).send().await.map_err(|e| { fbuild_core::FbuildError::PackageError(format!("failed to download ArduinoCore-API: {}", e)) })?; @@ -60,10 +62,10 @@ pub fn ensure_arduino_api(core_dir: &Path) -> Result<()> { } let archive_path = tmp_dir.path().join("ArduinoCore-API.tar.gz"); - let bytes = response.bytes().map_err(|e| { + let bytes = response.bytes().await.map_err(|e| { fbuild_core::FbuildError::PackageError(format!("failed to read response: {}", e)) })?; - std::fs::write(&archive_path, &bytes)?; + tokio::fs::write(&archive_path, &bytes).await?; // Extract let extract_dir = tmp_dir.path().join("extracted"); diff --git a/crates/fbuild-packages/src/library/arduino_core.rs b/crates/fbuild-packages/src/library/arduino_core.rs index f8a6e6c4..44b3da0e 100644 --- a/crates/fbuild-packages/src/library/arduino_core.rs +++ b/crates/fbuild-packages/src/library/arduino_core.rs @@ -118,25 +118,14 @@ impl ArduinoCore { } } +#[async_trait::async_trait] impl crate::Package for ArduinoCore { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_core_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/arduino_core_lpc8xx.rs b/crates/fbuild-packages/src/library/arduino_core_lpc8xx.rs index 2111493d..7c3393a3 100644 --- a/crates/fbuild-packages/src/library/arduino_core_lpc8xx.rs +++ b/crates/fbuild-packages/src/library/arduino_core_lpc8xx.rs @@ -160,24 +160,14 @@ fn find_core_root(install_dir: &Path) -> PathBuf { install_dir.to_path_buf() } +#[async_trait::async_trait] impl crate::Package for ArduinoCoreLpc8xx { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.base.install_path()); } - let rt = tokio::runtime::Handle::try_current().ok(); - if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))?; - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))?; - } + self.base.staged_install(Self::validate).await?; Ok(self.base.install_path()) } diff --git a/crates/fbuild-packages/src/library/arduino_mbed_core.rs b/crates/fbuild-packages/src/library/arduino_mbed_core.rs index bd9ad3dd..6567712d 100644 --- a/crates/fbuild-packages/src/library/arduino_mbed_core.rs +++ b/crates/fbuild-packages/src/library/arduino_mbed_core.rs @@ -167,29 +167,19 @@ impl ArduinoMbedCore { } } +#[async_trait::async_trait] impl crate::Package for ArduinoMbedCore { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { let root = self.resolved_dir(); - super::arduino_api::ensure_arduino_api(&root.join("cores").join("arduino"))?; + super::arduino_api::ensure_arduino_api(&root.join("cores").join("arduino")).await?; return Ok(root); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; + let install_path = self.base.staged_install(Self::validate).await?; let root = find_core_root(&install_path); - super::arduino_api::ensure_arduino_api(&root.join("cores").join("arduino"))?; + super::arduino_api::ensure_arduino_api(&root.join("cores").join("arduino")).await?; Ok(root) } diff --git a/crates/fbuild-packages/src/library/attiny_core.rs b/crates/fbuild-packages/src/library/attiny_core.rs index 2102d001..39c2d805 100644 --- a/crates/fbuild-packages/src/library/attiny_core.rs +++ b/crates/fbuild-packages/src/library/attiny_core.rs @@ -95,25 +95,14 @@ impl ATTinyCore { } } +#[async_trait::async_trait] impl crate::Package for ATTinyCore { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_core_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/avr_framework.rs b/crates/fbuild-packages/src/library/avr_framework.rs index f9ca611d..15e17c04 100644 --- a/crates/fbuild-packages/src/library/avr_framework.rs +++ b/crates/fbuild-packages/src/library/avr_framework.rs @@ -142,14 +142,15 @@ impl AvrFramework { } } +#[async_trait::async_trait] impl crate::Package for AvrFramework { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { let root = self.resolved_dir(); // Still ensure API is present (may have been cached without it) if self.needs_arduino_api { let core_dir = self.get_core_dir(&self.core_name); - super::arduino_api::ensure_arduino_api(&core_dir)?; + super::arduino_api::ensure_arduino_api(&core_dir).await?; } return Ok(root); } @@ -172,18 +173,7 @@ impl crate::Package for AvrFramework { Ok(()) }; - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(validate_fn))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(validate_fn))? - }; + let install_path = self.base.staged_install(validate_fn).await?; let root = find_framework_root(&install_path); @@ -191,7 +181,7 @@ impl crate::Package for AvrFramework { if self.needs_arduino_api { let core_dir_name = self.core_dir_override.as_deref().unwrap_or(&self.core_name); let core_dir = root.join("cores").join(core_dir_name); - super::arduino_api::ensure_arduino_api(&core_dir)?; + super::arduino_api::ensure_arduino_api(&core_dir).await?; } Ok(root) diff --git a/crates/fbuild-packages/src/library/ch32v_core.rs b/crates/fbuild-packages/src/library/ch32v_core.rs index 7bc95c1b..08d5a856 100644 --- a/crates/fbuild-packages/src/library/ch32v_core.rs +++ b/crates/fbuild-packages/src/library/ch32v_core.rs @@ -127,26 +127,16 @@ impl Ch32vCores { } } +#[async_trait::async_trait] impl crate::Package for Ch32vCores { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { let root = self.resolved_dir(); Self::patch_compatibility(&root)?; return Ok(root); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; + let install_path = self.base.staged_install(Self::validate).await?; let root = find_core_root(&install_path); Self::patch_compatibility(&root)?; diff --git a/crates/fbuild-packages/src/library/cmsis_atmel.rs b/crates/fbuild-packages/src/library/cmsis_atmel.rs index 8566c0b1..246b8bf9 100644 --- a/crates/fbuild-packages/src/library/cmsis_atmel.rs +++ b/crates/fbuild-packages/src/library/cmsis_atmel.rs @@ -96,25 +96,14 @@ fn find_device_root(install_dir: &Path) -> PathBuf { install_dir.to_path_buf() } +#[async_trait::async_trait] impl crate::Package for CmsisAtmel { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(find_device_root(&self.base.install_path())); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_device_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/cmsis_framework.rs b/crates/fbuild-packages/src/library/cmsis_framework.rs index 0e9e4d45..2308bae2 100644 --- a/crates/fbuild-packages/src/library/cmsis_framework.rs +++ b/crates/fbuild-packages/src/library/cmsis_framework.rs @@ -82,25 +82,14 @@ impl CmsisFramework { } } +#[async_trait::async_trait] impl crate::Package for CmsisFramework { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.base.install_path()); } - let rt = tokio::runtime::Handle::try_current().ok(); - if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))?; - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))?; - } - + self.base.staged_install(Self::validate).await?; Ok(self.base.install_path()) } diff --git a/crates/fbuild-packages/src/library/esp32_framework/libs.rs b/crates/fbuild-packages/src/library/esp32_framework/libs.rs index 2790a7f6..db62ef8d 100644 --- a/crates/fbuild-packages/src/library/esp32_framework/libs.rs +++ b/crates/fbuild-packages/src/library/esp32_framework/libs.rs @@ -98,7 +98,7 @@ fn patch_mcu_compatibility(mcu_dir: &Path, mcu: &str) -> fbuild_core::Result<()> impl Esp32Framework { /// Ensure the SDK libs are downloaded and extracted into the framework's `tools/` dir. - pub fn ensure_libs(&self, libs_url: &str) -> fbuild_core::Result<()> { + pub async fn ensure_libs(&self, libs_url: &str) -> fbuild_core::Result<()> { let root = self.resolved_dir(); let tools_dir = root.join("tools"); @@ -119,18 +119,7 @@ impl Esp32Framework { if !archive_path.exists() { tracing::info!("downloading ESP32 SDK libs"); - let rt = tokio::runtime::Handle::try_current().ok(); - if let Some(handle) = rt { - handle.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; - } + crate::downloader::download_file(libs_url, &tools_dir).await?; } // Extract to a short temp path to avoid Windows MAX_PATH (260 char) limit. @@ -158,7 +147,7 @@ impl Esp32Framework { /// skeleton package rather than the main `framework-arduinoespressif32-libs`. /// This merges the skeleton into the existing `tools/` directory without /// clobbering other MCU subdirs. - pub fn ensure_mcu_libs(&self, libs_url: &str, mcu: &str) -> fbuild_core::Result<()> { + pub async fn ensure_mcu_libs(&self, libs_url: &str, mcu: &str) -> fbuild_core::Result<()> { let root = self.resolved_dir(); let tools_dir = root.join("tools"); @@ -179,18 +168,7 @@ impl Esp32Framework { if !archive_path.exists() { tracing::info!("downloading {} skeleton libs", mcu); - let rt = tokio::runtime::Handle::try_current().ok(); - if let Some(handle) = rt { - handle.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(crate::downloader::download_file(libs_url, &tools_dir))?; - } + crate::downloader::download_file(libs_url, &tools_dir).await?; } let temp_dir = tempfile::Builder::new().prefix("fbuild_skel_").tempdir()?; diff --git a/crates/fbuild-packages/src/library/esp32_framework/mod.rs b/crates/fbuild-packages/src/library/esp32_framework/mod.rs index 5a8cc508..871feed3 100644 --- a/crates/fbuild-packages/src/library/esp32_framework/mod.rs +++ b/crates/fbuild-packages/src/library/esp32_framework/mod.rs @@ -125,25 +125,14 @@ impl Esp32Framework { } } +#[async_trait::async_trait] impl crate::Package for Esp32Framework { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_framework_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/esp32_platform.rs b/crates/fbuild-packages/src/library/esp32_platform.rs index ab55cedf..5eab3113 100644 --- a/crates/fbuild-packages/src/library/esp32_platform.rs +++ b/crates/fbuild-packages/src/library/esp32_platform.rs @@ -168,15 +168,14 @@ impl Esp32Platform { } } +#[async_trait::async_trait] impl crate::Package for Esp32Platform { - fn ensure_installed(&self) -> Result { + async fn ensure_installed(&self) -> Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let install_path = - crate::block_on_package_future(self.base.staged_install(Self::validate_install))?; - + let install_path = self.base.staged_install(Self::validate_install).await?; Ok(find_platform_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/esp8266_framework.rs b/crates/fbuild-packages/src/library/esp8266_framework.rs index 5336d4e8..701a0d38 100644 --- a/crates/fbuild-packages/src/library/esp8266_framework.rs +++ b/crates/fbuild-packages/src/library/esp8266_framework.rs @@ -171,8 +171,9 @@ impl Esp8266Framework { } } +#[async_trait::async_trait] impl crate::Package for Esp8266Framework { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } @@ -188,19 +189,7 @@ impl crate::Package for Esp8266Framework { Ok(()) }; - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(validate_fn))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(validate_fn))? - }; - + let install_path = self.base.staged_install(validate_fn).await?; Ok(find_framework_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/library_compiler.rs b/crates/fbuild-packages/src/library/library_compiler.rs index f26dab6e..39438520 100644 --- a/crates/fbuild-packages/src/library/library_compiler.rs +++ b/crates/fbuild-packages/src/library/library_compiler.rs @@ -50,7 +50,7 @@ fn is_cxx_only_flag(flag: &str) -> bool { /// /// Returns the archive path, or None if the library is header-only. #[allow(clippy::too_many_arguments)] -pub fn compile_library( +pub async fn compile_library( name: &str, source_files: &[PathBuf], include_dirs: &[PathBuf], @@ -77,11 +77,12 @@ pub fn compile_library( 1, compiler_cache, ) + .await } /// Compile all source files in a library with parallel jobs. #[allow(clippy::too_many_arguments)] -pub fn compile_library_with_jobs( +pub async fn compile_library_with_jobs( name: &str, source_files: &[PathBuf], include_dirs: &[PathBuf], @@ -112,7 +113,7 @@ pub fn compile_library_with_jobs( } // Build include flags once (shared across all compilations) - let include_flags = build_include_flags(include_dirs, output_dir)?; + let include_flags = build_include_flags(include_dirs, output_dir).await?; // Pre-compute C-safe flags once let c_safe_flags: Vec = c_flags @@ -175,7 +176,8 @@ pub fn compile_library_with_jobs( name, verbose, compiler_cache, - )?; + ) + .await?; } tracing::info!( @@ -184,7 +186,7 @@ pub fn compile_library_with_jobs( all_objects.len(), archive_path.display() ); - archive_objects(ar_path, &all_objects, &archive_path)?; + archive_objects(ar_path, &all_objects, &archive_path).await?; tracing::info!( "compiled library {}: {} changed / {} total files -> {}", name, @@ -195,72 +197,79 @@ pub fn compile_library_with_jobs( return Ok(Some(archive_path)); } - // Parallel path + // Parallel path — use a tokio Semaphore to bound concurrency. let total = stale_sources.len(); let thread_count = jobs.min(total); - - let work_iter = std::sync::Mutex::new(stale_sources.iter()); - let first_error: std::sync::Mutex> = std::sync::Mutex::new(None); - let compiled_count = std::sync::atomic::AtomicUsize::new(0); - - std::thread::scope(|scope| { - let handles: Vec<_> = (0..thread_count) - .map(|_| { - scope.spawn(|| { - loop { - if first_error.lock().unwrap().is_some() { - return; - } - - // Get next work item with its index - let item = { - let mut iter = work_iter.lock().unwrap(); - iter.next().cloned() - }; - - let source = match item { - Some(s) => s, - None => return, - }; - - match compile_one_source( - &source, - &obj_dir, - gcc_path, - gxx_path, - &c_safe_flags, - &cpp_flags, - &include_flags, - name, - verbose, - compiler_cache, - ) { - Ok(_) => { - let count = compiled_count - .fetch_add(1, std::sync::atomic::Ordering::Relaxed) - + 1; - if count % 20 == 0 || count == total { - tracing::info!("[{}/{}] compiled [{}]", count, total, name); - } - } - Err(e) => { - let mut err = first_error.lock().unwrap(); - if err.is_none() { - *err = Some(e.to_string()); - } - } - } - } - }) - }) - .collect(); - - for handle in handles { - handle.join().unwrap(); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(thread_count)); + + let mut tasks = tokio::task::JoinSet::new(); + let compiled_count = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + + let obj_dir_owned = obj_dir.clone(); + let gcc_path_owned = gcc_path.to_path_buf(); + let gxx_path_owned = gxx_path.to_path_buf(); + let c_safe_flags_arc = std::sync::Arc::new(c_safe_flags); + let cpp_flags_arc = std::sync::Arc::new(cpp_flags.clone()); + let include_flags_arc = std::sync::Arc::new(include_flags.clone()); + let name_owned = name.to_string(); + let compiler_cache_owned = compiler_cache.map(|p| p.to_path_buf()); + + for source in stale_sources.clone() { + let sem = sem.clone(); + let obj_dir_t = obj_dir_owned.clone(); + let gcc_t = gcc_path_owned.clone(); + let gxx_t = gxx_path_owned.clone(); + let c_safe_t = c_safe_flags_arc.clone(); + let cpp_t = cpp_flags_arc.clone(); + let inc_t = include_flags_arc.clone(); + let lib_name_t = name_owned.clone(); + let cache_t = compiler_cache_owned.clone(); + let counter = compiled_count.clone(); + tasks.spawn(async move { + let _permit = sem + .acquire() + .await + .map_err(|e| FbuildError::BuildFailed(format!("semaphore closed: {e}")))?; + let cache_ref = cache_t.as_deref(); + compile_one_source( + &source, + &obj_dir_t, + &gcc_t, + &gxx_t, + &c_safe_t, + &cpp_t, + &inc_t, + &lib_name_t, + verbose, + cache_ref, + ) + .await?; + let count = counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed) + 1; + if count % 20 == 0 || count == total { + tracing::info!("[{}/{}] compiled [{}]", count, total, lib_name_t); + } + Ok::<(), FbuildError>(()) + }); + } + + let mut first_error: Option = None; + while let Some(joined) = tasks.join_next().await { + match joined { + Ok(Ok(())) => {} + Ok(Err(e)) => { + if first_error.is_none() { + first_error = Some(e.to_string()); + } + } + Err(join_err) => { + if first_error.is_none() { + first_error = Some(format!("compile task panicked: {join_err}")); + } + } } - }); + } - if let Some(error) = first_error.into_inner().unwrap() { + if let Some(error) = first_error { return Err(FbuildError::BuildFailed(error)); } @@ -274,7 +283,7 @@ pub fn compile_library_with_jobs( all_objects.len(), archive_path.display() ); - archive_objects(ar_path, &all_objects, &archive_path)?; + archive_objects(ar_path, &all_objects, &archive_path).await?; tracing::info!( "compiled library {}: {} changed / {} total files ({} threads) -> {}", @@ -293,7 +302,7 @@ pub fn compile_library_with_jobs( /// On Windows, ALL compiler flags are written to a GCC response file (`@file`) /// to avoid exceeding the 32KB command-line limit (OS error 206). #[allow(clippy::too_many_arguments)] -fn compile_one_source( +async fn compile_one_source( source: &Path, obj_dir: &Path, gcc_path: &Path, @@ -343,7 +352,8 @@ fn compile_one_source( // without expanding them, so this is safe. let args = if cfg!(windows) { let rsp_path = - fbuild_core::response_file::write_response_file(&all_flags, &rsp_dir, "lib_compile")?; + fbuild_core::response_file::write_response_file(&all_flags, &rsp_dir, "lib_compile") + .await?; let rsp_path = invocation_response_file_path(&rsp_path)?; let raw_args = [ compiler.to_string_lossy().to_string(), @@ -360,7 +370,7 @@ fn compile_one_source( }; let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, None, None)?; + let result = run_command(&args_ref, None, None, None).await?; if !result.success() { return Err(FbuildError::BuildFailed(format!( @@ -505,7 +515,7 @@ fn modified_time(metadata: &Metadata) -> Result { /// When there are many include paths (>100), writes a response file. /// Uses `-iprefix` + `-iwithprefixbefore` for paths sharing a common prefix /// to keep the total command line under GCC 8.4.0's 32KB CreateProcess limit. -fn build_include_flags(include_dirs: &[PathBuf], _temp_dir: &Path) -> Result> { +async fn build_include_flags(include_dirs: &[PathBuf], _temp_dir: &Path) -> Result> { let flags: Vec = include_dirs .iter() .map(|d| format!("-I{}", d.display())) @@ -514,7 +524,8 @@ fn build_include_flags(include_dirs: &[PathBuf], _temp_dir: &Path) -> Result 100 { let rsp_dir = _temp_dir.join("tmp"); let rsp_path = - fbuild_core::response_file::write_response_file(&flags, &rsp_dir, "lib_includes")?; + fbuild_core::response_file::write_response_file(&flags, &rsp_dir, "lib_includes") + .await?; Ok(vec![format!("@{}", rsp_path.display())]) } else { Ok(flags) @@ -522,7 +533,7 @@ fn build_include_flags(include_dirs: &[PathBuf], _temp_dir: &Path) -> Result Result<()> { +async fn archive_objects(ar_path: &Path, objects: &[PathBuf], output: &Path) -> Result<()> { if output.exists() { std::fs::remove_file(output)?; } @@ -538,7 +549,7 @@ fn archive_objects(ar_path: &Path, objects: &[PathBuf], output: &Path) -> Result } let args_ref: Vec<&str> = args.iter().map(|s| s.as_str()).collect(); - let result = run_command(&args_ref, None, None, None)?; + let result = run_command(&args_ref, None, None, None).await?; if !result.success() { return Err(FbuildError::BuildFailed(format!( @@ -602,11 +613,11 @@ mod tests { assert_eq!(p.extension().unwrap(), "o"); } - #[test] - fn test_build_include_flags_small() { + #[tokio::test] + async fn test_build_include_flags_small() { let tmp = tempfile::TempDir::new().unwrap(); let dirs = vec![PathBuf::from("/a"), PathBuf::from("/b")]; - let flags = build_include_flags(&dirs, tmp.path()).unwrap(); + let flags = build_include_flags(&dirs, tmp.path()).await.unwrap(); assert_eq!(flags.len(), 2); assert!(flags[0].starts_with("-I")); } diff --git a/crates/fbuild-packages/src/library/library_manager.rs b/crates/fbuild-packages/src/library/library_manager.rs index 561f3ddf..152a8cd8 100644 --- a/crates/fbuild-packages/src/library/library_manager.rs +++ b/crates/fbuild-packages/src/library/library_manager.rs @@ -69,23 +69,24 @@ pub async fn ensure_libraries( let mut downloaded_names: std::collections::HashSet = std::collections::HashSet::new(); let libs_dir_owned = libs_dir.to_path_buf(); - let mut handles = Vec::new(); + let mut tasks: tokio::task::JoinSet< + std::result::Result<(std::path::PathBuf, String, String), fbuild_core::FbuildError>, + > = tokio::task::JoinSet::new(); for spec in &specs { let spec_clone = spec.clone(); let dir = libs_dir_owned.clone(); - handles.push(tokio::spawn(async move { + tasks.spawn(async move { let lib_dir = library_downloader::download_library(&spec_clone, &dir).await?; - Ok::<(std::path::PathBuf, String, String), fbuild_core::FbuildError>(( + Ok(( lib_dir, spec_clone.sanitized_name(), spec_clone.name.to_lowercase(), )) - })); + }); } - // Collect results (preserving order for determinism) - for handle in handles { - let (lib_dir, sanitized, name_lower) = handle.await.map_err(|e| { + while let Some(joined) = tasks.join_next().await { + let (lib_dir, sanitized, name_lower) = joined.map_err(|e| { fbuild_core::FbuildError::PackageError(format!("library download task failed: {}", e)) })??; installed.push(InstalledLibrary::new(&lib_dir, &sanitized)); @@ -133,7 +134,9 @@ pub async fn ensure_libraries( verbose, jobs, compiler_cache, - )? { + ) + .await? + { archives.push(archive_path); } } @@ -271,7 +274,11 @@ async fn resolve_transitive_deps( Ok(()) } -/// Synchronous wrapper for ensure_libraries. +/// Synchronous wrapper for ensure_libraries (legacy sync call-sites). +/// +/// New code should call the async `ensure_libraries` directly. This bridge +/// stays during the #813 migration so fbuild-build orchestrators that haven't +/// been converted yet can still link. #[allow(clippy::too_many_arguments)] pub fn ensure_libraries_sync( lib_specs: &[String], @@ -287,40 +294,27 @@ pub fn ensure_libraries_sync( jobs: usize, compiler_cache: Option<&Path>, ) -> Result { - let rt = tokio::runtime::Handle::try_current().ok(); - if let Some(handle) = rt { - handle.block_on(ensure_libraries( - lib_specs, - lib_ignore, - gcc_path, - gxx_path, - ar_path, - c_flags, - cpp_flags, - base_includes, - libs_dir, - verbose, - jobs, - compiler_cache, - )) + let fut = ensure_libraries( + lib_specs, + lib_ignore, + gcc_path, + gxx_path, + ar_path, + c_flags, + cpp_flags, + base_includes, + libs_dir, + verbose, + jobs, + compiler_cache, + ); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + tokio::task::block_in_place(|| handle.block_on(fut)) } else { let rt = tokio::runtime::Runtime::new().map_err(|e| { fbuild_core::FbuildError::PackageError(format!("failed to create tokio runtime: {}", e)) })?; - rt.block_on(ensure_libraries( - lib_specs, - lib_ignore, - gcc_path, - gxx_path, - ar_path, - c_flags, - cpp_flags, - base_includes, - libs_dir, - verbose, - jobs, - compiler_cache, - )) + rt.block_on(fut) } } diff --git a/crates/fbuild-packages/src/library/nrf52_core.rs b/crates/fbuild-packages/src/library/nrf52_core.rs index 79725f12..80bcec5b 100644 --- a/crates/fbuild-packages/src/library/nrf52_core.rs +++ b/crates/fbuild-packages/src/library/nrf52_core.rs @@ -126,25 +126,14 @@ impl Nrf52Cores { } } +#[async_trait::async_trait] impl crate::Package for Nrf52Cores { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_core_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/registry.rs b/crates/fbuild-packages/src/library/registry.rs index ed83a09b..320b555c 100644 --- a/crates/fbuild-packages/src/library/registry.rs +++ b/crates/fbuild-packages/src/library/registry.rs @@ -21,7 +21,7 @@ pub struct ResolvedLibrary { /// Returns the owner if found. pub async fn search_library(name: &str) -> Result> { let url = format!("{}/search?query={}", REGISTRY_API_URL, name); - let response = reqwest::get(&url).await.map_err(|e| { + let response = crate::http::client().get(&url).send().await.map_err(|e| { FbuildError::PackageError(format!("registry search failed for {}: {}", name, e)) })?; @@ -74,7 +74,7 @@ pub async fn resolve_library( // Use the package details API which returns all versions let url = format!("{}/packages/{}/library/{}", REGISTRY_API_URL, owner, name); - let response = reqwest::get(&url).await.map_err(|e| { + let response = crate::http::client().get(&url).send().await.map_err(|e| { FbuildError::PackageError(format!( "registry query failed for {}/{}: {}", owner, name, e @@ -186,7 +186,7 @@ pub async fn resolve_library( async fn resolve_library_via_search(owner: &str, name: &str) -> Result { let url = format!("{}/search?query={}", REGISTRY_API_URL, name); - let response = reqwest::get(&url).await.map_err(|e| { + let response = crate::http::client().get(&url).send().await.map_err(|e| { FbuildError::PackageError(format!( "registry search failed for {}/{}: {}", owner, name, e diff --git a/crates/fbuild-packages/src/library/renesas_core.rs b/crates/fbuild-packages/src/library/renesas_core.rs index 4f197cca..3dac918d 100644 --- a/crates/fbuild-packages/src/library/renesas_core.rs +++ b/crates/fbuild-packages/src/library/renesas_core.rs @@ -150,25 +150,14 @@ impl RenesasCores { } } +#[async_trait::async_trait] impl crate::Package for RenesasCores { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_core_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/rp2040_core.rs b/crates/fbuild-packages/src/library/rp2040_core.rs index b5d2a192..12013c60 100644 --- a/crates/fbuild-packages/src/library/rp2040_core.rs +++ b/crates/fbuild-packages/src/library/rp2040_core.rs @@ -150,25 +150,14 @@ impl Rp2040Cores { } } +#[async_trait::async_trait] impl crate::Package for Rp2040Cores { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_core_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/sam_core.rs b/crates/fbuild-packages/src/library/sam_core.rs index 7b11caac..63c36121 100644 --- a/crates/fbuild-packages/src/library/sam_core.rs +++ b/crates/fbuild-packages/src/library/sam_core.rs @@ -105,25 +105,14 @@ impl SamCores { } } +#[async_trait::async_trait] impl crate::Package for SamCores { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_core_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/samd_core.rs b/crates/fbuild-packages/src/library/samd_core.rs index 9621a9f7..62ae0e89 100644 --- a/crates/fbuild-packages/src/library/samd_core.rs +++ b/crates/fbuild-packages/src/library/samd_core.rs @@ -109,25 +109,14 @@ impl SamdCores { } } +#[async_trait::async_trait] impl crate::Package for SamdCores { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_core_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/silabs_core.rs b/crates/fbuild-packages/src/library/silabs_core.rs index 2dcc9eb1..3a07ac10 100644 --- a/crates/fbuild-packages/src/library/silabs_core.rs +++ b/crates/fbuild-packages/src/library/silabs_core.rs @@ -94,29 +94,19 @@ impl SilabsCores { } } +#[async_trait::async_trait] impl crate::Package for SilabsCores { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { let root = self.resolved_dir(); - super::arduino_api::ensure_arduino_api(&root.join("cores").join("silabs"))?; + super::arduino_api::ensure_arduino_api(&root.join("cores").join("silabs")).await?; return Ok(root); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; + let install_path = self.base.staged_install(Self::validate).await?; let root = find_core_root(&install_path); - super::arduino_api::ensure_arduino_api(&root.join("cores").join("silabs"))?; + super::arduino_api::ensure_arduino_api(&root.join("cores").join("silabs")).await?; Ok(root) } diff --git a/crates/fbuild-packages/src/library/stm32_core.rs b/crates/fbuild-packages/src/library/stm32_core.rs index 8b9a7790..8354e7b3 100644 --- a/crates/fbuild-packages/src/library/stm32_core.rs +++ b/crates/fbuild-packages/src/library/stm32_core.rs @@ -118,25 +118,14 @@ impl Stm32Cores { } } +#[async_trait::async_trait] impl crate::Package for Stm32Cores { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_core_root(&install_path)) } diff --git a/crates/fbuild-packages/src/library/teensy_core.rs b/crates/fbuild-packages/src/library/teensy_core.rs index a2cfd5ad..0c785d74 100644 --- a/crates/fbuild-packages/src/library/teensy_core.rs +++ b/crates/fbuild-packages/src/library/teensy_core.rs @@ -140,25 +140,14 @@ impl TeensyCores { } } +#[async_trait::async_trait] impl crate::Package for TeensyCores { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_core_root(&install_path)) } diff --git a/crates/fbuild-packages/src/lnk/resolver.rs b/crates/fbuild-packages/src/lnk/resolver.rs index 4145ef91..46b8d760 100644 --- a/crates/fbuild-packages/src/lnk/resolver.rs +++ b/crates/fbuild-packages/src/lnk/resolver.rs @@ -56,9 +56,8 @@ impl std::fmt::Debug for ResolvedBlob { /// cache miss → download, verify, record, return path + lease. /// /// The download path runs synchronously by blocking on the existing async -/// downloader via the workspace `block_on_package_future` helper. Callers -/// already on a tokio runtime get `block_in_place`; off-runtime callers -/// get a fresh single-thread runtime. +/// downloader. Callers already on a tokio runtime get `block_in_place`; +/// off-runtime callers get a fresh single-thread runtime. pub fn resolve(lnk: &LnkFile, cache: &DiskCache) -> Result { // Cache lookup uses (Kind, url, version) where "version" is the sha256. // This guarantees that a change to the .lnk's sha256 forces a refetch. @@ -113,8 +112,17 @@ pub fn resolve(lnk: &LnkFile, cache: &DiskCache) -> Result { )) })?; - let downloaded = - crate::block_on_package_future(async { download_file(&lnk.url, &staging_dir).await })?; + let downloaded = { + let fut = async { download_file(&lnk.url, &staging_dir).await }; + if let Ok(handle) = tokio::runtime::Handle::try_current() { + tokio::task::block_in_place(|| handle.block_on(fut))? + } else { + let rt = tokio::runtime::Runtime::new().map_err(|e| { + FbuildError::PackageError(format!("failed to create tokio runtime: {}", e)) + })?; + rt.block_on(fut)? + } + }; verify_sha256(&downloaded, &lnk.sha256).map_err(|e| { // Clean up the staging file so a retry starts fresh. diff --git a/crates/fbuild-packages/src/toolchain/arm.rs b/crates/fbuild-packages/src/toolchain/arm.rs index 665f6c31..b502da03 100644 --- a/crates/fbuild-packages/src/toolchain/arm.rs +++ b/crates/fbuild-packages/src/toolchain/arm.rs @@ -95,25 +95,14 @@ impl ArmToolchain { } } +#[async_trait::async_trait] impl crate::Package for ArmToolchain { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_bin_root(&install_path)) } diff --git a/crates/fbuild-packages/src/toolchain/arm_gcc8.rs b/crates/fbuild-packages/src/toolchain/arm_gcc8.rs index 59c0a3b8..42d59ffc 100644 --- a/crates/fbuild-packages/src/toolchain/arm_gcc8.rs +++ b/crates/fbuild-packages/src/toolchain/arm_gcc8.rs @@ -63,25 +63,14 @@ impl ArmGcc8Toolchain { } } +#[async_trait::async_trait] impl crate::Package for ArmGcc8Toolchain { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_bin_root(&install_path)) } diff --git a/crates/fbuild-packages/src/toolchain/avr.rs b/crates/fbuild-packages/src/toolchain/avr.rs index 3ee701f9..fd4d2e30 100644 --- a/crates/fbuild-packages/src/toolchain/avr.rs +++ b/crates/fbuild-packages/src/toolchain/avr.rs @@ -103,26 +103,14 @@ impl AvrToolchain { } } +#[async_trait::async_trait] impl crate::Package for AvrToolchain { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - // Use tokio runtime for async staged_install - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_bin_root(&install_path)) } @@ -359,10 +347,14 @@ mod tests { let tmp = tempfile::TempDir::new().unwrap(); let out_path = tmp.path().join("avr-gcc.archive"); - // Blocking download + // Blocking download (test uses standalone runtime since it's a sync #[test]). let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async { - let resp = reqwest::get(&url).await.expect("download failed"); + let resp = crate::http::client() + .get(&url) + .send() + .await + .expect("download failed"); assert!(resp.status().is_success(), "HTTP {}", resp.status()); let bytes = resp.bytes().await.expect("read body failed"); std::fs::write(&out_path, &bytes).expect("write failed"); diff --git a/crates/fbuild-packages/src/toolchain/clang.rs b/crates/fbuild-packages/src/toolchain/clang.rs index b8902348..0f8f7380 100644 --- a/crates/fbuild-packages/src/toolchain/clang.rs +++ b/crates/fbuild-packages/src/toolchain/clang.rs @@ -144,7 +144,7 @@ impl ClangComponent { async fn fetch_manifest(&self) -> fbuild_core::Result { let url = self.manifest_url(); - let resp = reqwest::get(&url).await.map_err(|e| { + let resp = crate::http::client().get(&url).send().await.map_err(|e| { fbuild_core::FbuildError::Other(format!( "failed to fetch {} manifest from {}: {}", self.kind.component_name(), diff --git a/crates/fbuild-packages/src/toolchain/esp32.rs b/crates/fbuild-packages/src/toolchain/esp32.rs index 5e67cbe0..c552a4eb 100644 --- a/crates/fbuild-packages/src/toolchain/esp32.rs +++ b/crates/fbuild-packages/src/toolchain/esp32.rs @@ -159,17 +159,18 @@ impl Esp32Toolchain { } } +#[async_trait::async_trait] impl crate::Package for Esp32Toolchain { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } let prefix = self.prefix.clone(); - let install_path = crate::block_on_package_future( - self.base - .staged_install(|dir| Self::validate_install(dir, &prefix)), - )?; + let install_path = self + .base + .staged_install(|dir| Self::validate_install(dir, &prefix)) + .await?; Ok(find_bin_root(&install_path)) } diff --git a/crates/fbuild-packages/src/toolchain/esp32_metadata.rs b/crates/fbuild-packages/src/toolchain/esp32_metadata.rs index e0d83f6d..8047db19 100644 --- a/crates/fbuild-packages/src/toolchain/esp32_metadata.rs +++ b/crates/fbuild-packages/src/toolchain/esp32_metadata.rs @@ -71,16 +71,24 @@ pub async fn resolve_toolchain_url( } /// Synchronous wrapper for resolve_toolchain_url. +/// +/// Bridges sync call-sites (legacy orchestrators not yet on the async API) +/// into the now-async resolver. New code should `.await resolve_toolchain_url` +/// directly. pub fn resolve_toolchain_url_sync( metadata_url: &str, toolchain_name: &str, cache_dir: &Path, ) -> Result { - crate::block_on_package_future(resolve_toolchain_url( - metadata_url, - toolchain_name, - cache_dir, - )) + let fut = resolve_toolchain_url(metadata_url, toolchain_name, cache_dir); + if let Ok(handle) = tokio::runtime::Handle::try_current() { + tokio::task::block_in_place(|| handle.block_on(fut)) + } else { + let rt = tokio::runtime::Runtime::new().map_err(|e| { + FbuildError::PackageError(format!("failed to create tokio runtime: {}", e)) + })?; + rt.block_on(fut) + } } /// Find tools.json in a directory (may be at root or one level deep). diff --git a/crates/fbuild-packages/src/toolchain/esp8266.rs b/crates/fbuild-packages/src/toolchain/esp8266.rs index 65a9f0c7..8fafd3b9 100644 --- a/crates/fbuild-packages/src/toolchain/esp8266.rs +++ b/crates/fbuild-packages/src/toolchain/esp8266.rs @@ -87,25 +87,14 @@ impl Esp8266Toolchain { } } +#[async_trait::async_trait] impl crate::Package for Esp8266Toolchain { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_bin_root(&install_path)) } diff --git a/crates/fbuild-packages/src/toolchain/esp_qemu.rs b/crates/fbuild-packages/src/toolchain/esp_qemu.rs index 7b87a167..d4b9f3af 100644 --- a/crates/fbuild-packages/src/toolchain/esp_qemu.rs +++ b/crates/fbuild-packages/src/toolchain/esp_qemu.rs @@ -113,7 +113,7 @@ impl EspQemu { self.arch } - pub fn resolve_executable(&self) -> Result { + pub async fn resolve_executable(&self) -> Result { if let Ok(raw) = std::env::var(self.arch.env_var()) { let path = PathBuf::from(raw); let path = validate_qemu_path(path, self.arch.env_var())?; @@ -141,7 +141,7 @@ impl EspQemu { return Ok(path); } - let _ = self.ensure_installed()?; + let _ = self.ensure_installed().await?; let path = find_qemu_binary(&self.base.install_path(), self.arch)?; hydrate_windows_runtime(&path)?; validate_windows_runtime(&path)?; @@ -159,8 +159,9 @@ impl EspQemu { } } +#[async_trait::async_trait] impl Package for EspQemu { - fn ensure_installed(&self) -> Result { + async fn ensure_installed(&self) -> Result { if self.is_installed() { return qemu_root(&self.base.install_path(), self.arch); } @@ -169,7 +170,7 @@ impl Package for EspQemu { EspQemuArch::Xtensa => Self::validate_install_xtensa, EspQemuArch::Riscv32 => Self::validate_install_riscv32, }; - let install_path = crate::block_on_package_future(self.base.staged_install(validate))?; + let install_path = self.base.staged_install(validate).await?; qemu_root(&install_path, self.arch) } @@ -194,14 +195,15 @@ impl EspQemuXtensa { Ok(Self(EspQemu::new(project_dir, EspQemuArch::Xtensa)?)) } - pub fn resolve_executable(&self) -> Result { - self.0.resolve_executable() + pub async fn resolve_executable(&self) -> Result { + self.0.resolve_executable().await } } +#[async_trait::async_trait] impl Package for EspQemuXtensa { - fn ensure_installed(&self) -> Result { - self.0.ensure_installed() + async fn ensure_installed(&self) -> Result { + self.0.ensure_installed().await } fn is_installed(&self) -> bool { @@ -221,14 +223,15 @@ impl EspQemuRiscv32 { Ok(Self(EspQemu::new(project_dir, EspQemuArch::Riscv32)?)) } - pub fn resolve_executable(&self) -> Result { - self.0.resolve_executable() + pub async fn resolve_executable(&self) -> Result { + self.0.resolve_executable().await } } +#[async_trait::async_trait] impl Package for EspQemuRiscv32 { - fn ensure_installed(&self) -> Result { - self.0.ensure_installed() + async fn ensure_installed(&self) -> Result { + self.0.ensure_installed().await } fn is_installed(&self) -> bool { diff --git a/crates/fbuild-packages/src/toolchain/riscv.rs b/crates/fbuild-packages/src/toolchain/riscv.rs index ecc0e405..9870a75c 100644 --- a/crates/fbuild-packages/src/toolchain/riscv.rs +++ b/crates/fbuild-packages/src/toolchain/riscv.rs @@ -172,25 +172,14 @@ impl RiscvToolchain { } } +#[async_trait::async_trait] impl crate::Package for RiscvToolchain { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_bin_root(&install_path)) } diff --git a/crates/fbuild-packages/src/toolchain/rp2040_pqt.rs b/crates/fbuild-packages/src/toolchain/rp2040_pqt.rs index b885f301..16edf5ea 100644 --- a/crates/fbuild-packages/src/toolchain/rp2040_pqt.rs +++ b/crates/fbuild-packages/src/toolchain/rp2040_pqt.rs @@ -93,25 +93,14 @@ impl Rp2040PqtToolchain { } } +#[async_trait::async_trait] impl crate::Package for Rp2040PqtToolchain { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_bin_root(&install_path)) } diff --git a/crates/fbuild-packages/src/toolchain/teensy_arm.rs b/crates/fbuild-packages/src/toolchain/teensy_arm.rs index 29c903fe..6685f5ae 100644 --- a/crates/fbuild-packages/src/toolchain/teensy_arm.rs +++ b/crates/fbuild-packages/src/toolchain/teensy_arm.rs @@ -86,25 +86,14 @@ impl TeensyArmToolchain { } } +#[async_trait::async_trait] impl crate::Package for TeensyArmToolchain { - fn ensure_installed(&self) -> fbuild_core::Result { + async fn ensure_installed(&self) -> fbuild_core::Result { if self.is_installed() { return Ok(self.resolved_dir()); } - let rt = tokio::runtime::Handle::try_current().ok(); - let install_path = if let Some(handle) = rt { - handle.block_on(self.base.staged_install(Self::validate))? - } else { - let rt = tokio::runtime::Runtime::new().map_err(|e| { - fbuild_core::FbuildError::PackageError(format!( - "failed to create tokio runtime: {}", - e - )) - })?; - rt.block_on(self.base.staged_install(Self::validate))? - }; - + let install_path = self.base.staged_install(Self::validate).await?; Ok(find_bin_root(&install_path)) } diff --git a/crates/fbuild-serial/src/crash_decoder.rs b/crates/fbuild-serial/src/crash_decoder.rs index 9801e702..501338a9 100644 --- a/crates/fbuild-serial/src/crash_decoder.rs +++ b/crates/fbuild-serial/src/crash_decoder.rs @@ -128,7 +128,7 @@ impl CrashDecoder { /// /// Returns `Some(lines)` when a crash dump has been fully decoded, /// or `None` if no output is ready yet. - pub fn process_line(&mut self, line: &str) -> Option> { + pub async fn process_line(&mut self, line: &str) -> Option> { match self.state { DecoderState::Idle => { if Self::detect_crash_start(line) { @@ -138,7 +138,7 @@ impl CrashDecoder { } DecoderState::Accumulating => { if self.detect_crash_end(line) { - let decoded = self.decode(); + let decoded = self.decode().await; self.reset(); if decoded.is_empty() { None @@ -188,7 +188,7 @@ impl CrashDecoder { } /// Decode the buffered crash dump using addr2line. - fn decode(&mut self) -> Vec { + async fn decode(&mut self) -> Vec { if self.buffer.is_empty() { return Vec::new(); } @@ -225,7 +225,7 @@ impl CrashDecoder { } // Run addr2line - Self::run_addr2line(&addr2line_path, &elf_path, &addresses) + Self::run_addr2line(&addr2line_path, &elf_path, &addresses).await } /// Clear the accumulator for the next crash. @@ -312,7 +312,11 @@ impl CrashDecoder { } /// Run addr2line on the extracted addresses. - fn run_addr2line(addr2line_path: &Path, elf_path: &Path, addresses: &[String]) -> Vec { + async fn run_addr2line( + addr2line_path: &Path, + elf_path: &Path, + addresses: &[String], + ) -> Vec { let addr2line_str = addr2line_path.to_string_lossy(); let elf_str = elf_path.to_string_lossy(); @@ -321,7 +325,7 @@ impl CrashDecoder { args.push(addr.as_str()); } - let result = match run_command(&args, None, None, Some(ADDR2LINE_TIMEOUT)) { + let result = match run_command(&args, None, None, Some(ADDR2LINE_TIMEOUT)).await { Ok(output) => output, Err(fbuild_core::FbuildError::Timeout(_)) => { tracing::warn!("addr2line timed out after {}s", ADDR2LINE_TIMEOUT.as_secs()); @@ -521,71 +525,82 @@ mod tests { // --- process_line state machine --- - #[test] - fn process_line_normal_line() { + #[tokio::test] + async fn process_line_normal_line() { let mut decoder = CrashDecoder::new(None, None); - assert!(decoder.process_line("Hello from ESP32!").is_none()); + assert!(decoder.process_line("Hello from ESP32!").await.is_none()); assert!(!decoder.is_accumulating()); } - #[test] - fn process_line_crash_start_begins_accumulating() { + #[tokio::test] + async fn process_line_crash_start_begins_accumulating() { let mut decoder = CrashDecoder::new(None, None); assert!(decoder .process_line("Guru Meditation Error: Core 0 panic'ed (LoadProhibited)") + .await .is_none()); assert!(decoder.is_accumulating()); } - #[test] - fn process_line_accumulates_then_ends() { + #[tokio::test] + async fn process_line_accumulates_then_ends() { let mut decoder = CrashDecoder::new(None, None); // Start - decoder.process_line("Guru Meditation Error: Core 0 panic'ed (LoadProhibited)"); + decoder + .process_line("Guru Meditation Error: Core 0 panic'ed (LoadProhibited)") + .await; assert!(decoder.is_accumulating()); // Middle lines - decoder.process_line("Core 0 register dump:"); - decoder.process_line("Backtrace: 0x42002a3c:0x3fc90000"); + decoder.process_line("Core 0 register dump:").await; + decoder.process_line("Backtrace: 0x42002a3c:0x3fc90000").await; // End — should produce output (warning about no elf) - let result = decoder.process_line("Rebooting..."); + let result = decoder.process_line("Rebooting...").await; assert!(result.is_some()); assert!(!decoder.is_accumulating()); } - #[test] - fn process_line_no_elf_warns_once() { + #[tokio::test] + async fn process_line_no_elf_warns_once() { let mut decoder = CrashDecoder::new(None, None); // First crash — should warn - decoder.process_line("abort() was called at PC 0x42002a3c"); - let result = decoder.process_line("Rebooting..."); + decoder + .process_line("abort() was called at PC 0x42002a3c") + .await; + let result = decoder.process_line("Rebooting...").await; let lines = result.unwrap(); assert!(lines[0].contains("no firmware.elf found")); // Second crash — should not produce output - decoder.process_line("abort() was called at PC 0x42002a3c"); - let result = decoder.process_line("Rebooting..."); + decoder + .process_line("abort() was called at PC 0x42002a3c") + .await; + let result = decoder.process_line("Rebooting...").await; assert!(result.is_none()); } // --- Duplicate debouncing --- - #[test] - fn debounce_identical_crash() { + #[tokio::test] + async fn debounce_identical_crash() { let elf = PathBuf::from("/nonexistent/firmware.elf"); let a2l = PathBuf::from("/nonexistent/addr2line"); let mut decoder = CrashDecoder::new(Some(elf), Some(a2l)); // First crash — will fail to run addr2line but will set the hash - decoder.process_line("abort() was called at PC 0x42002a3c"); - let _result1 = decoder.process_line("Rebooting..."); + decoder + .process_line("abort() was called at PC 0x42002a3c") + .await; + let _result1 = decoder.process_line("Rebooting...").await; // Identical crash immediately after — should be debounced - decoder.process_line("abort() was called at PC 0x42002a3c"); - let result2 = decoder.process_line("Rebooting..."); + decoder + .process_line("abort() was called at PC 0x42002a3c") + .await; + let result2 = decoder.process_line("Rebooting...").await; let lines = result2.unwrap(); assert!(lines[0].contains("duplicate within debounce window")); } @@ -649,8 +664,8 @@ mod tests { // --- Full Xtensa crash dump --- - #[test] - fn full_xtensa_crash_dump() { + #[tokio::test] + async fn full_xtensa_crash_dump() { let mut decoder = CrashDecoder::new(None, None); let lines = [ @@ -666,7 +681,7 @@ mod tests { let mut result = None; for line in &lines { - if let Some(r) = decoder.process_line(line) { + if let Some(r) = decoder.process_line(line).await { result = Some(r); } } @@ -678,8 +693,8 @@ mod tests { // --- Full RISC-V crash dump --- - #[test] - fn full_riscv_crash_dump() { + #[tokio::test] + async fn full_riscv_crash_dump() { let mut decoder = CrashDecoder::new(None, None); let lines = [ @@ -696,7 +711,7 @@ mod tests { let mut result = None; for line in &lines { - if let Some(r) = decoder.process_line(line) { + if let Some(r) = decoder.process_line(line).await { result = Some(r); } } @@ -710,9 +725,9 @@ mod tests { /// Integration test: feed a real ESP32-S3 Xtensa crash dump through the /// decoder with a real ELF + addr2line, verify we get `deliberate_crash` /// in the decoded output — matching the Python fbuild decoder. - #[test] + #[tokio::test] #[ignore] // requires build artifacts from esp32s3-crash-test - fn real_esp32s3_crash_decode() { + async fn real_esp32s3_crash_decode() { let elf = PathBuf::from(std::env::var("ESP32S3_CRASH_ELF").unwrap_or_else(|_| { format!( "{}/.pio/build/esp32s3-crash-test/firmware.elf", @@ -766,7 +781,7 @@ mod tests { let mut decoded_output = None; for line in &crash_lines { - if let Some(output) = decoder.process_line(line) { + if let Some(output) = decoder.process_line(line).await { decoded_output = Some(output); } } diff --git a/crates/fbuild-serial/src/manager.rs b/crates/fbuild-serial/src/manager.rs index d836f8ee..a17c3375 100644 --- a/crates/fbuild-serial/src/manager.rs +++ b/crates/fbuild-serial/src/manager.rs @@ -799,11 +799,18 @@ impl SharedSerialManager { /// Process a serial line through the crash decoder for a port. /// /// Returns decoded crash trace lines if a crash dump just completed. - pub fn process_crash_line(&self, port: &str, line: &str) -> Option> { + /// + /// The decoder is temporarily removed from the DashMap so the shard + /// lock isn't held across the `addr2line` `.await`. Re-inserted on + /// completion; the brief race window is acceptable because the + /// caller (the serial-monitor reader task) is the only producer for + /// a given port. + pub async fn process_crash_line(&self, port: &str, line: &str) -> Option> { let session_key = self.resolve_port_key(port); - self.crash_decoders - .get_mut(&session_key) - .and_then(|mut decoder| decoder.process_line(line)) + let (key, mut decoder) = self.crash_decoders.remove(&session_key)?; + let result = decoder.process_line(line).await; + self.crash_decoders.insert(key, decoder); + result } /// Get a snapshot of all active serial port sessions for lock/status reporting.