From 40d9c9ea9c5aff55cfa2914133668dc2cdad1b9a Mon Sep 17 00:00:00 2001 From: zackees Date: Tue, 30 Jun 2026 17:27:13 -0700 Subject: [PATCH 1/4] publish fbuild usb vidpid overlay --- .github/workflows/dylint.yml | 6 +- .github/workflows/loc-gate.yml | 42 ++- .github/workflows/template_build.yml | 5 + .github/workflows/update-data.yml | 16 +- ci/validate_boards.py | 11 + .../fbuild-build/src/apollo3/orchestrator.rs | 6 +- crates/fbuild-build/src/build_info.rs | 23 +- .../fbuild-build/src/ch32v/ch32v_compiler.rs | 11 +- crates/fbuild-build/src/ch32v/orchestrator.rs | 60 +++- crates/fbuild-build/src/compile_backend.rs | 2 +- crates/fbuild-build/src/compiler.rs | 4 +- crates/fbuild-build/src/compiler_tests.rs | 17 +- crates/fbuild-build/src/pipeline/library.rs | 21 +- .../src/renesas/renesas_compiler.rs | 11 +- crates/fbuild-build/src/sam/orchestrator.rs | 4 +- crates/fbuild-build/src/source_scanner.rs | 10 +- .../fbuild-build/src/symbol_analyzer/mod.rs | 8 +- crates/fbuild-build/src/zccache_embedded.rs | 2 +- crates/fbuild-build/tests/stm32_acceptance.rs | 11 +- .../fbuild-build/tests/teensy30_acceptance.rs | 20 +- .../fbuild-build/tests/teensylc_acceptance.rs | 20 +- crates/fbuild-cli/src/cli/build.rs | 14 +- crates/fbuild-cli/src/cli/clang_tools.rs | 60 ++-- crates/fbuild-cli/src/cli/clangd_config.rs | 16 +- crates/fbuild-cli/src/cli/compile_many.rs | 27 +- crates/fbuild-cli/src/cli/daemon_cmd.rs | 5 +- crates/fbuild-cli/src/cli/device.rs | 48 +++ crates/fbuild-cli/src/cli/dispatch.rs | 4 +- crates/fbuild-cli/src/cli/port_scan.rs | 91 ++++- crates/fbuild-cli/src/cli/purge.rs | 4 +- crates/fbuild-cli/src/cli/serial_probe.rs | 6 +- crates/fbuild-cli/src/cli/show.rs | 5 +- crates/fbuild-cli/src/cli/sync_cmd.rs | 4 +- crates/fbuild-cli/src/cli/tests.rs | 18 +- crates/fbuild-cli/src/daemon_client/types.rs | 12 + crates/fbuild-cli/src/sync/lockfile.rs | 30 +- crates/fbuild-cli/src/sync/mod.rs | 44 ++- crates/fbuild-cli/src/sync/source.rs | 9 +- crates/fbuild-cli/src/update_check.rs | 93 ++++-- crates/fbuild-config/Cargo.toml | 1 + crates/fbuild-config/src/bin/enrich_boards.rs | 17 +- crates/fbuild-config/src/board/tests.rs | 3 +- .../src/board/tests_enriched_json.rs | 3 +- crates/fbuild-config/src/ini_parser/tests.rs | 3 +- crates/fbuild-config/src/sdkconfig.rs | 10 +- crates/fbuild-core/src/fs.rs | 20 +- crates/fbuild-core/src/path.rs | 5 +- crates/fbuild-core/src/subprocess.rs | 22 +- crates/fbuild-core/src/usb/data.rs | 13 +- crates/fbuild-daemon/src/context.rs | 10 +- crates/fbuild-daemon/src/device_manager.rs | 60 +++- .../fbuild-daemon/src/device_manager/tests.rs | 4 + crates/fbuild-daemon/src/handlers/devices.rs | 7 + .../src/handlers/emulator/select.rs | 26 +- .../src/handlers/operations/build.rs | 26 +- .../src/handlers/operations/deploy.rs | 50 +-- .../src/handlers/operations/deploy_port.rs | 1 + .../fbuild-daemon/src/handlers/websockets.rs | 5 +- crates/fbuild-daemon/src/main.rs | 18 +- crates/fbuild-daemon/src/models.rs | 10 + crates/fbuild-deploy/src/teensy/usb_type.rs | 10 +- crates/fbuild-header-scan/Cargo.toml | 1 + crates/fbuild-header-scan/src/walker.rs | 20 +- crates/fbuild-library-select/Cargo.toml | 1 + crates/fbuild-library-select/src/cache.rs | 16 +- crates/fbuild-library-select/src/lib.rs | 24 +- crates/fbuild-packages/src/cache.rs | 38 ++- crates/fbuild-packages/src/downloader.rs | 13 +- crates/fbuild-packages/src/lib.rs | 2 +- .../fbuild-packages/src/library/ch32v_core.rs | 4 +- .../src/library/library_manager.rs | 5 +- crates/fbuild-packages/src/lnk/resolver.rs | 5 +- .../src/toolchain/esp32_metadata.rs | 5 +- crates/fbuild-packages/tests/lnk_e2e.rs | 311 +++++++++--------- crates/fbuild-paths/src/lib.rs | 12 +- crates/fbuild-paths/src/running_process.rs | 50 ++- crates/fbuild-python/src/daemon.rs | 13 +- crates/fbuild-serial/src/boards.rs | 20 +- crates/fbuild-serial/src/port_class.rs | 28 +- crates/fbuild-serial/src/preemption.rs | 11 +- crates/fbuild-test-support/Cargo.toml | 1 + crates/fbuild-test-support/src/elf_probe.rs | 3 +- .../fbuild-test-support/src/mini_framework.rs | 15 +- docs/online-data.md | 28 +- .../ban_env_var_set_after_import/src/lib.rs | 41 +-- dylints/ban_poison_panic/src/lib.rs | 41 +-- .../ban_runtime_new_outside_main/src/lib.rs | 41 +-- dylints/ban_std_fs_in_async/src/allowlist.txt | 7 + .../ban_std_sync_mutex_in_async/src/lib.rs | 44 +-- dylints/ban_unrooted_tempdir/src/lib.rs | 20 ++ dylints/ban_unwrap_in_production/src/lib.rs | 47 ++- .../src/lib.rs | 27 +- .../src/lib.rs | 41 +-- online-data-tools/README.md | 9 + online-data-tools/build_usb_vid_proto.py | 203 ++++++++++++ online-data-tools/test_usb_vid_proto.py | 175 ++++++++++ 96 files changed, 1632 insertions(+), 814 deletions(-) create mode 100644 online-data-tools/build_usb_vid_proto.py create mode 100644 online-data-tools/test_usb_vid_proto.py diff --git a/.github/workflows/dylint.yml b/.github/workflows/dylint.yml index 972a414b..eaf9af96 100644 --- a/.github/workflows/dylint.yml +++ b/.github/workflows/dylint.yml @@ -71,7 +71,9 @@ jobs: run: | export PATH="${CARGO_HOME}/bin:${PATH}" set +e - for attempt in 1 2 3; do + lint_count="$(find dylints -mindepth 2 -maxdepth 2 -name Cargo.toml | wc -l | tr -d ' ')" + max_attempts=$((lint_count + 2)) + for attempt in $(seq 1 "$max_attempts"); do cargo dylint --all -- --workspace --all-targets rc=$? if [ $rc -eq 0 ]; then @@ -102,5 +104,5 @@ jobs: fi echo "Created @ aliases; retrying cargo dylint (attempt $((attempt+1)))..." done - echo "::error::cargo dylint did not succeed after 3 attempts" + echo "::error::cargo dylint did not succeed after ${max_attempts} attempts" exit 1 diff --git a/.github/workflows/loc-gate.yml b/.github/workflows/loc-gate.yml index febf8d84..aff7cf39 100644 --- a/.github/workflows/loc-gate.yml +++ b/.github/workflows/loc-gate.yml @@ -13,28 +13,54 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@v4 + with: + fetch-depth: 0 - name: Check .rs file line counts shell: bash + env: + BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.before || '' }} run: | set -euo pipefail THRESHOLD=1000 + ZERO_SHA=0000000000000000000000000000000000000000 + base_ref="${BASE_SHA:-}" + if [[ -n "$base_ref" && "$base_ref" != "$ZERO_SHA" ]]; then + git fetch --no-tags --depth=1 origin "$base_ref" || true + else + base_ref="" + fi + + base_line_count() { + local file="$1" + if [[ -z "$base_ref" ]] || ! git cat-file -e "${base_ref}:${file}" 2>/dev/null; then + echo "" + return + fi + git show "${base_ref}:${file}" | wc -l | tr -d ' ' + } + violations=0 while IFS= read -r -d '' file; do - lines=$(wc -l < "$file") + rel="${file#./}" + lines=$(wc -l < "$file" | tr -d ' ') if [ "$lines" -gt "$THRESHOLD" ]; then + base_lines="$(base_line_count "$rel")" + if [[ -n "$base_lines" && "$base_lines" -gt "$THRESHOLD" ]]; then + echo "BASELINE ${lines} ${file} (base already has ${base_lines} LOC)" + continue + fi echo "::error file=${file}::${file} has ${lines} LOC (limit: ${THRESHOLD})" echo "FAIL ${lines} ${file}" violations=$((violations + 1)) fi - done < <(find . -type f -name '*.rs' \ - -not -path './target/*' \ - -not -path './.git/*' \ - -print0) + done < <(find . \ + \( -path './target' -o -path './.git' \) -prune -o \ + -type f -name '*.rs' -print0) if [ "$violations" -gt 0 ]; then echo "" - echo "Found ${violations} .rs file(s) over ${THRESHOLD} LOC." - echo "Split them into smaller modules before merging." + echo "Found ${violations} new .rs file(s) over ${THRESHOLD} LOC." + echo "Split new large files into smaller modules before merging." exit 1 fi - echo "All .rs files are within the ${THRESHOLD} LOC limit." + echo "No newly over-limit .rs files found." diff --git a/.github/workflows/template_build.yml b/.github/workflows/template_build.yml index 49049176..2fd3e297 100644 --- a/.github/workflows/template_build.yml +++ b/.github/workflows/template_build.yml @@ -24,6 +24,11 @@ env: CARGO_TERM_COLOR: always RUSTFLAGS: "-D warnings" FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + # setup-soldr's default 5 GiB target free-space guard is too conservative + # for hosted per-board Linux builds after the runner image and caches load. + # A 2 GiB hard stop still catches genuinely exhausted runners without + # failing healthy jobs before the board build starts. + SOLDR_TARGET_BLOCK_FREE_GB: "2" jobs: build: diff --git a/.github/workflows/update-data.yml b/.github/workflows/update-data.yml index f0688dc8..6352ce4a 100644 --- a/.github/workflows/update-data.yml +++ b/.github/workflows/update-data.yml @@ -25,7 +25,7 @@ # single failure is non-fatal — the merger downstream sees only the # sources that actually arrived intact; # 4. runs the USB-VID merger → sorted `usb-vid.json` + conflict log + -# per-dataset manifest fragment; +# per-dataset manifest fragment + compact runtime `usb-vids.proto.zstd`; # 5. runs the PlatformIO board merger → `pio-boards.json` (deep-union # with the previously committed copy so transient field drops in # `pio boards` don't lose data) + per-dataset manifest fragment; @@ -538,6 +538,20 @@ jobs: --out "${ONLINE_WORKTREE}/data/usb-vendors.tar.zst" ls -la "${ONLINE_WORKTREE}/data/usb-vendors.tar.zst" + - name: Package usb-vids.proto.zstd (runtime USB VID:PID overlay) + id: package-usb-proto + if: steps.merge-usb.outcome == 'success' + # Full product-level runtime cache for fbuild CLI/device scans. This + # is derived after source priority has already been resolved into + # usb-vid.json; weak third-party rows only fill gaps left by + # first-party/vendor/generic sources and never override their winners. + run: | + uv run --no-project --script \ + "${{ github.workspace }}/online-data-tools/build_usb_vid_proto.py" \ + --upstream "${ONLINE_WORKTREE}/data/usb-vid.json" \ + --out "${ONLINE_WORKTREE}/data/usb-vids.proto.zstd" + ls -la "${ONLINE_WORKTREE}/data/usb-vids.proto.zstd" + - name: Assemble manifest.json id: build-manifest # We rebuild the manifest whenever at least one dataset succeeded, diff --git a/ci/validate_boards.py b/ci/validate_boards.py index e259f8da..f4fbdd0a 100644 --- a/ci/validate_boards.py +++ b/ci/validate_boards.py @@ -84,6 +84,16 @@ } ) +# Intentional local corrections where fbuild's checked-in board asset is more +# specific than the current PlatformIO registry row. Keep this narrow: it is +# only for board-local facts that are also asserted by fbuild tests. +FBUILD_BUILD_FIELD_OVERRIDES = { + # FastLED/fbuild#905 split Unexpected Maker TinyS3 and FeatherS3 USB PIDs. + # PlatformIO 6.13.0 still reports FeatherS3 as 303A:80D0, which collides + # with TinyS3. The local board asset and board::tests_usb_vid pin 80D6. + "um_feathers3": {"pid": "0x80D6"}, +} + MEGATINYCORE_EXTRA_FLAGS = ( "-DCLOCK_SOURCE=0", '-DMEGATINYCORE="2.6.11"', @@ -253,6 +263,7 @@ def validate_board(board_path: Path, pio_dir: Path) -> list[str] | None: pio_build = pio_board.get("build", {}) if isinstance(pio_build, dict): expected_build = extract_build(pio_build) + expected_build.update(FBUILD_BUILD_FIELD_OVERRIDES.get(board_id, {})) actual_build = board.get("build", {}) # Strip intentional fbuild-only extensions from the actual side so # they aren't reported as drift (see FBUILD_EXTENSION_BUILD_FIELDS). diff --git a/crates/fbuild-build/src/apollo3/orchestrator.rs b/crates/fbuild-build/src/apollo3/orchestrator.rs index 7836ba23..4e35136d 100644 --- a/crates/fbuild-build/src/apollo3/orchestrator.rs +++ b/crates/fbuild-build/src/apollo3/orchestrator.rs @@ -1,4 +1,4 @@ -//! Apollo3 build orchestrator — wires together config, packages, compiler, linker. +//! Apollo3 build orchestrator — wires together config, packages, compiler, linker. //! //! Build phases: //! 1. Parse platformio.ini @@ -67,7 +67,9 @@ impl BuildOrchestrator for Apollo3Orchestrator { crate::package_override::resolve_override(env, "framework-arduinoambiqapollo3") }); let framework = match __ovr { - Some(o) => fbuild_packages::library::Apollo3Cores::with_override(¶ms.project_dir, o), + Some(o) => { + fbuild_packages::library::Apollo3Cores::with_override(¶ms.project_dir, o) + } None => fbuild_packages::library::Apollo3Cores::new(¶ms.project_dir), }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; diff --git a/crates/fbuild-build/src/build_info.rs b/crates/fbuild-build/src/build_info.rs index 4bebb805..deac2734 100644 --- a/crates/fbuild-build/src/build_info.rs +++ b/crates/fbuild-build/src/build_info.rs @@ -293,18 +293,17 @@ pub fn emit_build_info(project_dir: &Path, env_name: &str, info: &BuildInfo) -> // pipeline code, but always under the daemon's tokio runtime. // Bridge to the async `write_atomic` via `block_in_place`. Same // pattern as `fbuild_packages::toolchain::esp32_metadata`. - let write_res = - if let Ok(handle) = tokio::runtime::Handle::try_current() { - tokio::task::block_in_place(|| { - handle.block_on(fbuild_core::fs::write_atomic(path, json.as_bytes())) - }) - } else { - // No runtime — happens in unit tests of this module. - // Fall back to plain `std::fs::write`; the integration - // path always has a runtime so the atomic guarantee is - // preserved where it matters. - std::fs::write(path, &json) - }; + let write_res = if let Ok(handle) = tokio::runtime::Handle::try_current() { + tokio::task::block_in_place(|| { + handle.block_on(fbuild_core::fs::write_atomic(path, json.as_bytes())) + }) + } else { + // No runtime — happens in unit tests of this module. + // Fall back to plain `std::fs::write`; the integration + // path always has a runtime so the atomic guarantee is + // preserved where it matters. + std::fs::write(path, &json) + }; if let Err(e) = write_res { tracing::warn!("failed to write {}: {}", path.display(), e); } diff --git a/crates/fbuild-build/src/ch32v/ch32v_compiler.rs b/crates/fbuild-build/src/ch32v/ch32v_compiler.rs index 5fa4d9a1..5e710c76 100644 --- a/crates/fbuild-build/src/ch32v/ch32v_compiler.rs +++ b/crates/fbuild-build/src/ch32v/ch32v_compiler.rs @@ -6,6 +6,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; +use fbuild_core::path::NormalizedPath; use fbuild_core::{BuildProfile, Result}; use super::mcu_config::Ch32vMcuConfig; @@ -101,13 +102,9 @@ impl Ch32vCompiler { let Some(root) = self.framework_root.as_ref() else { return false; }; - match ( - std::fs::canonicalize(source).ok(), - std::fs::canonicalize(root).ok(), - ) { - (Some(s), Some(r)) => s.starts_with(&r), - _ => source.starts_with(root), - } + let source = NormalizedPath::new(source).into_path_buf(); + let root = NormalizedPath::new(root).into_path_buf(); + source.starts_with(&root) } } diff --git a/crates/fbuild-build/src/ch32v/orchestrator.rs b/crates/fbuild-build/src/ch32v/orchestrator.rs index b33b0f39..2723cfcc 100644 --- a/crates/fbuild-build/src/ch32v/orchestrator.rs +++ b/crates/fbuild-build/src/ch32v/orchestrator.rs @@ -300,6 +300,20 @@ fn resolve_variant_dir( requested_variant: &str, system_series: &str, ) -> PathBuf { + // OpenWCH d767162 adds CH32VM00X/CH32V006K8, but that variant currently + // references PB_* pins not defined by the CH32V006 build. Before that + // upstream directory existed, fbuild built this board with the CH32V003F4 + // family fallback; keep that known-good path for the pinned core. + if requested_variant == "CH32VM00X/CH32V006K8" { + let fallback = framework_dir + .join("variants") + .join("CH32V00x") + .join("CH32V003F4"); + if fallback.is_dir() { + return fallback; + } + } + let requested = framework_dir.join("variants").join(requested_variant); if requested.is_dir() { return requested; @@ -333,13 +347,14 @@ fn resolve_variant_dir( } /// Map a series name to the system directory name in the OpenWCH core. -/// e.g. "ch32v003" -> "CH32V00x", "ch32v103" -> "CH32V10x", "ch32x035" -> "CH32X035" +/// e.g. "ch32v003" -> "CH32V00x", "ch32l103" -> "CH32L10x", "ch32x035" -> "CH32X035" fn series_to_system_dir(series: &str) -> String { let upper = series.to_uppercase(); if upper.len() >= 7 { - // CH32V series use the "replace last digit with x" pattern (CH32V00x, CH32V10x, etc.) - // CH32X/CH32L series use the exact uppercase name (CH32X035, CH32L103, etc.) - if upper.starts_with("CH32V") { + // CH32V/CH32L series use the "replace last digit with x" family + // directory pattern (CH32V00x, CH32V10x, CH32L10x, etc.). + // CH32X035 uses its exact uppercase directory name. + if upper.starts_with("CH32V") || upper.starts_with("CH32L") { format!("{}x", &upper[..upper.len() - 1]) } else { upper @@ -358,6 +373,10 @@ pub fn is_ch32v_project(project_dir: &Path, env_name: &str) -> bool { mod tests { use super::*; + fn tempdir() -> tempfile::TempDir { + tempfile::TempDir::new_in(fbuild_paths::temp_subdir("fbuild-ch32v-tests")).unwrap() + } + #[test] fn test_ch32v_orchestrator_platform() { let orch = Ch32vOrchestrator; @@ -366,7 +385,7 @@ mod tests { #[test] fn test_is_ch32v_project() { - let tmp = tempfile::TempDir::new().unwrap(); + let tmp = tempdir(); std::fs::write( tmp.path().join("platformio.ini"), "[env:ch32v003]\nplatform = ch32v\nboard = genericCH32V003F4P6\nframework = arduino\n", @@ -378,7 +397,7 @@ mod tests { #[test] fn test_is_not_ch32v_project() { - let tmp = tempfile::TempDir::new().unwrap(); + let tmp = tempdir(); std::fs::write( tmp.path().join("platformio.ini"), "[env:uno]\nplatform = atmelavr\nboard = uno\nframework = arduino\n", @@ -395,14 +414,15 @@ mod tests { assert_eq!(series_to_system_dir("ch32v203"), "CH32V20x"); assert_eq!(series_to_system_dir("ch32v303"), "CH32V30x"); assert_eq!(series_to_system_dir("ch32v307"), "CH32V30x"); - // CH32X/CH32L: exact uppercase name + // CH32L follows the same family-directory pattern as CH32V. + assert_eq!(series_to_system_dir("ch32l103"), "CH32L10x"); + // CH32X: exact uppercase name assert_eq!(series_to_system_dir("ch32x035"), "CH32X035"); - assert_eq!(series_to_system_dir("ch32l103"), "CH32L103"); } #[test] fn test_resolve_variant_dir_falls_back_to_family_variant() { - let tmp = tempfile::TempDir::new().unwrap(); + let tmp = tempdir(); let fallback = tmp .path() .join("variants") @@ -414,9 +434,29 @@ mod tests { assert_eq!(resolved, fallback); } + #[test] + fn test_resolve_variant_dir_skips_broken_ch32v006_upstream_variant() { + let tmp = tempdir(); + let requested = tmp + .path() + .join("variants") + .join("CH32VM00X") + .join("CH32V006K8"); + let fallback = tmp + .path() + .join("variants") + .join("CH32V00x") + .join("CH32V003F4"); + std::fs::create_dir_all(&requested).unwrap(); + std::fs::create_dir_all(&fallback).unwrap(); + + let resolved = resolve_variant_dir(tmp.path(), "CH32VM00X/CH32V006K8", "CH32V00x"); + assert_eq!(resolved, fallback); + } + #[test] fn test_resolve_variant_h_ignores_missing_preferred_header() { - let tmp = tempfile::TempDir::new().unwrap(); + let tmp = tempdir(); std::fs::write(tmp.path().join("variant_CH32V003F4.h"), "").unwrap(); let resolved = resolve_variant_h(tmp.path(), Some("variant_CH32V006K8.h")); diff --git a/crates/fbuild-build/src/compile_backend.rs b/crates/fbuild-build/src/compile_backend.rs index 0fb84edf..958e0472 100644 --- a/crates/fbuild-build/src/compile_backend.rs +++ b/crates/fbuild-build/src/compile_backend.rs @@ -11,7 +11,7 @@ //! `fbuild-build` and don't have a handle to `DaemonContext`, so the //! installed backend lives in a process-wide `OnceLock` set by the //! daemon's `#[tokio::main]` before any compile fires. Reads -//! ([`get`]) are lock-free. +//! ([`get_global`]) are lock-free. use std::sync::{Arc, OnceLock}; diff --git a/crates/fbuild-build/src/compiler.rs b/crates/fbuild-build/src/compiler.rs index b0631f2d..e98231c8 100644 --- a/crates/fbuild-build/src/compiler.rs +++ b/crates/fbuild-build/src/compiler.rs @@ -751,8 +751,8 @@ pub async fn compile_source( .clone() .or_else(|| output.parent().map(Path::to_path_buf)) .unwrap_or_else(|| PathBuf::from(".")); - let compile_env = fbuild_core::subprocess::compile_env_for_build(&build_scratch_root) - .unwrap_or_default(); + let compile_env = + fbuild_core::subprocess::compile_env_for_build(&build_scratch_root).unwrap_or_default(); let compile_fut = svc.compile(compiler, sanitized, cwd, compile_env); let outcome = tokio::time::timeout(std::time::Duration::from_secs(300), compile_fut) .await diff --git a/crates/fbuild-build/src/compiler_tests.rs b/crates/fbuild-build/src/compiler_tests.rs index 87a63343..4a5c5813 100644 --- a/crates/fbuild-build/src/compiler_tests.rs +++ b/crates/fbuild-build/src/compiler_tests.rs @@ -15,8 +15,8 @@ use super::*; /// the raw relative string — so gcc received absolute `cwd` + relative `-o`, /// resolving to a doubled path whose parent directory (`core/`) was never /// created. -#[test] -fn compile_path_contract_pairs_cwd_and_output_arg_for_282() { +#[tokio::test] +async fn compile_path_contract_pairs_cwd_and_output_arg_for_282() { use crate::zccache::{compile_cwd_from_output, path_arg_for_compile_cwd}; let tmp = tempfile::tempdir().unwrap(); // Normalize the tempdir to the same form `compile_cwd_from_output` will @@ -28,15 +28,10 @@ fn compile_path_contract_pairs_cwd_and_output_arg_for_282() { // `compile_cwd_from_output` runs the result through `strip_unc_prefix`. // Strip the prefix here too so both sides stay on the plain `C:\...` // form. - let tmp_canon = { - let canon = std::fs::canonicalize(tmp.path()).unwrap(); - let s = canon.to_string_lossy().into_owned(); - if let Some(rest) = s.strip_prefix(r"\\?\") { - std::path::PathBuf::from(rest) - } else { - canon - } - }; + let tmp_canon = fbuild_core::path::canonicalize_existing(tmp.path()) + .await + .unwrap() + .into_path_buf(); // Workspace shape mirrors CI: /.fbuild/build//quick/core let workspace = tmp_canon.join("proj_for_282"); let core = workspace diff --git a/crates/fbuild-build/src/pipeline/library.rs b/crates/fbuild-build/src/pipeline/library.rs index 42ac20c4..592ce9ac 100644 --- a/crates/fbuild-build/src/pipeline/library.rs +++ b/crates/fbuild-build/src/pipeline/library.rs @@ -3,6 +3,7 @@ use std::path::{Path, PathBuf}; +use fbuild_core::path::NormalizedPath; use fbuild_core::Result; use super::project_discovery::is_project_a_library; @@ -75,17 +76,6 @@ fn library_name(path: &Path) -> String { .to_lowercase() } -fn strip_windows_extended_prefix(path: PathBuf) -> PathBuf { - if !cfg!(windows) { - return path; - } - let s = path.to_string_lossy(); - if let Some(rest) = s.strip_prefix(r"\\?\").or_else(|| s.strip_prefix("//?/")) { - return PathBuf::from(rest); - } - path -} - /// Resolve `lib_extra_dirs`/`PLATFORMIO_LIB_EXTRA_DIRS` entries to library roots. /// /// `pio ci --lib ` commonly points directly at a library root (FastLED @@ -118,9 +108,7 @@ pub fn discover_extra_library_roots(project_dir: &Path, entries: &[String]) -> V } for candidate in candidates { - let key = std::fs::canonicalize(&candidate) - .map(strip_windows_extended_prefix) - .unwrap_or(candidate.clone()); + let key = NormalizedPath::new(&candidate).into_path_buf(); if seen.insert(key.clone()) { roots.push(key); } @@ -354,10 +342,7 @@ mod extra_library_tests { let roots = discover_extra_library_roots(tmp.path(), &[".".to_string()]); assert_eq!(roots.len(), 1); - assert_eq!( - roots[0], - strip_windows_extended_prefix(std::fs::canonicalize(tmp.path()).unwrap()) - ); + assert_eq!(roots[0], NormalizedPath::new(tmp.path()).into_path_buf()); } #[test] diff --git a/crates/fbuild-build/src/renesas/renesas_compiler.rs b/crates/fbuild-build/src/renesas/renesas_compiler.rs index 8314aa41..6218e025 100644 --- a/crates/fbuild-build/src/renesas/renesas_compiler.rs +++ b/crates/fbuild-build/src/renesas/renesas_compiler.rs @@ -16,6 +16,7 @@ use std::collections::HashMap; use std::path::{Path, PathBuf}; +use fbuild_core::path::NormalizedPath; use fbuild_core::{BuildProfile, Result}; use super::mcu_config::RenesasMcuConfig; @@ -112,13 +113,9 @@ impl RenesasCompiler { let Some(root) = self.framework_root.as_ref() else { return false; }; - match ( - std::fs::canonicalize(source).ok(), - std::fs::canonicalize(root).ok(), - ) { - (Some(s), Some(r)) => s.starts_with(&r), - _ => source.starts_with(root), - } + let source = NormalizedPath::new(source).into_path_buf(); + let root = NormalizedPath::new(root).into_path_buf(); + source.starts_with(&root) } } diff --git a/crates/fbuild-build/src/sam/orchestrator.rs b/crates/fbuild-build/src/sam/orchestrator.rs index 82c7a123..b6486b4b 100644 --- a/crates/fbuild-build/src/sam/orchestrator.rs +++ b/crates/fbuild-build/src/sam/orchestrator.rs @@ -373,9 +373,7 @@ async fn install_samd_core( override_: Option, ) -> Result<(PathBuf, PathBuf, PathBuf, PathBuf, Vec)> { let framework = match override_ { - Some(o) => { - fbuild_packages::library::SamdCores::with_override(¶ms.project_dir, o) - } + Some(o) => fbuild_packages::library::SamdCores::with_override(¶ms.project_dir, o), None => fbuild_packages::library::SamdCores::new(¶ms.project_dir), }; let framework_dir = fbuild_packages::Package::ensure_installed(&framework).await?; diff --git a/crates/fbuild-build/src/source_scanner.rs b/crates/fbuild-build/src/source_scanner.rs index 408c2d42..28d09215 100644 --- a/crates/fbuild-build/src/source_scanner.rs +++ b/crates/fbuild-build/src/source_scanner.rs @@ -10,7 +10,7 @@ //! (`src_filter`, `lib_ldf_mode`) as raw strings. Those patterns are not //! filesystem paths yet — `NormalizedPath` is the wrong type. Instead, //! every call site routes the pattern-level `\` → `/` rewrite through -//! [`normalize_glob_separators`], the single auditable owner of that +//! `normalize_glob_separators`, the single auditable owner of that //! transform. The workspace's `ban_manual_slash_normalize` dylint //! allowlists this file's definition site for exactly that reason //! (FastLED/fbuild#911). @@ -415,12 +415,8 @@ impl SourceFilter { return true; } - let rel = normalize_glob_separators( - &path - .strip_prefix(root) - .unwrap_or(path) - .to_string_lossy(), - ); + let rel = + normalize_glob_separators(&path.strip_prefix(root).unwrap_or(path).to_string_lossy()); let mut included = !self.has_include_rules; for rule in &self.rules { diff --git a/crates/fbuild-build/src/symbol_analyzer/mod.rs b/crates/fbuild-build/src/symbol_analyzer/mod.rs index 8f41e60c..54412a93 100644 --- a/crates/fbuild-build/src/symbol_analyzer/mod.rs +++ b/crates/fbuild-build/src/symbol_analyzer/mod.rs @@ -386,13 +386,7 @@ async fn run_objdump_and_attribute( // FastLED/fbuild#809: `objdump -d` on a large ESP32 ELF can emit // 100+ MB; bound to 2 min so a wedge cannot stall the post-link // analysis step (which is off the critical build path). - let result = run_command( - &args, - None, - None, - Some(std::time::Duration::from_secs(120)), - ) - .await?; + let result = run_command(&args, None, None, Some(std::time::Duration::from_secs(120))).await?; if !result.success() { return Err(FbuildError::BuildFailed(format!( "objdump exit={}: {}", diff --git a/crates/fbuild-build/src/zccache_embedded.rs b/crates/fbuild-build/src/zccache_embedded.rs index c66e796b..4872cc42 100644 --- a/crates/fbuild-build/src/zccache_embedded.rs +++ b/crates/fbuild-build/src/zccache_embedded.rs @@ -27,7 +27,7 @@ //! //! ## Identity defaults //! -//! Constructed via [`HostIdentity::default_for_product("fbuild")`], +//! Constructed via `HostIdentity::default_for_product("fbuild")`, //! which hashes the current exe path so two fbuild installs at //! different paths get distinct cache identities while an //! upgrade-in-place keeps cache continuity. See zccache#925 for the diff --git a/crates/fbuild-build/tests/stm32_acceptance.rs b/crates/fbuild-build/tests/stm32_acceptance.rs index c9af3ab9..e3a1e5a5 100644 --- a/crates/fbuild-build/tests/stm32_acceptance.rs +++ b/crates/fbuild-build/tests/stm32_acceptance.rs @@ -28,7 +28,7 @@ use std::path::{Path, PathBuf}; -use fbuild_build::{BuildOrchestrator, BuildParams}; +use fbuild_build::{compile_backend, BuildOrchestrator, BuildParams}; use fbuild_core::BuildProfile; use fbuild_test_support::{CompileDb, ElfProbe}; @@ -45,9 +45,18 @@ async fn under_test_timeout(fut: F) -> F::Output { } } +async fn install_test_compile_backend() { + let backend = compile_backend::CompileBackend::start() + .await + .expect("compile backend starts for acceptance gate"); + compile_backend::install_global(backend); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "downloads STM32duino + builds firmware; CI-only"] async fn stm32f103c8_blink_with_spi_auto_discovers_library_205_ac4() { + install_test_compile_backend().await; + // Use a temporary project dir so we can write our own SPI-using sketch // independent of whatever ships in the fixture. let tmp = tempfile::TempDir::new().unwrap(); diff --git a/crates/fbuild-build/tests/teensy30_acceptance.rs b/crates/fbuild-build/tests/teensy30_acceptance.rs index 4644eb59..274f14e5 100644 --- a/crates/fbuild-build/tests/teensy30_acceptance.rs +++ b/crates/fbuild-build/tests/teensy30_acceptance.rs @@ -35,7 +35,7 @@ //! ELF section size and forbidden-symbol substring checks, not //! probes for `setup`/`loop`/`analogWrite` symbols. -use fbuild_build::{BuildOrchestrator, BuildParams}; +use fbuild_build::{compile_backend, BuildOrchestrator, BuildParams}; use fbuild_core::BuildProfile; use fbuild_test_support::{CompileDb, ElfProbe}; @@ -52,9 +52,18 @@ async fn under_test_timeout(fut: F) -> F::Output { } } +async fn install_test_compile_backend() { + let backend = compile_backend::CompileBackend::start() + .await + .expect("compile backend starts for acceptance gate"); + compile_backend::install_global(backend); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "downloads Teensyduino + arm-gcc; CI-only"] async fn teensy30_analog_output_meets_205_ac2() { + install_test_compile_backend().await; + // Use a temporary project dir so the committed teensy30 fixture // at tests/platform/teensy30/ stays untouched and no scratch // build artifacts land in the repo. @@ -113,11 +122,10 @@ async fn teensy30_analog_output_meets_205_ac2() { bloat_analysis: false, }; - let result = under_test_timeout( - fbuild_build::teensy::orchestrator::TeensyOrchestrator.build(¶ms), - ) - .await - .expect("teensy30 AnalogOutput build must succeed for AC#2 gate"); + let result = + under_test_timeout(fbuild_build::teensy::orchestrator::TeensyOrchestrator.build(¶ms)) + .await + .expect("teensy30 AnalogOutput build must succeed for AC#2 gate"); assert!(result.success, "build did not report success"); // ── ELF probes (AC#2 + #204 regression guard) ─────────────────────── diff --git a/crates/fbuild-build/tests/teensylc_acceptance.rs b/crates/fbuild-build/tests/teensylc_acceptance.rs index 253fc1b6..bdfd4d4f 100644 --- a/crates/fbuild-build/tests/teensylc_acceptance.rs +++ b/crates/fbuild-build/tests/teensylc_acceptance.rs @@ -18,7 +18,7 @@ use std::path::PathBuf; -use fbuild_build::{BuildOrchestrator, BuildParams}; +use fbuild_build::{compile_backend, BuildOrchestrator, BuildParams}; use fbuild_core::BuildProfile; use fbuild_test_support::{CompileDb, ElfProbe}; @@ -35,9 +35,18 @@ async fn under_test_timeout(fut: F) -> F::Output { } } +async fn install_test_compile_backend() { + let backend = compile_backend::CompileBackend::start() + .await + .expect("compile backend starts for acceptance gate"); + compile_backend::install_global(backend); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[ignore = "downloads Teensyduino + builds firmware; CI-only"] async fn teensylc_blink_meets_205_acceptance_criteria() { + install_test_compile_backend().await; + let project_dir = repo_fixture("teensylc"); let build_dir = tempfile::TempDir::new().unwrap(); @@ -65,11 +74,10 @@ async fn teensylc_blink_meets_205_acceptance_criteria() { bloat_analysis: false, }; - let result = under_test_timeout( - fbuild_build::teensy::orchestrator::TeensyOrchestrator.build(¶ms), - ) - .await - .expect("teensyLC build must succeed for acceptance gate"); + let result = + under_test_timeout(fbuild_build::teensy::orchestrator::TeensyOrchestrator.build(¶ms)) + .await + .expect("teensyLC build must succeed for acceptance gate"); assert!(result.success, "build did not report success"); // ── ELF probes (AC#1) ─────────────────────────────────────────────── diff --git a/crates/fbuild-cli/src/cli/build.rs b/crates/fbuild-cli/src/cli/build.rs index 9486ed3e..a32054ca 100644 --- a/crates/fbuild-cli/src/cli/build.rs +++ b/crates/fbuild-cli/src/cli/build.rs @@ -153,7 +153,7 @@ pub async fn run_build( } /// Convert MSYS/Git-Bash paths (/c/Users/...) to native Windows paths and canonicalize. -pub fn normalize_path(path: &str) -> fbuild_core::Result { +pub async fn normalize_path(path: &str) -> fbuild_core::Result { let converted = if cfg!(windows) { // /c/foo → C:\foo let bytes = path.as_bytes(); @@ -170,10 +170,10 @@ pub fn normalize_path(path: &str) -> fbuild_core::Result { } else { path.to_string() }; - let canon = std::fs::canonicalize(&converted).map_err(|e| { - fbuild_core::FbuildError::Other(format!("cannot resolve path '{}': {}", path, e)) - })?; - let s = canon.to_string_lossy().to_string(); - // Strip \\?\ prefix that canonicalize adds on Windows - Ok(s.strip_prefix(r"\\?\").unwrap_or(&s).to_string()) + let canon = fbuild_core::path::canonicalize_existing(&converted) + .await + .map_err(|e| { + fbuild_core::FbuildError::Other(format!("cannot resolve path '{}': {}", path, e)) + })?; + Ok(canon.as_path().to_string_lossy().to_string()) } diff --git a/crates/fbuild-cli/src/cli/clang_tools.rs b/crates/fbuild-cli/src/cli/clang_tools.rs index 4da1a056..67a413e8 100644 --- a/crates/fbuild-cli/src/cli/clang_tools.rs +++ b/crates/fbuild-cli/src/cli/clang_tools.rs @@ -21,7 +21,7 @@ pub async fn run_iwyu( environment: Option, verbose: bool, ) -> fbuild_core::Result<()> { - let project_dir = normalize_path(&project_dir)?; + let project_dir = normalize_path(&project_dir).await?; // Step 1: Ensure IWYU is installed let component = fbuild_packages::toolchain::ClangComponent::new( @@ -313,22 +313,19 @@ pub async fn run_iwyu( cmd.arg(&file); // FastLED/fbuild#810: cap each IWYU invocation at 120s so a wedged // subprocess can't hang the whole fan-out forever. - let output = match tokio::time::timeout( - std::time::Duration::from_secs(120), - cmd.output(), - ) - .await - { - Ok(res) => res, - Err(_) => { - return ( - file, - Err("include-what-you-use timed out after 120s".to_string()), - false, - src_path, - ); - } - }; + let output = + match tokio::time::timeout(std::time::Duration::from_secs(120), cmd.output()).await + { + Ok(res) => res, + Err(_) => { + return ( + file, + Err("include-what-you-use timed out after 120s".to_string()), + false, + src_path, + ); + } + }; match output { Ok(out) => { @@ -504,16 +501,12 @@ pub async fn run_clang_tool( verbose: bool, extra_args: &[&str], ) -> fbuild_core::Result<()> { - let project_dir = normalize_path(&project_dir)?; + let project_dir = normalize_path(&project_dir).await?; // Step 1: Ensure tool is installed let component = fbuild_packages::toolchain::ClangComponent::new(kind); let tool_path = component.get_binary(binary_name).await?; - output::progress(format!( - "Using {}: {}", - binary_name, - tool_path.display() - )); + output::progress(format!("Using {}: {}", binary_name, tool_path.display())); // Step 2: Generate compile_commands.json via fbuild daemon output::progress("Generating compile_commands.json..."); @@ -605,18 +598,15 @@ pub async fn run_clang_tool( } // FastLED/fbuild#810: cap each clang-tidy invocation at 120s so a // wedged subprocess can't hang the whole fan-out forever. - let output = match tokio::time::timeout( - std::time::Duration::from_secs(120), - cmd.output(), - ) - .await - { - Ok(res) => res, - Err(_) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "clang-tidy timed out after 120s", - )), - }; + let output = + match tokio::time::timeout(std::time::Duration::from_secs(120), cmd.output()).await + { + Ok(res) => res, + Err(_) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "clang-tidy timed out after 120s", + )), + }; (file, output) }); handles.push(handle); diff --git a/crates/fbuild-cli/src/cli/clangd_config.rs b/crates/fbuild-cli/src/cli/clangd_config.rs index 40e9ba03..23f39fcc 100644 --- a/crates/fbuild-cli/src/cli/clangd_config.rs +++ b/crates/fbuild-cli/src/cli/clangd_config.rs @@ -19,7 +19,7 @@ pub async fn run_clangd_config( environment: Option, verbose: bool, ) -> fbuild_core::Result<()> { - let project_dir = normalize_path(&project_dir)?; + let project_dir = normalize_path(&project_dir).await?; let project_path = std::path::Path::new(&project_dir); // Step 1: Resolve the environment name (explicit -e wins, else default). @@ -113,7 +113,9 @@ pub async fn run_clangd_config( )); } output::result("\nInstall the clangd extension (llvm-vs-code-extensions.vscode-clangd),"); - output::result("then run \"clangd: Restart language server\" in VS Code to pick up the config."); + output::result( + "then run \"clangd: Restart language server\" in VS Code to pick up the config.", + ); Ok(()) } @@ -341,11 +343,17 @@ mod tests { ); } - #[cfg(windows)] #[test] fn clangd_yaml_mentions_compiler_and_database() { - let yaml = render_clangd_yaml(r"C:\tc\bin\avr-g++"); + let yaml = render_clangd_yaml("/tc/bin/avr-g++"); assert!(yaml.contains("CompilationDatabase: .")); + assert!(yaml.contains("Compiler: /tc/bin/avr-g++")); + } + + #[test] + #[cfg(windows)] + fn clangd_yaml_rewrites_windows_backslashes() { + let yaml = render_clangd_yaml(r"C:\tc\bin\avr-g++"); assert!(yaml.contains("Compiler: C:/tc/bin/avr-g++")); } diff --git a/crates/fbuild-cli/src/cli/compile_many.rs b/crates/fbuild-cli/src/cli/compile_many.rs index accce6fe..90eed2c2 100644 --- a/crates/fbuild-cli/src/cli/compile_many.rs +++ b/crates/fbuild-cli/src/cli/compile_many.rs @@ -50,30 +50,31 @@ pub fn normalize_ci_sketches(entries: &[String]) -> Vec { /// Build the `PLATFORMIO_*` env overlay for `fbuild ci` from `--lib` and /// `--project-conf`. Returns an empty map when neither flag was set. -pub fn build_ci_pio_env( +pub async fn build_ci_pio_env( libs: &[String], project_conf: Option<&str>, ) -> std::collections::HashMap { let mut env = std::collections::HashMap::new(); if !libs.is_empty() { - let libs: Vec = libs - .iter() - .map(|lib| { - std::fs::canonicalize(lib) - .ok() - .map(|p| p.to_string_lossy().to_string()) - .unwrap_or_else(|| lib.clone()) - }) - .collect(); + let mut canonical_libs = Vec::with_capacity(libs.len()); + for lib in libs { + let value = fbuild_core::path::canonicalize_existing(lib) + .await + .ok() + .map(|p| p.as_path().to_string_lossy().to_string()) + .unwrap_or_else(|| lib.clone()); + canonical_libs.push(value); + } env.insert( "PLATFORMIO_LIB_EXTRA_DIRS".to_string(), - libs.join(ci_lib_extra_dirs_sep()), + canonical_libs.join(ci_lib_extra_dirs_sep()), ); } if let Some(conf) = project_conf { - let canonical = std::fs::canonicalize(conf) + let canonical = fbuild_core::path::canonicalize_existing(conf) + .await .ok() - .map(|p| p.to_string_lossy().to_string()) + .map(|p| p.as_path().to_string_lossy().to_string()) .unwrap_or_else(|| conf.to_string()); env.insert("PLATFORMIO_PROJECT_CONFIG".to_string(), canonical); } diff --git a/crates/fbuild-cli/src/cli/daemon_cmd.rs b/crates/fbuild-cli/src/cli/daemon_cmd.rs index 269279e8..05bb7db4 100644 --- a/crates/fbuild-cli/src/cli/daemon_cmd.rs +++ b/crates/fbuild-cli/src/cli/daemon_cmd.rs @@ -270,10 +270,7 @@ fn run_daemon_running_process(json: bool) -> fbuild_core::Result<()> { output::result(format!(" - temp: {}", cache_roots.temp.display())); output::result(format!(" - log: {}", cache_roots.log.display())); output::result(format!(" - lock: {}", cache_roots.lock.display())); - output::result(format!( - " - runtime: {}", - cache_roots.runtime.display() - )); + output::result(format!(" - runtime: {}", cache_roots.runtime.display())); output::result(format!(" - config: {}", cache_roots.config.display())); Ok(()) } diff --git a/crates/fbuild-cli/src/cli/device.rs b/crates/fbuild-cli/src/cli/device.rs index 23342a0b..0529829a 100644 --- a/crates/fbuild-cli/src/cli/device.rs +++ b/crates/fbuild-cli/src/cli/device.rs @@ -37,6 +37,7 @@ pub async fn run_device(action: DeviceAction) -> fbuild_core::Result<()> { // older than the resolver wiring), fall back to the raw // description so behavior is identical to pre-resolver. let pretty = device_pretty_name(dev); + let pretty = with_cdc_suffix(pretty, dev.vid, dev.is_cdc); output::result(format!( "{:<20} {:<12} {:<12} {:<24} {}", dev.port, @@ -61,12 +62,18 @@ pub async fn run_device(action: DeviceAction) -> fbuild_core::Result<()> { }; output::result(format!(" {}", resp.port)); output::result(format!(" Device ID: {}", resp.device_id)); + if let (Some(vid), Some(pid)) = (resp.vid, resp.pid) { + output::result(format!(" USB ID: {vid:04X}:{pid:04X}")); + } if let Some(ref vendor) = resp.vendor_name { output::result(format!(" Vendor: {}", vendor)); } if let Some(ref product) = resp.product_name { output::result(format!(" Product: {}", product)); } + if resp.vid.is_some() { + output::result(format!(" CDC: {}", cdc_label(resp.is_cdc))); + } output::result(format!(" Description: {}", resp.description)); if let Some(ref serial) = resp.serial_number { output::result(format!(" Serial: {}", serial)); @@ -191,6 +198,22 @@ fn device_description(description: &str, previous_port: Option<&str>) -> String } } +fn with_cdc_suffix(description: String, vid: Option, is_cdc: Option) -> String { + if vid.is_some() { + format!("{description} [cdc={}]", cdc_label(is_cdc)) + } else { + description + } +} + +fn cdc_label(is_cdc: Option) -> &'static str { + match is_cdc { + Some(true) => "yes", + Some(false) => "no", + None => "unknown", + } +} + /// Compose the canonical `"vendor product (VVVV:PPPP)"` display string /// for a device row. Falls back to the daemon-provided `description` /// (and bare hex VID:PID, when available) so this code remains usable @@ -209,3 +232,28 @@ fn device_pretty_name(dev: &crate::daemon_client::DeviceInfoResponse) -> String _ => dev.description.clone(), } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cdc_suffix_only_applies_to_usb_devices() { + assert_eq!( + with_cdc_suffix("Espressif ESP32-S3".to_string(), Some(0x303A), Some(true)), + "Espressif ESP32-S3 [cdc=yes]" + ); + assert_eq!( + with_cdc_suffix("CP210x bridge".to_string(), Some(0x10C4), Some(false)), + "CP210x bridge [cdc=no]" + ); + assert_eq!( + with_cdc_suffix("USB device".to_string(), Some(0x303A), None), + "USB device [cdc=unknown]" + ); + assert_eq!( + with_cdc_suffix("Bluetooth Serial".to_string(), None, None), + "Bluetooth Serial" + ); + } +} diff --git a/crates/fbuild-cli/src/cli/dispatch.rs b/crates/fbuild-cli/src/cli/dispatch.rs index ee036dca..a15b9db1 100644 --- a/crates/fbuild-cli/src/cli/dispatch.rs +++ b/crates/fbuild-cli/src/cli/dispatch.rs @@ -338,7 +338,7 @@ pub async fn async_main() { upgrade_package, }) => { let code = run_sync_cmd( - Some(std::path::PathBuf::from(project_dir)), + Some(fbuild_core::path::NormalizedPath::new(project_dir)), environment, yes, locked, @@ -513,7 +513,7 @@ pub async fn async_main() { )); } let normalized = normalize_ci_sketches(&sketches); - let pio_env = build_ci_pio_env(&libs, project_conf.as_deref()); + let pio_env = build_ci_pio_env(&libs, project_conf.as_deref()).await; run_compile_many(CompileManyArgs { board, framework_jobs, diff --git a/crates/fbuild-cli/src/cli/port_scan.rs b/crates/fbuild-cli/src/cli/port_scan.rs index 079a06f7..96811a96 100644 --- a/crates/fbuild-cli/src/cli/port_scan.rs +++ b/crates/fbuild-cli/src/cli/port_scan.rs @@ -11,18 +11,14 @@ //! //! Different from [`super::serial_probe::SerialAction::Probe`]'s `list` //! action (FastLED/fbuild#686) which annotates from a tiny hardcoded -//! `BOARD_FINGERPRINTS` table — `port scan` consults the full canonical -//! FastLED/boards aggregate via the tiered resolver, so an unrecognized +//! `BOARD_FINGERPRINTS` table — `port scan` consults the fbuild online-data +//! VID:PID overlay via the tiered resolver, so an unrecognized //! device shows the actual vendor + product name instead of a blank //! hint. //! -//! The canonical data source is [FastLED/boards] (see -//! for the live portal). The -//! resolver in `fbuild_core::usb` is wired to consume it via tier-2 -//! overlay; that's separate plumbing — this command takes whatever the -//! resolver returns. -//! -//! [FastLED/boards]: https://github.com/FastLED/boards +//! The canonical runtime data source is the `fastled/fbuild` `online-data` +//! branch. The resolver in `fbuild_core::usb` is wired to consume it via the +//! tier-2 overlay; this command takes whatever the resolver returns. use clap::Subcommand; use fbuild_core::{FbuildError, Result}; @@ -36,7 +32,7 @@ pub enum PortAction { /// the OS-visible identity + a `└─ vendor / product` second row /// resolved via [`fbuild_core::usb::resolve`]. Scan { - /// Skip the network fetch of the FastLED/boards online overlay + /// Skip the network fetch of the fbuild online-data overlay /// (tier-2 of the resolver). Useful for offline runs — the /// embedded vendor archive (tier-1) still provides vendor /// names; product columns fall through to the synthetic @@ -71,7 +67,7 @@ fn run_scan(offline: bool) -> Result<()> { Ok(()) } -/// Fetch the FastLED/boards `usb-vids.proto.zstd` tier-2 overlay backing +/// Fetch the fbuild online-data `usb-vids.proto.zstd` tier-2 overlay backing /// [`fbuild_core::usb::resolve`] into the local cache root, then install it. /// /// Best-effort: any I/O / network / parse failure is swallowed and the @@ -227,6 +223,19 @@ fn render_usb_port( pid: u16, product: Option<&str>, serial: Option<&str>, +) { + let kernel_class = fbuild_serial::port_class::detect_port_kernel_class(name); + render_usb_port_with_kernel_class(out, name, vid, pid, product, serial, kernel_class); +} + +fn render_usb_port_with_kernel_class( + out: &mut String, + name: &str, + vid: u16, + pid: u16, + product: Option<&str>, + serial: Option<&str>, + kernel_class: Option, ) { use std::fmt::Write as _; let descriptor = product.unwrap_or("USB Serial Device"); @@ -240,7 +249,21 @@ fn render_usb_port( ); let info = fbuild_core::usb::resolve(vid, pid); let friendly_product = friendly_product_name(vid, pid, &info.product, product); - let _ = writeln!(out, " └─ {} / {}", info.vendor, friendly_product); + let _ = writeln!( + out, + " └─ {} / {} cdc={}", + info.vendor, + friendly_product, + cdc_label(kernel_class) + ); +} + +fn cdc_label(kernel_class: Option) -> &'static str { + match kernel_class { + Some(fbuild_serial::port_class::PortKernelClass::CdcAcm) => "yes", + Some(fbuild_serial::port_class::PortKernelClass::UsbSerialBridge) => "no", + None => "unknown", + } } /// Pick the most "friendly" product label for the resolver row. @@ -291,8 +314,9 @@ fn is_generic_descriptor(d: &str) -> bool { } /// Small inline supplement for common embedded VID:PIDs that the -/// canonical FastLED/boards `vidpid` table doesn't carry yet. Keep it -/// short — anything that lands upstream should be removed here. +/// canonical online overlay may not carry in offline/test paths. Keep it +/// short — anything that lands into the embedded product map should be +/// removed here. const FRIENDLY_PRODUCTS: &[(u16, u16, &str)] = &[ // Espressif Systems (VID 0x303A) — ESP32 series USB-CDC ACM. (0x303A, 0x1001, "ESP32-S3 USB-CDC"), @@ -540,11 +564,44 @@ mod tests { assert!(!out.contains("1 USB ports,")); } + #[test] + fn usb_port_rows_show_cdc_classification() { + use fbuild_serial::port_class::PortKernelClass; + + let mut cdc = String::new(); + render_usb_port_with_kernel_class( + &mut cdc, + "COM1", + 0x303A, + 0x1001, + None, + None, + Some(PortKernelClass::CdcAcm), + ); + assert!(cdc.contains("cdc=yes"), "got: {cdc}"); + + let mut bridge = String::new(); + render_usb_port_with_kernel_class( + &mut bridge, + "COM2", + 0x10C4, + 0xEA60, + None, + None, + Some(PortKernelClass::UsbSerialBridge), + ); + assert!(bridge.contains("cdc=no"), "got: {bridge}"); + + let mut unknown = String::new(); + render_usb_port_with_kernel_class(&mut unknown, "COM3", 0x303A, 0x1001, None, None, None); + assert!(unknown.contains("cdc=unknown"), "got: {unknown}"); + } + #[test] fn esp32_s3_cdc_pid_gets_friendly_supplement() { - // 303A:1001 lacks a product entry in both the embedded archive - // and the FastLED/boards `vidpid` table; the inline supplement - // is what makes the row a friendly name instead of synthetic. + // In the tier-1/offline path the embedded archive carries vendor + // names only; the inline supplement keeps this common PID friendly + // when the online product overlay is not installed. let ports = vec![usb_port( "COM25", 0x303A, diff --git a/crates/fbuild-cli/src/cli/purge.rs b/crates/fbuild-cli/src/cli/purge.rs index 6ab01b78..17fde58f 100644 --- a/crates/fbuild-cli/src/cli/purge.rs +++ b/crates/fbuild-cli/src/cli/purge.rs @@ -183,7 +183,9 @@ pub fn list_cached_packages(cache_root: &std::path::Path) -> fbuild_core::Result total_count, format_size(total_size) )); - output::result("\nUse 'fbuild purge all' to remove all, or 'fbuild purge ' for specific."); + output::result( + "\nUse 'fbuild purge all' to remove all, or 'fbuild purge ' for specific.", + ); Ok(()) } diff --git a/crates/fbuild-cli/src/cli/serial_probe.rs b/crates/fbuild-cli/src/cli/serial_probe.rs index 7d65745d..983737f9 100644 --- a/crates/fbuild-cli/src/cli/serial_probe.rs +++ b/crates/fbuild-cli/src/cli/serial_probe.rs @@ -69,7 +69,7 @@ pub enum ProbeAction { }, /// Open a port, optionally send a payload, then read bytes to /// stdout for up to `--seconds`. DTR/RTS is picked from the - /// inferred [`BoardFamily`] so CDC-ACM bridges (LPC11U35, FTDI + /// inferred `BoardFamily` so CDC-ACM bridges (LPC11U35, FTDI /// CDC) see "host ready" rather than the ESP-default /// "host not ready" that drops bytes silently. Read { @@ -215,8 +215,8 @@ fn parse_vid_pid(s: &str) -> Result<(u16, u16)> { /// `fbuild serial probe read PORT …` — open with correct DTR/RTS, /// optionally send a payload, then read bytes to stdout for the /// duration. Uses `family_for_vid_pid` against the resolved port to -/// pick a [`BoardFamily`] and consults -/// [`BoardFamily::idle_dtr_rts`] for the open-time state. +/// pick a `BoardFamily` and consults +/// `BoardFamily::idle_dtr_rts` for the open-time state. fn read_port(port_name: &str, baud: u32, seconds: f64, send: Option<&str>) -> Result<()> { // Resolve the family by looking up the connected port's VID:PID, // if `serialport` can see it. Unknown VID:PID → safe-default diff --git a/crates/fbuild-cli/src/cli/show.rs b/crates/fbuild-cli/src/cli/show.rs index c5a0fa46..79e79a22 100644 --- a/crates/fbuild-cli/src/cli/show.rs +++ b/crates/fbuild-cli/src/cli/show.rs @@ -19,10 +19,7 @@ pub fn run_show(target: &str, follow: bool, lines: usize) -> fbuild_core::Result pub fn show_daemon_logs(follow: bool, initial_lines: usize) -> fbuild_core::Result<()> { let log_path = fbuild_paths::get_daemon_log_file(); if !log_path.exists() { - output::error(format!( - "daemon log file not found: {}", - log_path.display() - )); + output::error(format!("daemon log file not found: {}", log_path.display())); output::error("the daemon may not have been started yet"); return Ok(()); } diff --git a/crates/fbuild-cli/src/cli/sync_cmd.rs b/crates/fbuild-cli/src/cli/sync_cmd.rs index 0fad2354..9ba5877d 100644 --- a/crates/fbuild-cli/src/cli/sync_cmd.rs +++ b/crates/fbuild-cli/src/cli/sync_cmd.rs @@ -5,14 +5,12 @@ //! `crates/fbuild-cli/src/sync/`; this file is intentionally thin so the //! CLI wiring is easy to audit. -use std::path::PathBuf; - use crate::sync::{run_sync, SyncArgs, SyncOutcome}; /// Adapter for `Commands::Sync` — invoked from `cli::dispatch`. #[allow(clippy::too_many_arguments)] // Matches clap's parsed variant fields 1:1. pub async fn run_sync_cmd( - project_dir: Option, + project_dir: Option, environment: Option, yes: bool, locked: bool, diff --git a/crates/fbuild-cli/src/cli/tests.rs b/crates/fbuild-cli/src/cli/tests.rs index e2dab7aa..c1cea4d3 100644 --- a/crates/fbuild-cli/src/cli/tests.rs +++ b/crates/fbuild-cli/src/cli/tests.rs @@ -51,10 +51,10 @@ fn normalize_batch_preserves_order() { assert_eq!(got[1], "examples/Fire2012"); } -#[test] -fn build_pio_env_joins_libs_with_platform_separator() { +#[tokio::test] +async fn build_pio_env_joins_libs_with_platform_separator() { let libs = vec!["a".to_string(), "b".to_string()]; - let env = build_ci_pio_env(&libs, None); + let env = build_ci_pio_env(&libs, None).await; let expected = if cfg!(windows) { "a;b" } else { "a:b" }; assert_eq!( env.get("PLATFORMIO_LIB_EXTRA_DIRS").map(String::as_str), @@ -63,16 +63,16 @@ fn build_pio_env_joins_libs_with_platform_separator() { assert!(!env.contains_key("PLATFORMIO_PROJECT_CONFIG")); } -#[test] -fn build_pio_env_omits_libs_key_when_empty() { - let env = build_ci_pio_env(&[], None); +#[tokio::test] +async fn build_pio_env_omits_libs_key_when_empty() { + let env = build_ci_pio_env(&[], None).await; assert!(env.is_empty()); } -#[test] -fn build_pio_env_falls_back_to_as_given_when_canonicalize_fails() { +#[tokio::test] +async fn build_pio_env_falls_back_to_as_given_when_canonicalize_fails() { let bogus = "/this/path/does/not/exist/conf.ini"; - let env = build_ci_pio_env(&[], Some(bogus)); + let env = build_ci_pio_env(&[], Some(bogus)).await; assert_eq!( env.get("PLATFORMIO_PROJECT_CONFIG").map(String::as_str), Some(bogus) diff --git a/crates/fbuild-cli/src/daemon_client/types.rs b/crates/fbuild-cli/src/daemon_client/types.rs index 65c1ba6b..9a2dcc86 100644 --- a/crates/fbuild-cli/src/daemon_client/types.rs +++ b/crates/fbuild-cli/src/daemon_client/types.rs @@ -321,6 +321,10 @@ pub struct DeviceInfoResponse { /// Pretty USB product name (same provenance as `vendor_name`). #[serde(default)] pub product_name: Option, + /// `true` for CDC-ACM, `false` for a USB-serial bridge, `None` for + /// unknown or older daemons that did not report the field. + #[serde(default)] + pub is_cdc: Option, #[serde(default)] pub serial_number: Option, #[serde(default)] @@ -340,6 +344,10 @@ pub struct DeviceStatusResponse { pub port: String, pub device_id: String, pub description: String, + #[serde(default)] + pub vid: Option, + #[serde(default)] + pub pid: Option, /// Pretty USB vendor name resolved by the daemon. `None` for /// bluetooth/PCI/unknown serials. #[serde(default)] @@ -347,6 +355,10 @@ pub struct DeviceStatusResponse { /// Pretty USB product name (same provenance as `vendor_name`). #[serde(default)] pub product_name: Option, + /// `true` for CDC-ACM, `false` for a USB-serial bridge, `None` for + /// unknown or older daemons that did not report the field. + #[serde(default)] + pub is_cdc: Option, #[serde(default)] pub serial_number: Option, #[serde(default)] diff --git a/crates/fbuild-cli/src/sync/lockfile.rs b/crates/fbuild-cli/src/sync/lockfile.rs index b15140ef..d96603dd 100644 --- a/crates/fbuild-cli/src/sync/lockfile.rs +++ b/crates/fbuild-cli/src/sync/lockfile.rs @@ -47,7 +47,9 @@ impl Lockfile { packages.sort_by(|a, b| { a.name .cmp(&b.name) - .then_with(|| format!("{:?}", a.source_type).cmp(&format!("{:?}", b.source_type))) + .then_with(|| { + format!("{:?}", a.source_type).cmp(&format!("{:?}", b.source_type)) + }) .then_with(|| a.raw.cmp(&b.raw)) }); out_envs.insert(env_name, LockEnv { packages }); @@ -116,12 +118,17 @@ impl Lockfile { return LockDiff::Stale(format!("env `{env}` missing from lock")); }; // Re-classify the lock's packages to build a compare-shape. - let mut new_pkgs: Vec = - new_deps.iter().cloned().map(LockPackage::from_dep).collect(); + let mut new_pkgs: Vec = new_deps + .iter() + .cloned() + .map(LockPackage::from_dep) + .collect(); new_pkgs.sort_by(|a, b| { a.name .cmp(&b.name) - .then_with(|| format!("{:?}", a.source_type).cmp(&format!("{:?}", b.source_type))) + .then_with(|| { + format!("{:?}", a.source_type).cmp(&format!("{:?}", b.source_type)) + }) .then_with(|| a.raw.cmp(&b.raw)) }); if new_pkgs != lock_env.packages { @@ -221,7 +228,10 @@ impl std::fmt::Display for LockfileError { Self::Parse(m) => write!(f, "lockfile parse: {m}"), Self::Serialize(m) => write!(f, "lockfile serialize: {m}"), Self::UnsupportedVersion(v) => { - write!(f, "unsupported lockfile version {v}, expected {LOCKFILE_VERSION}") + write!( + f, + "unsupported lockfile version {v}, expected {LOCKFILE_VERSION}" + ) } } } @@ -271,10 +281,7 @@ mod tests { #[test] fn packages_are_sorted_by_name() { - let lock = Lockfile::from_classified( - "t".into(), - deps("uno", &["Zebra", "Alpha", "Mango"]), - ); + let lock = Lockfile::from_classified("t".into(), deps("uno", &["Zebra", "Alpha", "Mango"])); let names: Vec<&str> = lock.envs["uno"] .packages .iter() @@ -355,10 +362,7 @@ mod tests { #[test] fn compare_fresh_when_deps_match() { - let lock = Lockfile::from_classified( - "t".into(), - deps("uno", &["FastLED", "./libs/local"]), - ); + let lock = Lockfile::from_classified("t".into(), deps("uno", &["FastLED", "./libs/local"])); let diff = lock.compare_to_classified(&deps("uno", &["FastLED", "./libs/local"])); assert_eq!(diff, LockDiff::Fresh); } diff --git a/crates/fbuild-cli/src/sync/mod.rs b/crates/fbuild-cli/src/sync/mod.rs index eef54dff..1766b79d 100644 --- a/crates/fbuild-cli/src/sync/mod.rs +++ b/crates/fbuild-cli/src/sync/mod.rs @@ -29,10 +29,11 @@ pub mod lockfile; pub mod source; use std::collections::BTreeMap; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::time::SystemTime; use fbuild_config::PlatformIOConfig; +use fbuild_core::path::NormalizedPath; use self::lockfile::{LockDiff, Lockfile, LockfileError}; use self::source::{classify, ClassifiedDep}; @@ -62,7 +63,7 @@ pub struct SyncArgs { #[allow(dead_code)] // FastLED/fbuild#618 Phase 2 hook — CLI surface stable now pub upgrade_package: Option, /// Explicit project dir. Defaults to CWD. - pub project_dir: Option, + pub project_dir: Option, } impl SyncArgs { @@ -87,7 +88,7 @@ impl SyncArgs { #[derive(Debug)] pub enum SyncOutcome { /// Lockfile written / re-written successfully. Exit 0. - Wrote(PathBuf), + Wrote(NormalizedPath), /// Lockfile matches the current `platformio.ini` (nothing to do). /// Exit 0. NoOp, @@ -124,7 +125,7 @@ impl SyncOutcome { /// these into a [`SyncOutcome::Error`] before returning to the CLI. #[derive(Debug)] pub enum SyncError { - NoPlatformioIni(PathBuf), + NoPlatformioIni(NormalizedPath), ConfigParse(String), UnknownEnv(String), NoEnvsDeclared, @@ -166,7 +167,9 @@ pub async fn run_sync(args: SyncArgs) -> SyncOutcome { async fn do_run_sync(args: SyncArgs) -> Result { let project_dir = match &args.project_dir { Some(p) => p.clone(), - None => std::env::current_dir().map_err(|e| SyncError::Io(e.to_string()))?, + None => { + NormalizedPath::new(std::env::current_dir().map_err(|e| SyncError::Io(e.to_string()))?) + } }; let ini_path = project_dir.join("platformio.ini"); if !ini_path.is_file() { @@ -178,7 +181,11 @@ async fn do_run_sync(args: SyncArgs) -> Result { // Discover envs. `get_environments` returns borrowed slices; own them // so the rest of the pipeline can move envs into the classified map. - let all_envs: Vec = config.get_environments().iter().map(|s| s.to_string()).collect(); + let all_envs: Vec = config + .get_environments() + .iter() + .map(|s| s.to_string()) + .collect(); if all_envs.is_empty() { return Err(SyncError::NoEnvsDeclared); } @@ -187,9 +194,7 @@ async fn do_run_sync(args: SyncArgs) -> Result { let selected_envs = select_envs(&all_envs, args.environment.as_deref())?; // Multi-env prompt (skipped by --yes / --check / -e ). - if selected_envs.len() > 1 - && !args.skip_multi_env_prompt() - && !prompt_multi_env(&selected_envs) + if selected_envs.len() > 1 && !args.skip_multi_env_prompt() && !prompt_multi_env(&selected_envs) { return Ok(SyncOutcome::UserCancelled); } @@ -367,7 +372,7 @@ mod tests { fn args_for(dir: &Path) -> SyncArgs { SyncArgs { - project_dir: Some(dir.to_path_buf()), + project_dir: Some(NormalizedPath::new(dir)), ..Default::default() } } @@ -411,7 +416,10 @@ lib_deps = FastLED let mut args = args_for(tmp.path()); args.check = true; let outcome = run_sync(args).await; - assert!(matches!(outcome, SyncOutcome::CheckFailed(_)), "got {outcome:?}"); + assert!( + matches!(outcome, SyncOutcome::CheckFailed(_)), + "got {outcome:?}" + ); } #[tokio::test] @@ -541,7 +549,7 @@ lib_deps = FastLED #[test] fn exit_code_matrix() { - assert_eq!(SyncOutcome::Wrote(PathBuf::new()).exit_code(), 0); + assert_eq!(SyncOutcome::Wrote(NormalizedPath::new("")).exit_code(), 0); assert_eq!(SyncOutcome::NoOp.exit_code(), 0); assert_eq!(SyncOutcome::CheckPassed.exit_code(), 0); assert_eq!(SyncOutcome::DryRun.exit_code(), 0); @@ -553,13 +561,21 @@ lib_deps = FastLED #[test] fn skip_multi_env_prompt_matrix() { - assert!(SyncArgs { yes: true, ..Default::default() }.skip_multi_env_prompt()); + assert!(SyncArgs { + yes: true, + ..Default::default() + } + .skip_multi_env_prompt()); assert!(SyncArgs { environment: Some("uno".into()), ..Default::default() } .skip_multi_env_prompt()); - assert!(SyncArgs { check: true, ..Default::default() }.skip_multi_env_prompt()); + assert!(SyncArgs { + check: true, + ..Default::default() + } + .skip_multi_env_prompt()); assert!(!SyncArgs::default().skip_multi_env_prompt()); } diff --git a/crates/fbuild-cli/src/sync/source.rs b/crates/fbuild-cli/src/sync/source.rs index dcdb06b0..41225644 100644 --- a/crates/fbuild-cli/src/sync/source.rs +++ b/crates/fbuild-cli/src/sync/source.rs @@ -190,7 +190,9 @@ pub fn classify(raw: &str) -> ClassifiedDep { return ClassifiedDep { raw: raw.to_string(), name: archive_name_from_url(&url_no_ref).unwrap_or_else(|| { - last_segment(&url_no_ref).trim_end_matches(".git").to_string() + last_segment(&url_no_ref) + .trim_end_matches(".git") + .to_string() }), source_type: SourceType::HttpArchive, version_spec: hash_suffix, @@ -272,9 +274,8 @@ fn is_github_url(url: &str) -> bool { /// Extract `(owner, repo)` from `https://github.com//[.git]`. fn github_owner_repo(url: &str) -> Option<(String, String)> { - let after_host = url - .split_once("github.com/") - .map(|(_, rest)| rest)?; + let lower = url.to_ascii_lowercase(); + let after_host = &url[lower.find("github.com/")? + "github.com/".len()..]; let mut parts = after_host.splitn(3, '/'); let owner = parts.next()?.trim(); let repo = parts.next()?.trim().trim_end_matches(".git"); diff --git a/crates/fbuild-cli/src/update_check.rs b/crates/fbuild-cli/src/update_check.rs index 5c74b7e3..a7e3d92f 100644 --- a/crates/fbuild-cli/src/update_check.rs +++ b/crates/fbuild-cli/src/update_check.rs @@ -40,9 +40,11 @@ //! command exit code. use serde::{Deserialize, Serialize}; -use std::path::{Path, PathBuf}; +use std::path::Path; use std::time::{Duration, SystemTime}; +use fbuild_core::path::NormalizedPath; + /// Where the running `fbuild` binary lives, per the schema in issue #626. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "kebab-case")] @@ -92,9 +94,7 @@ impl InstallSource { fn suggestion(self, latest: &str) -> String { match self { Self::Pypi => "run: python -m pip install --upgrade fbuild".to_string(), - Self::LocalSource => { - "editable install — run: git pull && pip install -e .".to_string() - } + Self::LocalSource => "editable install — run: git pull && pip install -e .".to_string(), Self::Vcs => { "VCS install — re-run: pip install --upgrade git+https://github.com/FastLED/fbuild" .to_string() @@ -196,7 +196,11 @@ async fn try_run_passive_check( if let Ok(cached) = read_cache(&cache_path) { if cached.is_fresh(now) && cached.current_version == current_version { if cached.stale { - emit_warning(current_version, &cached.latest_version, cached.install_source); + emit_warning( + current_version, + &cached.latest_version, + cached.install_source, + ); } return Ok(()); } @@ -247,7 +251,9 @@ fn suppress_from_env() -> bool { /// case-fold of `false` are treated as falsy — matches the convention /// PowerShell / GitHub Actions / bash all seem to converge on. fn is_truthy_env(key: &str) -> bool { - let Ok(v) = std::env::var(key) else { return false }; + let Ok(v) = std::env::var(key) else { + return false; + }; let trimmed = v.trim(); if trimmed.is_empty() || trimmed == "0" || trimmed.eq_ignore_ascii_case("false") { return false; @@ -258,9 +264,15 @@ fn is_truthy_env(key: &str) -> bool { fn is_ci_env() -> bool { // Common CI markers. `CI=true` is universal per // https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables. - ["CI", "GITHUB_ACTIONS", "GITLAB_CI", "CIRCLECI", "JENKINS_URL"] - .iter() - .any(|k| is_truthy_env(k)) + [ + "CI", + "GITHUB_ACTIONS", + "GITLAB_CI", + "CIRCLECI", + "JENKINS_URL", + ] + .iter() + .any(|k| is_truthy_env(k)) } /// Classify the running binary's install source. Never panics; returns @@ -325,7 +337,9 @@ fn probe_dist_info(dir: &Path) -> Option { }; for entry in entries.flatten() { let name = entry.file_name(); - let Some(name_str) = name.to_str() else { continue }; + let Some(name_str) = name.to_str() else { + continue; + }; if !name_str.starts_with("fbuild-") || !name_str.ends_with(".dist-info") { continue; } @@ -353,7 +367,11 @@ fn classify_direct_url(path: &Path) -> InstallSource { return InstallSource::Vcs; } if let Some(dir_info) = v.get("dir_info") { - if dir_info.get("editable").and_then(|e| e.as_bool()).unwrap_or(false) { + if dir_info + .get("editable") + .and_then(|e| e.as_bool()) + .unwrap_or(false) + { return InstallSource::LocalSource; } } @@ -372,7 +390,9 @@ fn classify_direct_url(path: &Path) -> InstallSource { async fn fetch_latest_for_source( source: InstallSource, ) -> Result<(String, String), UpdateCheckError> { - if source.is_stable_pypi_source() || matches!(source, InstallSource::LocalSource | InstallSource::Vcs) { + if source.is_stable_pypi_source() + || matches!(source, InstallSource::LocalSource | InstallSource::Vcs) + { // Editable / VCS installs still get the PyPI version as the target // (the user's env has fbuild-the-package; the "latest published" is // the meaningful comparison). @@ -446,7 +466,9 @@ where { let mut best: Option = None; for s in iter { - let Ok(v) = semver::Version::parse(s) else { continue }; + let Ok(v) = semver::Version::parse(s) else { + continue; + }; if !v.pre.is_empty() { continue; } @@ -482,13 +504,13 @@ fn is_newer(latest: &str, current: &str) -> Result { // Cache I/O // -------------------------------------------------------------------------- -fn cache_file_path() -> PathBuf { - fbuild_paths::get_cache_root().join(CACHE_FILENAME) +fn cache_file_path() -> NormalizedPath { + NormalizedPath::new(fbuild_paths::get_cache_root().join(CACHE_FILENAME)) } fn read_cache(path: &Path) -> Result { - let raw = std::fs::read_to_string(path) - .map_err(|e| UpdateCheckError::Cache(format!("read: {e}")))?; + let raw = + std::fs::read_to_string(path).map_err(|e| UpdateCheckError::Cache(format!("read: {e}")))?; serde_json::from_str(&raw).map_err(|e| UpdateCheckError::Cache(format!("parse: {e}"))) } @@ -545,8 +567,14 @@ mod tests { #[test] fn install_source_env_parsing() { - assert_eq!(InstallSource::from_env_str("pypi"), Some(InstallSource::Pypi)); - assert_eq!(InstallSource::from_env_str("PIP"), Some(InstallSource::Pypi)); + assert_eq!( + InstallSource::from_env_str("pypi"), + Some(InstallSource::Pypi) + ); + assert_eq!( + InstallSource::from_env_str("PIP"), + Some(InstallSource::Pypi) + ); assert_eq!( InstallSource::from_env_str(" local "), Some(InstallSource::LocalSource) @@ -589,7 +617,13 @@ mod tests { #[test] fn pypi_pick_latest_stable_skips_prereleases() { - let versions = ["2.3.14", "2.3.15", "2.4.0-rc.1", "2.4.0-alpha.1", "2.3.15.dev0"]; + let versions = [ + "2.3.14", + "2.3.15", + "2.4.0-rc.1", + "2.4.0-alpha.1", + "2.3.15.dev0", + ]; assert_eq!( pick_latest_stable_from_strings(versions.iter().copied()), Some("2.3.15".to_string()) @@ -599,7 +633,10 @@ mod tests { #[test] fn pypi_pick_latest_stable_all_prereleases_returns_none() { let versions = ["2.4.0-rc.1", "2.4.0-alpha.1"]; - assert_eq!(pick_latest_stable_from_strings(versions.iter().copied()), None); + assert_eq!( + pick_latest_stable_from_strings(versions.iter().copied()), + None + ); } #[test] @@ -739,9 +776,17 @@ mod tests { // Snapshot + clear ALL CI markers we recognize, not just CI — // otherwise GitHub Actions' own GITHUB_ACTIONS=true poisons the // `assert!(!is_ci_env())` checks below. - const CI_KEYS: &[&str] = &["CI", "GITHUB_ACTIONS", "GITLAB_CI", "CIRCLECI", "JENKINS_URL"]; - let saved: Vec<(&str, Option)> = - CI_KEYS.iter().map(|k| (*k, std::env::var(*k).ok())).collect(); + const CI_KEYS: &[&str] = &[ + "CI", + "GITHUB_ACTIONS", + "GITLAB_CI", + "CIRCLECI", + "JENKINS_URL", + ]; + let saved: Vec<(&str, Option)> = CI_KEYS + .iter() + .map(|k| (*k, std::env::var(*k).ok())) + .collect(); // SAFETY: single-threaded test process. for k in CI_KEYS { std::env::remove_var(k); diff --git a/crates/fbuild-config/Cargo.toml b/crates/fbuild-config/Cargo.toml index c1e3a0d3..06af8f79 100644 --- a/crates/fbuild-config/Cargo.toml +++ b/crates/fbuild-config/Cargo.toml @@ -20,4 +20,5 @@ tracing = { workspace = true } regex = { workspace = true } [dev-dependencies] +fbuild-paths = { path = "../fbuild-paths" } tempfile = { workspace = true } diff --git a/crates/fbuild-config/src/bin/enrich_boards.rs b/crates/fbuild-config/src/bin/enrich_boards.rs index d724b697..bf753982 100644 --- a/crates/fbuild-config/src/bin/enrich_boards.rs +++ b/crates/fbuild-config/src/bin/enrich_boards.rs @@ -241,16 +241,13 @@ fn enrich_board(board_path: &Path, pio_dir: &Path) -> Result { .map_err(|e| format!("parse {}: {e}", board_path.display()))?; let obj = board.as_object().ok_or("board JSON is not an object")?; - let board_id = obj - .get("id") - .and_then(|v| v.as_str()) - .unwrap_or_else(|| { - board_path - .file_stem() - .expect("fbuild-config: board JSON path always has a file stem") - .to_str() - .expect("fbuild-config: board JSON file stems are ASCII") - }); + let board_id = obj.get("id").and_then(|v| v.as_str()).unwrap_or_else(|| { + board_path + .file_stem() + .expect("fbuild-config: board JSON path always has a file stem") + .to_str() + .expect("fbuild-config: board JSON file stems are ASCII") + }); let platform = obj.get("platform").and_then(|v| v.as_str()).unwrap_or(""); if platform.is_empty() { diff --git a/crates/fbuild-config/src/board/tests.rs b/crates/fbuild-config/src/board/tests.rs index 1bed745e..f60f6281 100644 --- a/crates/fbuild-config/src/board/tests.rs +++ b/crates/fbuild-config/src/board/tests.rs @@ -10,7 +10,8 @@ use super::loaders::parse_boards_txt; use super::BoardConfig; fn write_boards_txt(content: &str) -> NamedTempFile { - let mut f = NamedTempFile::new().unwrap(); + let mut f = + NamedTempFile::new_in(fbuild_paths::temp_subdir("fbuild-config-board-tests")).unwrap(); f.write_all(content.as_bytes()).unwrap(); f.flush().unwrap(); f diff --git a/crates/fbuild-config/src/board/tests_enriched_json.rs b/crates/fbuild-config/src/board/tests_enriched_json.rs index 024905b5..3ff0683d 100644 --- a/crates/fbuild-config/src/board/tests_enriched_json.rs +++ b/crates/fbuild-config/src/board/tests_enriched_json.rs @@ -12,7 +12,8 @@ use super::db::get_board_db; use super::{BoardConfig, Esp32QemuPsramConfig}; fn write_boards_txt(content: &str) -> NamedTempFile { - let mut f = NamedTempFile::new().unwrap(); + let mut f = + NamedTempFile::new_in(fbuild_paths::temp_subdir("fbuild-config-board-tests")).unwrap(); f.write_all(content.as_bytes()).unwrap(); f.flush().unwrap(); f diff --git a/crates/fbuild-config/src/ini_parser/tests.rs b/crates/fbuild-config/src/ini_parser/tests.rs index fc21b1eb..2afca0ac 100644 --- a/crates/fbuild-config/src/ini_parser/tests.rs +++ b/crates/fbuild-config/src/ini_parser/tests.rs @@ -8,7 +8,8 @@ use std::path::Path; use tempfile::NamedTempFile; fn write_ini(content: &str) -> NamedTempFile { - let mut f = NamedTempFile::new().unwrap(); + let mut f = + NamedTempFile::new_in(fbuild_paths::temp_subdir("fbuild-config-ini-parser-tests")).unwrap(); f.write_all(content.as_bytes()).unwrap(); f.flush().unwrap(); f diff --git a/crates/fbuild-config/src/sdkconfig.rs b/crates/fbuild-config/src/sdkconfig.rs index a82fe1d6..bd695114 100644 --- a/crates/fbuild-config/src/sdkconfig.rs +++ b/crates/fbuild-config/src/sdkconfig.rs @@ -120,6 +120,10 @@ mod tests { use std::fs; use tempfile::TempDir; + fn tempdir() -> TempDir { + TempDir::new_in(fbuild_paths::temp_subdir("fbuild-config-sdkconfig-tests")).unwrap() + } + #[test] fn arduino_default_values() { let d = SdkConfigSummary::arduino_default(); @@ -210,14 +214,14 @@ CONFIG_COMPILER_OPTIMIZATION_DEBUG=y #[test] fn from_project_dir_empty_returns_default() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let s = SdkConfigSummary::from_project_dir(tmp.path()); assert_eq!(s, SdkConfigSummary::arduino_default()); } #[test] fn from_project_dir_defaults_only() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); fs::write( tmp.path().join("sdkconfig.defaults"), "CONFIG_ESP_SYSTEM_PANIC_GDBSTUB=y\n", @@ -229,7 +233,7 @@ CONFIG_COMPILER_OPTIMIZATION_DEBUG=y #[test] fn from_project_dir_active_wins_over_defaults() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); // defaults says gdbstub=y fs::write( tmp.path().join("sdkconfig.defaults"), diff --git a/crates/fbuild-core/src/fs.rs b/crates/fbuild-core/src/fs.rs index 1e17700e..fc8a87d3 100644 --- a/crates/fbuild-core/src/fs.rs +++ b/crates/fbuild-core/src/fs.rs @@ -31,9 +31,9 @@ // items here is preferable to having callers `use tokio::fs` directly — // the matching `ban_tokio_fs_direct_import` dylint enforces this. pub use tokio::fs::{ - canonicalize, copy, create_dir, create_dir_all, hard_link, metadata, read, read_dir, - read_link, read_to_string, remove_dir, remove_dir_all, remove_file, rename, - set_permissions, symlink_metadata, write, DirEntry, File, OpenOptions, ReadDir, + canonicalize, copy, create_dir, create_dir_all, hard_link, metadata, read, read_dir, read_link, + read_to_string, remove_dir, remove_dir_all, remove_file, rename, set_permissions, + symlink_metadata, write, DirEntry, File, OpenOptions, ReadDir, }; use std::path::Path; @@ -82,7 +82,7 @@ pub async fn write_atomic( tmp_name.push(format!(".tmp.{pid}.{nonce}")); let tmp_path = match path.parent() { Some(parent) => parent.join(&tmp_name), - None => std::path::PathBuf::from(&tmp_name), + None => crate::path::NormalizedPath::new(Path::new(&tmp_name)).into_path_buf(), }; // Ensure parent dir exists. Mirrors `tokio::fs::write`'s contract @@ -156,10 +156,7 @@ pub async fn write_atomic( /// Use this from any sync caller (the daemon's status writer, in-process /// snapshots, test harnesses). Prefer [`write_atomic`] from async paths /// that need to avoid blocking the reactor on the fsync. -pub fn write_atomic_sync( - path: impl AsRef, - content: impl AsRef<[u8]>, -) -> std::io::Result<()> { +pub fn write_atomic_sync(path: impl AsRef, content: impl AsRef<[u8]>) -> std::io::Result<()> { use std::io::Write as _; let path = path.as_ref(); @@ -179,7 +176,7 @@ pub fn write_atomic_sync( tmp_name.push(format!(".tmp.{pid}.{nonce}")); let tmp_path = match path.parent() { Some(parent) => parent.join(&tmp_name), - None => std::path::PathBuf::from(&tmp_name), + None => crate::path::NormalizedPath::new(Path::new(&tmp_name)).into_path_buf(), }; // Mirror `write_atomic`'s parent-must-exist contract. @@ -260,10 +257,7 @@ mod tests { count += 1; let name = entry.file_name(); let name = name.to_string_lossy(); - assert!( - !name.contains(".tmp."), - "leftover tempfile: {name}" - ); + assert!(!name.contains(".tmp."), "leftover tempfile: {name}"); } assert_eq!(count, 1); } diff --git a/crates/fbuild-core/src/path.rs b/crates/fbuild-core/src/path.rs index 81b186ac..6a5f8af1 100644 --- a/crates/fbuild-core/src/path.rs +++ b/crates/fbuild-core/src/path.rs @@ -587,10 +587,7 @@ mod tests { let canon = canonicalize_existing(&file).await.unwrap(); // Round-trip through `as_path` and back to `NormalizedPath` // produces the same key. - assert_eq!( - canon.key(), - NormalizedPath::new(canon.as_path()).key() - ); + assert_eq!(canon.key(), NormalizedPath::new(canon.as_path()).key()); } /// `canonicalize_existing` propagates a `NotFound` for a missing diff --git a/crates/fbuild-core/src/subprocess.rs b/crates/fbuild-core/src/subprocess.rs index 55d7b2aa..781ac771 100644 --- a/crates/fbuild-core/src/subprocess.rs +++ b/crates/fbuild-core/src/subprocess.rs @@ -22,7 +22,7 @@ //! The legacy "what running-process gives us" output shape is preserved //! byte-for-byte: stdout/stderr are returned as `String`s composed of //! lossy-UTF-8 lines joined by `\n` with a trailing newline when -//! non-empty (see [`join_lines`]). +//! non-empty (see `join_lines`). use std::path::Path; use std::process::Stdio; @@ -211,7 +211,7 @@ impl ToolOutput { /// subcommands, tests). /// /// Timeout policy: passing `timeout: None` applies the workspace -/// default cap (see [`DEFAULT_SUBPROCESS_TIMEOUT_SECS`] / +/// default cap (see `DEFAULT_SUBPROCESS_TIMEOUT_SECS` / /// `FBUILD_SUBPROCESS_DEFAULT_TIMEOUT_SECS`). Pass `Some(Duration)` for /// an explicit budget. For the (rare) legitimately-unbounded case use /// [`run_command_no_timeout`]. @@ -270,8 +270,14 @@ pub async fn run_command_with_stdin( env: Option<&[(&str, &str)]>, timeout: Option, ) -> Result { - run_command_with_stdin_inner(args, stdin_bytes, cwd, env, resolve_default_timeout(timeout)) - .await + run_command_with_stdin_inner( + args, + stdin_bytes, + cwd, + env, + resolve_default_timeout(timeout), + ) + .await } async fn run_command_with_stdin_inner( @@ -880,10 +886,7 @@ mod tests { let keys: Vec<&str> = env.iter().map(|(k, _)| k.as_str()).collect(); for required in ["TMPDIR", "TMP", "TEMP"] { - assert!( - keys.contains(&required), - "missing {required}; got {keys:?}" - ); + assert!(keys.contains(&required), "missing {required}; got {keys:?}"); } for (k, v) in &env { @@ -902,8 +905,7 @@ mod tests { fn compile_env_for_build_is_idempotent() { let tmp = tempfile::tempdir().expect("tempdir"); let first = compile_env_for_build(tmp.path()).expect("first call"); - let second = - compile_env_for_build(tmp.path()).expect("second call (dir already exists)"); + let second = compile_env_for_build(tmp.path()).expect("second call (dir already exists)"); assert_eq!(first, second, "helper must be idempotent"); } diff --git a/crates/fbuild-core/src/usb/data.rs b/crates/fbuild-core/src/usb/data.rs index 9d1e5d4d..a95c1084 100644 --- a/crates/fbuild-core/src/usb/data.rs +++ b/crates/fbuild-core/src/usb/data.rs @@ -1,8 +1,8 @@ //! Tier-2 online overlay: an optional per-VID protobuf map loaded from disk //! at runtime. //! -//! Current on-disk schema is `usb-vids.proto.zstd`, published by -//! : +//! Current on-disk schema is `usb-vids.proto.zstd`, published by the +//! `fastled/fbuild` `online-data` branch: //! //! ```protobuf //! message UsbVidDatabase { @@ -45,7 +45,7 @@ //! shape on disk just avoids duplicating the vendor name for every //! product entry under a VID (significantly smaller payload). //! -//! The CLI downloads the zstd-compressed protobuf from FastLED/boards, +//! The CLI downloads the zstd-compressed protobuf from `online-data`, //! writes it to the global fbuild cache root, and calls //! [`install_online_cache_proto_zstd`] to plug it into the resolver. //! Replacing the cache is supported (`RwLock`, not `OnceLock`) so the @@ -71,8 +71,9 @@ pub const MANIFEST_URL: &str = pub const USB_VID_JSON_URL: &str = "https://raw.githubusercontent.com/fastled/fbuild/online-data/data/usb-vid.json"; -/// Current compact USB VID:PID overlay published by FastLED/boards. -pub const USB_VIDS_PROTO_ZSTD_URL: &str = "https://fastled.github.io/boards/usb-vids.proto.zstd"; +/// Current compact USB VID:PID overlay published by the `online-data` branch. +pub const USB_VIDS_PROTO_ZSTD_URL: &str = + "https://raw.githubusercontent.com/fastled/fbuild/online-data/data/usb-vids.proto.zstd"; static ONLINE_MAP: RwLock>> = RwLock::new(None); @@ -245,7 +246,7 @@ fn parse_hex_u16(s: &str) -> Option { #[cfg(test)] pub(crate) fn clear_online_cache_for_tests() { - let mut guard = ONLINE_MAP.write().unwrap(); + let mut guard = ONLINE_MAP.write().unwrap_or_else(|e| e.into_inner()); *guard = None; } diff --git a/crates/fbuild-daemon/src/context.rs b/crates/fbuild-daemon/src/context.rs index 365855c7..fc69418a 100644 --- a/crates/fbuild-daemon/src/context.rs +++ b/crates/fbuild-daemon/src/context.rs @@ -399,9 +399,9 @@ impl DaemonContext { match tokio::time::timeout(Duration::from_secs(5), join).await { Ok(Ok(())) => {} Ok(Err(e)) => tracing::warn!("device refresh task panicked: {}", e), - Err(_) => tracing::warn!( - "device refresh exceeded 5s; continuing — USB stack may be wedged" - ), + Err(_) => { + tracing::warn!("device refresh exceeded 5s; continuing — USB stack may be wedged") + } } self.rebind_recent_device_port_moves().await; } @@ -422,9 +422,7 @@ impl DaemonContext { false } Err(_) => { - tracing::warn!( - "device refresh exceeded 5s; continuing — USB stack may be wedged" - ); + tracing::warn!("device refresh exceeded 5s; continuing — USB stack may be wedged"); false } }; diff --git a/crates/fbuild-daemon/src/device_manager.rs b/crates/fbuild-daemon/src/device_manager.rs index ff855917..81d6a852 100644 --- a/crates/fbuild-daemon/src/device_manager.rs +++ b/crates/fbuild-daemon/src/device_manager.rs @@ -108,6 +108,10 @@ pub struct DeviceState { pub vendor_name: Option, /// Human-readable USB product name (same provenance as `vendor_name`). pub product_name: Option, + /// Whether the OS classified this USB serial endpoint as CDC-ACM. + /// `Some(false)` means a USB-serial bridge driver; `None` means + /// non-USB or unknown on this platform. + pub is_cdc: Option, pub serial_number: Option, pub previous_port: Option, pub exclusive_lease: Option, @@ -159,6 +163,7 @@ struct DiscoveredDevice { pid: Option, vendor_name: Option, product_name: Option, + is_cdc: Option, serial_number: Option, } @@ -206,7 +211,10 @@ impl DeviceManager { /// *recently enough* — we just don't need one on every deploy. pub fn refresh_devices_if_stale(&self, max_age: std::time::Duration) -> bool { { - let last = self.last_refresh_at.lock().unwrap_or_else(|e| e.into_inner()); + let last = self + .last_refresh_at + .lock() + .unwrap_or_else(|e| e.into_inner()); if let Some(t) = *last { if t.elapsed() < max_age { return false; @@ -245,6 +253,10 @@ impl DeviceManager { serialport::SerialPortType::PciPort => (None, None, "PCI Serial".to_string()), serialport::SerialPortType::Unknown => (None, None, "Unknown".to_string()), }; + let is_cdc = match &port_info.port_type { + serialport::SerialPortType::UsbPort(_) => detect_is_cdc(&port_info.port_name), + _ => None, + }; // Resolve VID:PID → pretty (vendor, product) via the bundled // `usb-ids` snapshot + any online overlay the daemon has // installed. When both are present, the resolver-derived @@ -275,13 +287,17 @@ impl DeviceManager { pid, vendor_name, product_name, + is_cdc, serial_number, } }) .collect(); self.refresh_from_discovered(discovered); - *self.last_refresh_at.lock().unwrap_or_else(|e| e.into_inner()) = Some(Instant::now()); + *self + .last_refresh_at + .lock() + .unwrap_or_else(|e| e.into_inner()) = Some(Instant::now()); } fn refresh_from_discovered(&self, discovered: Vec) { @@ -324,13 +340,17 @@ impl DeviceManager { state.pid = device.pid; state.vendor_name = device.vendor_name; state.product_name = device.product_name; + state.is_cdc = device.is_cdc; state.serial_number = device.serial_number; if let Some(previous_port) = state.previous_port.clone() { - self.recent_port_moves.lock().unwrap_or_else(|e| e.into_inner()).push(DevicePortMove { - previous_port, - port: key.clone(), - serial_number, - }); + self.recent_port_moves + .lock() + .unwrap_or_else(|e| e.into_inner()) + .push(DevicePortMove { + previous_port, + port: key.clone(), + serial_number, + }); } devices.insert(key, state); continue; @@ -347,6 +367,7 @@ impl DeviceManager { pid: device.pid, vendor_name: device.vendor_name.clone(), product_name: device.product_name.clone(), + is_cdc: device.is_cdc, serial_number: device.serial_number.clone(), previous_port: None, exclusive_lease: None, @@ -365,6 +386,7 @@ impl DeviceManager { entry.pid = device.pid; entry.vendor_name = device.vendor_name; entry.product_name = device.product_name; + entry.is_cdc = device.is_cdc; entry.serial_number = device.serial_number; } @@ -386,17 +408,27 @@ impl DeviceManager { /// Get all devices. pub fn get_all_devices(&self) -> HashMap { - self.devices.lock().unwrap_or_else(|e| e.into_inner()).clone() + self.devices + .lock() + .unwrap_or_else(|e| e.into_inner()) + .clone() } pub fn take_recent_port_moves(&self) -> Vec { - let mut moves = self.recent_port_moves.lock().unwrap_or_else(|e| e.into_inner()); + let mut moves = self + .recent_port_moves + .lock() + .unwrap_or_else(|e| e.into_inner()); std::mem::take(&mut *moves) } /// Get status for a specific device (by port name). pub fn get_device_status(&self, port: &str) -> Option { - self.devices.lock().unwrap_or_else(|e| e.into_inner()).get(port).cloned() + self.devices + .lock() + .unwrap_or_else(|e| e.into_inner()) + .get(port) + .cloned() } /// Acquire an exclusive lease on a device. @@ -670,6 +702,7 @@ impl DeviceManager { pid: Some(0x5678), vendor_name: Some("Test Vendor".to_string()), product_name: Some("Test Device".to_string()), + is_cdc: None, serial_number: Some("TEST-SERIAL".to_string()), previous_port: None, exclusive_lease: None, @@ -683,5 +716,12 @@ impl DeviceManager { } } +fn detect_is_cdc(port_name: &str) -> Option { + match fbuild_serial::port_class::detect_port_kernel_class(port_name)? { + fbuild_serial::port_class::PortKernelClass::CdcAcm => Some(true), + fbuild_serial::port_class::PortKernelClass::UsbSerialBridge => Some(false), + } +} + #[cfg(test)] mod tests; diff --git a/crates/fbuild-daemon/src/device_manager/tests.rs b/crates/fbuild-daemon/src/device_manager/tests.rs index 0ad68c25..9f45bfa9 100644 --- a/crates/fbuild-daemon/src/device_manager/tests.rs +++ b/crates/fbuild-daemon/src/device_manager/tests.rs @@ -188,12 +188,14 @@ fn tracked_serial_lease_moves_to_new_port_on_refresh() { pid: Some(0x5678), vendor_name: Some("Test Vendor".to_string()), product_name: Some("Test Device".to_string()), + is_cdc: Some(true), serial_number: Some("TEST-SERIAL".to_string()), }]); assert!(mgr.get_device_status("COM3").is_none()); let moved = mgr.get_device_status("COM4").unwrap(); assert_eq!(moved.previous_port.as_deref(), Some("COM3")); + assert_eq!(moved.is_cdc, Some(true)); assert_eq!( moved.exclusive_lease.as_ref().map(|l| l.client_id.as_str()), Some("c1") @@ -230,6 +232,7 @@ fn untracked_serial_lease_stays_on_old_disconnected_port() { pid: Some(0x5678), vendor_name: Some("Test Vendor".to_string()), product_name: Some("Test Device".to_string()), + is_cdc: Some(false), serial_number: Some("TEST-SERIAL".to_string()), }]); @@ -238,6 +241,7 @@ fn untracked_serial_lease_stays_on_old_disconnected_port() { assert!(old.exclusive_lease.is_some()); let new = mgr.get_device_status("COM4").unwrap(); assert!(new.exclusive_lease.is_none()); + assert_eq!(new.is_cdc, Some(false)); } #[test] diff --git a/crates/fbuild-daemon/src/handlers/devices.rs b/crates/fbuild-daemon/src/handlers/devices.rs index be15e3ae..d9895fd9 100644 --- a/crates/fbuild-daemon/src/handlers/devices.rs +++ b/crates/fbuild-daemon/src/handlers/devices.rs @@ -41,8 +41,11 @@ pub async fn device_status( port: port.clone(), device_id: String::new(), description: format!("device '{}' not found", port), + vid: None, + pid: None, vendor_name: None, product_name: None, + is_cdc: None, serial_number: None, previous_port: None, is_connected: false, @@ -63,6 +66,7 @@ fn device_info(state: &DeviceState) -> DeviceInfo { pid: state.pid, vendor_name: state.vendor_name.clone(), product_name: state.product_name.clone(), + is_cdc: state.is_cdc, serial_number: state.serial_number.clone(), previous_port: state.previous_port.clone(), description: state.description.clone(), @@ -83,8 +87,11 @@ fn device_status_response(state: DeviceState) -> DeviceStatusResponse { port: state.port, device_id: state.device_id, description: state.description, + vid: state.vid, + pid: state.pid, vendor_name: state.vendor_name, product_name: state.product_name, + is_cdc: state.is_cdc, serial_number: state.serial_number, previous_port: state.previous_port, is_connected: state.is_connected, diff --git a/crates/fbuild-daemon/src/handlers/emulator/select.rs b/crates/fbuild-daemon/src/handlers/emulator/select.rs index dc0cce33..be5d765b 100644 --- a/crates/fbuild-daemon/src/handlers/emulator/select.rs +++ b/crates/fbuild-daemon/src/handlers/emulator/select.rs @@ -327,20 +327,20 @@ pub async fn test_emu( // FastLED/fbuild#808 (CRITICAL): wall-clock cap on the // pre-emulator build so a wedged compiler doesn't hang the // test-emu handler indefinitely. - const EMU_BUILD_HARD_DEADLINE: std::time::Duration = std::time::Duration::from_secs(60 * 60); + const EMU_BUILD_HARD_DEADLINE: std::time::Duration = + std::time::Duration::from_secs(60 * 60); match fbuild_build::get_orchestrator(p) { - Ok(orchestrator) => match tokio::time::timeout( - EMU_BUILD_HARD_DEADLINE, - orchestrator.build(¶ms), - ) - .await - { - Ok(r) => r, - Err(_) => Err(fbuild_core::FbuildError::Other(format!( - "pre-emulator build exceeded hard deadline ({}s); aborting", - EMU_BUILD_HARD_DEADLINE.as_secs() - ))), - }, + Ok(orchestrator) => { + match tokio::time::timeout(EMU_BUILD_HARD_DEADLINE, orchestrator.build(¶ms)) + .await + { + Ok(r) => r, + Err(_) => Err(fbuild_core::FbuildError::Other(format!( + "pre-emulator build exceeded hard deadline ({}s); aborting", + EMU_BUILD_HARD_DEADLINE.as_secs() + ))), + } + } Err(e) => Err(e), } }; diff --git a/crates/fbuild-daemon/src/handlers/operations/build.rs b/crates/fbuild-daemon/src/handlers/operations/build.rs index 30fa12ad..b19a9e2b 100644 --- a/crates/fbuild-daemon/src/handlers/operations/build.rs +++ b/crates/fbuild-daemon/src/handlers/operations/build.rs @@ -730,18 +730,17 @@ pub async fn build( const NON_STREAM_BUILD_HARD_DEADLINE: std::time::Duration = std::time::Duration::from_secs(60 * 60); let result = match fbuild_build::get_orchestrator(platform) { - Ok(orch) => match tokio::time::timeout( - NON_STREAM_BUILD_HARD_DEADLINE, - orch.build(¶ms), - ) - .await - { - Ok(r) => r, - Err(_) => Err(fbuild_core::FbuildError::Other(format!( - "build exceeded hard deadline ({}s); aborting — a compiler may be wedged", - NON_STREAM_BUILD_HARD_DEADLINE.as_secs() - ))), - }, + Ok(orch) => { + match tokio::time::timeout(NON_STREAM_BUILD_HARD_DEADLINE, orch.build(¶ms)) + .await + { + Ok(r) => r, + Err(_) => Err(fbuild_core::FbuildError::Other(format!( + "build exceeded hard deadline ({}s); aborting — a compiler may be wedged", + NON_STREAM_BUILD_HARD_DEADLINE.as_secs() + ))), + } + } Err(e) => Err(e), }; @@ -982,8 +981,7 @@ mod tests { fired.store(true, Ordering::Release); } // Waiter must NOT have been notified. - let res = - tokio::time::timeout(std::time::Duration::from_millis(100), waiter).await; + let res = tokio::time::timeout(std::time::Duration::from_millis(100), waiter).await; assert!( res.is_err(), "cancel must not fire when fired_normal_terminal is set" diff --git a/crates/fbuild-daemon/src/handlers/operations/deploy.rs b/crates/fbuild-daemon/src/handlers/operations/deploy.rs index 643b3a1b..cce9f625 100644 --- a/crates/fbuild-daemon/src/handlers/operations/deploy.rs +++ b/crates/fbuild-daemon/src/handlers/operations/deploy.rs @@ -580,11 +580,8 @@ pub async fn deploy( 16, ) .unwrap_or(0), - u32::from_str_radix( - mcu_config.firmware_offset().trim_start_matches("0x"), - 16, - ) - .unwrap_or(0), + u32::from_str_radix(mcu_config.firmware_offset().trim_start_matches("0x"), 16) + .unwrap_or(0), ); // Session-trusted verify-skip: if the daemon last @@ -643,17 +640,20 @@ pub async fn deploy( "verify-flash: device already running this exact image; skipping write" ); // VerifySkip → recovery skipped (#605). - return Ok((None, fbuild_deploy::DeploymentResult { - success: true, - message: format!( - "firmware already current on {} (skipped via verify-flash)", - port - ), - port: Some(port.to_string()), - stdout, - stderr, - outcome: fbuild_deploy::DeployOutcome::VerifySkip, - })); + return Ok(( + None, + fbuild_deploy::DeploymentResult { + success: true, + message: format!( + "firmware already current on {} (skipped via verify-flash)", + port + ), + port: Some(port.to_string()), + stdout, + stderr, + outcome: fbuild_deploy::DeployOutcome::VerifySkip, + }, + )); } Ok(fbuild_deploy::esp32::VerifyOutcome::Mismatch { regions, .. }) => { // Pick only the regions that actually differ @@ -702,7 +702,9 @@ pub async fn deploy( .set_trusted_firmware_hash(port, hash); } _ => { - ctx_for_deploy.device_manager.clear_trusted_firmware_hash(port); + ctx_for_deploy + .device_manager + .clear_trusted_firmware_hash(port); } } } @@ -773,8 +775,18 @@ pub async fn deploy( ); Box::new(deployer) } - fbuild_core::Platform::NxpLpc => fbuild_deploy::lpc::dispatch_box(&board_id, &deploy_board_overrides, deploy_project.as_path(), baud_override), - _ => return Err(fbuild_core::FbuildError::DeployFailed(format!("deployer for {:?} not yet implemented", platform))), + fbuild_core::Platform::NxpLpc => fbuild_deploy::lpc::dispatch_box( + &board_id, + &deploy_board_overrides, + deploy_project.as_path(), + baud_override, + ), + _ => { + return Err(fbuild_core::FbuildError::DeployFailed(format!( + "deployer for {:?} not yet implemented", + platform + ))) + } }; let result = deployer .deploy( diff --git a/crates/fbuild-daemon/src/handlers/operations/deploy_port.rs b/crates/fbuild-daemon/src/handlers/operations/deploy_port.rs index 6490c525..d4a520a9 100644 --- a/crates/fbuild-daemon/src/handlers/operations/deploy_port.rs +++ b/crates/fbuild-daemon/src/handlers/operations/deploy_port.rs @@ -201,6 +201,7 @@ mod tests { pid, vendor_name: None, product_name: None, + is_cdc: None, serial_number: None, previous_port: None, exclusive_lease: None, diff --git a/crates/fbuild-daemon/src/handlers/websockets.rs b/crates/fbuild-daemon/src/handlers/websockets.rs index 5737b696..f1000817 100644 --- a/crates/fbuild-daemon/src/handlers/websockets.rs +++ b/crates/fbuild-daemon/src/handlers/websockets.rs @@ -24,9 +24,8 @@ use tokio::sync::oneshot; /// purely as a panic-free guarantee for the cold path; FastLED/fbuild#826 /// flagged the prior `.unwrap()` calls as a stability hazard. fn serialize_or_fallback(value: &T) -> String { - serde_json::to_string(value).unwrap_or_else(|_| { - r#"{"type":"error","message":""}"#.to_string() - }) + serde_json::to_string(value) + .unwrap_or_else(|_| r#"{"type":"error","message":""}"#.to_string()) } // ReaderControl -- inbound -> reader cross-task RPC for the small set diff --git a/crates/fbuild-daemon/src/main.rs b/crates/fbuild-daemon/src/main.rs index 5b680a06..9cc0ec45 100644 --- a/crates/fbuild-daemon/src/main.rs +++ b/crates/fbuild-daemon/src/main.rs @@ -169,12 +169,12 @@ async fn main() { // timeout. Cleaning up the PID file lets `fbuild daemon list` reflect // reality, and the bind retry below handles the kernel-state case. // See ISSUES.md "Issue B5a". - if let Some(stale_pid) = read_stale_daemon_pid() { + if let Some(stale_pid) = read_stale_daemon_pid().await { tracing::warn!( "found stale daemon PID file pointing at dead PID {}; cleaning up", stale_pid ); - let _ = std::fs::remove_file(fbuild_paths::get_daemon_pid_file()); + let _ = fbuild_core::fs::remove_file(fbuild_paths::get_daemon_pid_file()).await; } let listener = bind_listener_with_retry(&addr).await; @@ -185,12 +185,12 @@ async fn main() { let pid_file = fbuild_paths::get_daemon_pid_file(); let port_file = fbuild_paths::get_daemon_port_file(); if let Some(parent) = pid_file.parent() { - let _ = std::fs::create_dir_all(parent); + let _ = fbuild_core::fs::create_dir_all(parent).await; } - if let Err(e) = std::fs::write(&pid_file, std::process::id().to_string()) { + if let Err(e) = fbuild_core::fs::write(&pid_file, std::process::id().to_string()).await { tracing::warn!("failed to write PID file: {}", e); } - if let Err(e) = std::fs::write(&port_file, port.to_string()) { + if let Err(e) = fbuild_core::fs::write(&port_file, port.to_string()).await { tracing::warn!("failed to write port file: {}", e); } @@ -432,8 +432,8 @@ async fn main() { }); // Clean up PID and port files - let _ = std::fs::remove_file(&pid_file); - let _ = std::fs::remove_file(&port_file); + let _ = fbuild_core::fs::remove_file(&pid_file).await; + let _ = fbuild_core::fs::remove_file(&port_file).await; tracing::info!("daemon exiting"); std::process::exit(0); @@ -447,9 +447,9 @@ async fn main() { /// Used at startup to clean up after a crashed previous instance so that /// `fbuild daemon list` reflects reality and the bind-retry loop has a /// chance to claim the port. See ISSUES.md "Issue B5a". -fn read_stale_daemon_pid() -> Option { +async fn read_stale_daemon_pid() -> Option { let path = fbuild_paths::get_daemon_pid_file(); - let raw = std::fs::read_to_string(&path).ok()?; + let raw = fbuild_core::fs::read_to_string(&path).await.ok()?; let pid: u32 = raw.trim().parse().ok()?; if is_pid_alive(pid) { None diff --git a/crates/fbuild-daemon/src/models.rs b/crates/fbuild-daemon/src/models.rs index 579b7456..73ca181d 100644 --- a/crates/fbuild-daemon/src/models.rs +++ b/crates/fbuild-daemon/src/models.rs @@ -289,6 +289,9 @@ pub struct DeviceInfo { /// Human-readable USB product name (same provenance as `vendor_name`). #[serde(skip_serializing_if = "Option::is_none")] pub product_name: Option, + /// `true` for CDC-ACM, `false` for a USB-serial bridge, `null` if + /// the host could not classify this port. + pub is_cdc: Option, #[serde(skip_serializing_if = "Option::is_none")] pub serial_number: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -447,12 +450,19 @@ pub struct DeviceStatusResponse { pub port: String, pub device_id: String, pub description: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub vid: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub pid: Option, /// Human-readable USB vendor name (only present for USB ports). #[serde(skip_serializing_if = "Option::is_none")] pub vendor_name: Option, /// Human-readable USB product name (only present for USB ports). #[serde(skip_serializing_if = "Option::is_none")] pub product_name: Option, + /// `true` for CDC-ACM, `false` for a USB-serial bridge, `null` if + /// the host could not classify this port. + pub is_cdc: Option, #[serde(skip_serializing_if = "Option::is_none")] pub serial_number: Option, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/crates/fbuild-deploy/src/teensy/usb_type.rs b/crates/fbuild-deploy/src/teensy/usb_type.rs index 6be53698..ca5321cc 100644 --- a/crates/fbuild-deploy/src/teensy/usb_type.rs +++ b/crates/fbuild-deploy/src/teensy/usb_type.rs @@ -110,6 +110,10 @@ mod tests { use super::*; use tempfile::TempDir; + fn tempdir() -> TempDir { + TempDir::new_in(fbuild_paths::temp_subdir("fbuild-deploy-usb-type-tests")).unwrap() + } + #[test] fn classify_known_serial_variants() { assert_eq!( @@ -164,7 +168,7 @@ mod tests { #[test] fn read_usb_type_finds_sibling_file() { - let dir = TempDir::new().unwrap(); + let dir = tempdir(); let fw = dir.path().join("firmware.hex"); fs::write(&fw, b":00000001FF").unwrap(); fs::write(dir.path().join("firmware.usb_type"), b"USB_MIDI_SERIAL\n").unwrap(); @@ -173,7 +177,7 @@ mod tests { #[test] fn read_usb_type_falls_back_to_usb_type_txt() { - let dir = TempDir::new().unwrap(); + let dir = tempdir(); let fw = dir.path().join("firmware.hex"); fs::write(&fw, b":00000001FF").unwrap(); fs::write(dir.path().join("usb_type.txt"), b"USB_SERIAL").unwrap(); @@ -182,7 +186,7 @@ mod tests { #[test] fn read_usb_type_returns_none_when_absent() { - let dir = TempDir::new().unwrap(); + let dir = tempdir(); let fw = dir.path().join("firmware.hex"); fs::write(&fw, b":00000001FF").unwrap(); assert!(read_usb_type_near(&fw).is_none()); diff --git a/crates/fbuild-header-scan/Cargo.toml b/crates/fbuild-header-scan/Cargo.toml index 6bc99006..26003418 100644 --- a/crates/fbuild-header-scan/Cargo.toml +++ b/crates/fbuild-header-scan/Cargo.toml @@ -12,6 +12,7 @@ tracing = { workspace = true } [dev-dependencies] criterion = { workspace = true } +fbuild-paths = { path = "../fbuild-paths" } tempfile = { workspace = true } [[bench]] diff --git a/crates/fbuild-header-scan/src/walker.rs b/crates/fbuild-header-scan/src/walker.rs index 9933fbfa..3845fb3a 100644 --- a/crates/fbuild-header-scan/src/walker.rs +++ b/crates/fbuild-header-scan/src/walker.rs @@ -212,6 +212,10 @@ mod tests { use super::*; use tempfile::TempDir; + fn tempdir() -> TempDir { + TempDir::new_in(fbuild_paths::temp_subdir("fbuild-header-scan-tests")).unwrap() + } + fn write(path: &Path, contents: &str) { if let Some(parent) = path.parent() { std::fs::create_dir_all(parent).unwrap(); @@ -221,7 +225,7 @@ mod tests { #[test] fn w01_quoted_resolves_same_dir_first() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let main = tmp.path().join("main.cpp"); let local = tmp.path().join("foo.h"); let other = tmp.path().join("other").join("foo.h"); @@ -241,7 +245,7 @@ mod tests { #[test] fn w02_angled_skips_same_dir() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let main = tmp.path().join("main.cpp"); let local = tmp.path().join("foo.h"); let other_dir = tmp.path().join("other"); @@ -264,7 +268,7 @@ mod tests { #[test] fn w03_search_path_precedence_first_hit_wins() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let main = tmp.path().join("main.cpp"); let a = tmp.path().join("a"); let b = tmp.path().join("b"); @@ -279,7 +283,7 @@ mod tests { #[test] fn w04_missing_header_goes_to_unresolved() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let main = tmp.path().join("main.cpp"); write(&main, "#include \n"); let res = walk(std::slice::from_ref(&main), &[]); @@ -288,7 +292,7 @@ mod tests { #[test] fn w10_cycle_terminates() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let a = tmp.path().join("a.h"); let b = tmp.path().join("b.h"); write(&a, "#include \"b.h\"\n"); @@ -303,7 +307,7 @@ mod tests { #[test] fn w11_diamond_dedupes() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let main = tmp.path().join("main.cpp"); let a = tmp.path().join("a.h"); let b = tmp.path().join("b.h"); @@ -321,7 +325,7 @@ mod tests { #[test] fn w12_depth_5_chain() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); for i in 1..=5 { let next = if i == 5 { String::new() @@ -341,7 +345,7 @@ mod tests { #[test] fn w20_deterministic_order() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let main = tmp.path().join("main.cpp"); let z = tmp.path().join("z.h"); let a = tmp.path().join("a.h"); diff --git a/crates/fbuild-library-select/Cargo.toml b/crates/fbuild-library-select/Cargo.toml index bdd2d5e8..c4e6e861 100644 --- a/crates/fbuild-library-select/Cargo.toml +++ b/crates/fbuild-library-select/Cargo.toml @@ -21,6 +21,7 @@ prost = { workspace = true } tempfile = { workspace = true } criterion = { workspace = true } fbuild-test-support = { path = "../fbuild-test-support" } +fbuild-paths = { path = "../fbuild-paths" } tracing-test = { workspace = true, features = ["no-env-filter"] } [[bench]] diff --git a/crates/fbuild-library-select/src/cache.rs b/crates/fbuild-library-select/src/cache.rs index f0b23246..b09b491e 100644 --- a/crates/fbuild-library-select/src/cache.rs +++ b/crates/fbuild-library-select/src/cache.rs @@ -29,7 +29,7 @@ use std::path::{Path, PathBuf}; use fbuild_packages::library::FrameworkLibrary; use prost::Message; -use crate::{resolve, Selection}; +use crate::{canon, resolve, Selection}; /// Bump when the scanner's lexical grammar changes in a way that could change /// which `#include` directives it emits for the same source. @@ -231,10 +231,10 @@ pub fn cache_key( let mut seed_pairs: Vec<(String, [u8; 32])> = seeds .iter() .map(|p| { - let canon = std::fs::canonicalize(p).unwrap_or_else(|_| p.clone()); - let bytes = std::fs::read(&canon).unwrap_or_default(); + let canonical = canon(p); + let bytes = std::fs::read(&canonical).unwrap_or_default(); ( - canon.to_string_lossy().into_owned(), + canonical.to_string_lossy().into_owned(), *blake3::hash(&bytes).as_bytes(), ) }) @@ -391,8 +391,12 @@ mod tests { } } + fn tempdir() -> TempDir { + TempDir::new_in(fbuild_paths::temp_subdir("fbuild-library-select-tests")).unwrap() + } + fn build_simple_project() -> (TempDir, Vec, Vec, Vec) { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write(&project_src.join("main.cpp"), "#include \n"); @@ -477,7 +481,7 @@ mod tests { #[test] fn c05_library_input_order_does_not_affect_key() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write(&project_src.join("main.cpp"), "#include \n"); diff --git a/crates/fbuild-library-select/src/lib.rs b/crates/fbuild-library-select/src/lib.rs index c72a467e..cc092039 100644 --- a/crates/fbuild-library-select/src/lib.rs +++ b/crates/fbuild-library-select/src/lib.rs @@ -258,9 +258,13 @@ mod tests { } } + fn tempdir() -> TempDir { + TempDir::new_in(fbuild_paths::temp_subdir("fbuild-library-select-tests")).unwrap() + } + #[test] fn r01_direct_include_selects_library() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write(&project_src.join("main.cpp"), "#include \n"); let mut spi = lib(tmp.path(), "SPI"); @@ -277,7 +281,7 @@ mod tests { #[test] fn r02_transitive_library_selection() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write(&project_src.join("main.cpp"), "#include \n"); @@ -315,7 +319,7 @@ mod tests { // Expected: pass 1 selects {SPI}; pass 2 (with SPI.cpp as a seed) // selects {SPI, Wire}. A regression that drops the second pass would // produce {SPI} only and silently miss Wire at link time. - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write(&project_src.join("main.cpp"), "#include \n"); @@ -345,7 +349,7 @@ mod tests { #[test] fn r03_no_includes_selects_nothing() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write(&project_src.join("main.cpp"), "int main() { return 0; }\n"); let spi = lib(tmp.path(), "SPI"); @@ -358,7 +362,7 @@ mod tests { #[test] fn r13_unrelated_library_not_selected() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write(&project_src.join("main.cpp"), "#include \n"); @@ -381,7 +385,7 @@ mod tests { #[test] fn path_prefix_attribution_distinguishes_same_basename() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write(&project_src.join("main.cpp"), "#include \"foo/config.h\"\n"); @@ -417,7 +421,7 @@ mod tests { // Adversary: no libraries at all. resolve must terminate cleanly with // no required_libraries, no panics, and any reached files limited to // what the walker found from seeds alone. - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write(&project_src.join("main.cpp"), "int main() { return 0; }\n"); let seeds = vec![project_src.join("main.cpp")]; @@ -433,7 +437,7 @@ mod tests { // downloaded). canon() falls back and emits a tracing::warn; the // resolver must not panic and must return a sensible empty // selection. - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write(&project_src.join("main.cpp"), "int main() { return 0; }\n"); let phantom = FrameworkLibrary { @@ -452,7 +456,7 @@ mod tests { // Adversary: 6 libs in deliberately scrambled input order. The // output must be sorted lexicographically, independent of input // order — required for stable cache keys (#205 Phase 4). - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write( &project_src.join("main.cpp"), @@ -488,7 +492,7 @@ mod tests { // The doc on `Selection::required_libraries` and the cache-key story // in #205 Phase 4 both depend on this being a pure function of the // selected *set* of libraries, not their input position. - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let project_src = tmp.path().join("project").join("src"); write( &project_src.join("main.cpp"), diff --git a/crates/fbuild-packages/src/cache.rs b/crates/fbuild-packages/src/cache.rs index ca8ee40d..bb42eef4 100644 --- a/crates/fbuild-packages/src/cache.rs +++ b/crates/fbuild-packages/src/cache.rs @@ -290,6 +290,10 @@ mod tests { use super::*; use tempfile::TempDir; + fn tempdir() -> TempDir { + TempDir::new_in(fbuild_paths::temp_subdir("fbuild-packages-cache-tests")).unwrap() + } + #[test] fn test_hash_url_deterministic() { let h1 = hash_url("https://example.com/package.tar.gz"); @@ -339,21 +343,21 @@ mod tests { #[test] fn test_packages_dir() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::with_cache_root(tmp.path(), tmp.path().join("cache").as_path()); assert!(cache.packages_dir().ends_with("packages")); } #[test] fn test_toolchains_dir() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::with_cache_root(tmp.path(), tmp.path().join("cache").as_path()); assert!(cache.toolchains_dir().ends_with("toolchains")); } #[test] fn global_artifact_roots_stay_under_authoritative_cache_root() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache_root = tmp.path().join("cache-root"); let cache = Cache::with_cache_root(tmp.path(), cache_root.as_path()); @@ -375,7 +379,7 @@ mod tests { #[test] fn test_get_package_path_with_stem_hash() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::with_cache_root(tmp.path(), tmp.path().join("cache").as_path()); let path = cache.get_package_path("https://example.com/pkg.tar.gz", "1.0.0"); let path_str = path.to_string_lossy(); @@ -386,7 +390,7 @@ mod tests { #[test] fn test_get_toolchain_path_with_stem_hash() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::with_cache_root(tmp.path(), tmp.path().join("cache").as_path()); let path = cache.get_toolchain_path("https://downloads.arduino.cc/tools", "7.3.0"); let path_str = path.to_string_lossy(); @@ -398,7 +402,7 @@ mod tests { #[test] fn test_get_build_dir() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::new(tmp.path()); let release_dir = cache.get_build_dir("uno", BuildProfile::Release); assert!(release_dir.to_string_lossy().contains("uno")); @@ -410,7 +414,7 @@ mod tests { #[test] fn test_get_core_build_dir() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::new(tmp.path()); let dir = cache.get_core_build_dir("uno", BuildProfile::Release); assert!(dir.ends_with("core")); @@ -418,7 +422,7 @@ mod tests { #[test] fn test_get_src_build_dir() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::new(tmp.path()); let dir = cache.get_src_build_dir("uno", BuildProfile::Release); assert!(dir.ends_with("src")); @@ -426,7 +430,7 @@ mod tests { #[test] fn test_ensure_directories() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::with_cache_root(tmp.path(), tmp.path().join("cache").as_path()); cache.ensure_directories().unwrap(); assert!(cache.packages_dir().exists()); @@ -437,7 +441,7 @@ mod tests { #[test] fn test_ensure_build_directories() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::new(tmp.path()); cache .ensure_build_directories("uno", BuildProfile::Release) @@ -452,7 +456,7 @@ mod tests { #[test] fn test_clean_build() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::new(tmp.path()); cache .ensure_build_directories("uno", BuildProfile::Release) @@ -464,7 +468,7 @@ mod tests { #[test] fn test_clean_build_nonexistent() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::new(tmp.path()); cache .clean_build("nonexistent", BuildProfile::Release) @@ -473,7 +477,7 @@ mod tests { #[test] fn test_is_package_cached() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::with_cache_root(tmp.path(), tmp.path().join("cache").as_path()); let url = "https://example.com/pkg.tar.gz"; assert!(!cache.is_package_cached(url, "1.0.0")); @@ -485,7 +489,7 @@ mod tests { #[test] fn test_is_toolchain_cached() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::with_cache_root(tmp.path(), tmp.path().join("cache").as_path()); let url = "https://example.com/gcc.tar.gz"; assert!(!cache.is_toolchain_cached(url, "7.3.0")); @@ -497,7 +501,7 @@ mod tests { #[test] fn test_is_toolchain_cached_file_not_dir() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::with_cache_root(tmp.path(), tmp.path().join("cache").as_path()); let url = "https://example.com/gcc.tar.gz"; let path = cache.get_toolchain_path(url, "7.3.0"); @@ -508,7 +512,7 @@ mod tests { #[test] fn test_multiple_environments() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::new(tmp.path()); cache .ensure_build_directories("uno", BuildProfile::Release) @@ -525,7 +529,7 @@ mod tests { #[test] fn test_version_isolation() { - let tmp = TempDir::new().unwrap(); + let tmp = tempdir(); let cache = Cache::with_cache_root(tmp.path(), tmp.path().join("cache").as_path()); let url = "https://example.com/pkg.tar.gz"; let v1 = cache.get_package_path(url, "1.0.0"); diff --git a/crates/fbuild-packages/src/downloader.rs b/crates/fbuild-packages/src/downloader.rs index ca9bd432..4df8e0d2 100644 --- a/crates/fbuild-packages/src/downloader.rs +++ b/crates/fbuild-packages/src/downloader.rs @@ -387,9 +387,16 @@ mod tests { use std::io::Write; use tempfile::NamedTempFile; + fn named_temp_file() -> NamedTempFile { + NamedTempFile::new_in(fbuild_paths::temp_subdir( + "fbuild-packages-downloader-tests", + )) + .unwrap() + } + #[test] fn test_verify_checksum_valid() { - let mut f = NamedTempFile::new().unwrap(); + let mut f = named_temp_file(); f.write_all(b"hello world").unwrap(); f.flush().unwrap(); @@ -400,7 +407,7 @@ mod tests { #[test] fn test_verify_checksum_invalid() { - let mut f = NamedTempFile::new().unwrap(); + let mut f = named_temp_file(); f.write_all(b"hello world").unwrap(); f.flush().unwrap(); @@ -439,7 +446,7 @@ mod tests { Err(_) => break, }; let resp = { - let mut guard = responses.lock().unwrap(); + let mut guard = responses.lock().unwrap_or_else(|err| err.into_inner()); if guard.is_empty() { break; } diff --git a/crates/fbuild-packages/src/lib.rs b/crates/fbuild-packages/src/lib.rs index 0bfd2a8c..bca75ba8 100644 --- a/crates/fbuild-packages/src/lib.rs +++ b/crates/fbuild-packages/src/lib.rs @@ -477,7 +477,7 @@ fn mark_package_touch_needed(key: String) -> bool { #[cfg(test)] fn clear_package_touch_cache_for_tests() { if let Some(touched) = PACKAGE_TOUCHES.get() { - touched.lock().unwrap().clear(); + touched.lock().unwrap_or_else(|e| e.into_inner()).clear(); } } diff --git a/crates/fbuild-packages/src/library/ch32v_core.rs b/crates/fbuild-packages/src/library/ch32v_core.rs index 73ff4ba7..0397ee65 100644 --- a/crates/fbuild-packages/src/library/ch32v_core.rs +++ b/crates/fbuild-packages/src/library/ch32v_core.rs @@ -7,9 +7,9 @@ use std::path::{Path, PathBuf}; use crate::{CacheSubdir, Framework, PackageBase, PackageInfo}; -const CH32V_CORE_VERSION: &str = "1.0.4"; +const CH32V_CORE_VERSION: &str = "1.0.4+d767162.ch32l103"; const CH32V_CORE_URL: &str = - "https://github.com/openwch/arduino_core_ch32/archive/refs/tags/1.0.4.tar.gz"; + "https://github.com/openwch/arduino_core_ch32/archive/d76716239cdf8a084a5045c3dfd3151b3f69eeec.tar.gz"; /// OpenWCH CH32V Arduino core framework manager. pub struct Ch32vCores { diff --git a/crates/fbuild-packages/src/library/library_manager.rs b/crates/fbuild-packages/src/library/library_manager.rs index 261e3ad5..691d3233 100644 --- a/crates/fbuild-packages/src/library/library_manager.rs +++ b/crates/fbuild-packages/src/library/library_manager.rs @@ -25,9 +25,8 @@ fn fallback_runtime() -> Result<&'static Runtime> { if let Some(rt) = RT.get() { return Ok(rt); } - let rt = Runtime::new().map_err(|e| { - FbuildError::PackageError(format!("failed to create tokio runtime: {}", e)) - })?; + let rt = Runtime::new() + .map_err(|e| FbuildError::PackageError(format!("failed to create tokio runtime: {}", e)))?; // If another thread won the race, our `rt` is dropped and we use theirs. Ok(RT.get_or_init(|| rt)) } diff --git a/crates/fbuild-packages/src/lnk/resolver.rs b/crates/fbuild-packages/src/lnk/resolver.rs index 33955470..0e7f630c 100644 --- a/crates/fbuild-packages/src/lnk/resolver.rs +++ b/crates/fbuild-packages/src/lnk/resolver.rs @@ -37,9 +37,8 @@ fn fallback_runtime() -> Result<&'static Runtime> { if let Some(rt) = RT.get() { return Ok(rt); } - let rt = Runtime::new().map_err(|e| { - FbuildError::PackageError(format!("failed to create tokio runtime: {}", e)) - })?; + let rt = Runtime::new() + .map_err(|e| FbuildError::PackageError(format!("failed to create tokio runtime: {}", e)))?; Ok(RT.get_or_init(|| rt)) } diff --git a/crates/fbuild-packages/src/toolchain/esp32_metadata.rs b/crates/fbuild-packages/src/toolchain/esp32_metadata.rs index 13453c3f..b3e04c7e 100644 --- a/crates/fbuild-packages/src/toolchain/esp32_metadata.rs +++ b/crates/fbuild-packages/src/toolchain/esp32_metadata.rs @@ -21,9 +21,8 @@ fn fallback_runtime() -> Result<&'static Runtime> { if let Some(rt) = RT.get() { return Ok(rt); } - let rt = Runtime::new().map_err(|e| { - FbuildError::PackageError(format!("failed to create tokio runtime: {}", e)) - })?; + let rt = Runtime::new() + .map_err(|e| FbuildError::PackageError(format!("failed to create tokio runtime: {}", e)))?; Ok(RT.get_or_init(|| rt)) } diff --git a/crates/fbuild-packages/tests/lnk_e2e.rs b/crates/fbuild-packages/tests/lnk_e2e.rs index 015c91dc..13ca27af 100644 --- a/crates/fbuild-packages/tests/lnk_e2e.rs +++ b/crates/fbuild-packages/tests/lnk_e2e.rs @@ -41,6 +41,10 @@ use fbuild_packages::DiskCache; /// `spawn_blocking`, which is overkill for loopback-only fixtures. const LNK_TEST_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(15); +fn tempdir() -> tempfile::TempDir { + tempfile::TempDir::new_in(fbuild_paths::temp_subdir("fbuild-packages-lnk-e2e-tests")).unwrap() +} + /// Run `body` under `LNK_TEST_TIMEOUT`; panic with a clear message if it /// trips. Keeps each `#[tokio::test]` body free of right-shift indentation. async fn with_timeout(name: &str, body: F) @@ -105,55 +109,56 @@ async fn lnk_pipeline_e2e_fetches_verifies_and_materializes() { with_timeout( "lnk_pipeline_e2e_fetches_verifies_and_materializes", || async { - let blob_bytes = b"hello from the lnk e2e test".to_vec(); - let blob_sha = sha256_of(&blob_bytes); - - let (port, server_handle) = - spawn_test_server(vec![("asset.bin".to_string(), blob_bytes.clone())]).await; - let url = format!("http://127.0.0.1:{port}/asset.bin"); - - // Set up a project tree with one .lnk pointing at our test server. - let work = tempfile::tempdir().unwrap(); - let src_root = work.path().join("src"); - let build_dir = work.path().join("build/resources"); - let cache_dir = work.path().join("cache"); - - let lnk_path = src_root.join("data/asset.bin.lnk"); - std::fs::create_dir_all(lnk_path.parent().unwrap()).unwrap(); - let lnk_json = format!( - r#"{{"v":1,"url":"{url}","sha256":"{blob_sha}","size":{}}}"#, - blob_bytes.len() - ); - std::fs::write(&lnk_path, &lnk_json).unwrap(); - - let cache = DiskCache::open_at(&cache_dir).unwrap(); - - // Scan finds the lnk. - let discovered = scan_for_lnk(&src_root).unwrap(); - assert_eq!(discovered.len(), 1, "scanner should find the one .lnk"); - assert_eq!(discovered[0].lnk.sha256, blob_sha); - - // Materialize fetches + verifies + writes into the build tree. - let materialized = materialize_all(&discovered, &src_root, &build_dir, &cache).unwrap(); - assert_eq!(materialized.len(), 1); - - let target = build_dir.join("data/asset.bin"); - assert!( - target.exists(), - "materialized file should exist at {}", - target.display() - ); - let got = std::fs::read(&target).unwrap(); - assert_eq!(got, blob_bytes, "materialized bytes should match source"); - - // Second materialization is a cache hit — no network would be required. - // (We could shut down the server here to *prove* it, but the cleanest - // assertion is just that it succeeds and the bytes are still right.) - let materialized_again = materialize_all(&discovered, &src_root, &build_dir, &cache).unwrap(); - assert_eq!(materialized_again.len(), 1); - assert_eq!(std::fs::read(&target).unwrap(), blob_bytes); - - server_handle.abort(); + let blob_bytes = b"hello from the lnk e2e test".to_vec(); + let blob_sha = sha256_of(&blob_bytes); + + let (port, server_handle) = + spawn_test_server(vec![("asset.bin".to_string(), blob_bytes.clone())]).await; + let url = format!("http://127.0.0.1:{port}/asset.bin"); + + // Set up a project tree with one .lnk pointing at our test server. + let work = tempdir(); + let src_root = work.path().join("src"); + let build_dir = work.path().join("build/resources"); + let cache_dir = work.path().join("cache"); + + let lnk_path = src_root.join("data/asset.bin.lnk"); + std::fs::create_dir_all(lnk_path.parent().unwrap()).unwrap(); + let lnk_json = format!( + r#"{{"v":1,"url":"{url}","sha256":"{blob_sha}","size":{}}}"#, + blob_bytes.len() + ); + std::fs::write(&lnk_path, &lnk_json).unwrap(); + + let cache = DiskCache::open_at(&cache_dir).unwrap(); + + // Scan finds the lnk. + let discovered = scan_for_lnk(&src_root).unwrap(); + assert_eq!(discovered.len(), 1, "scanner should find the one .lnk"); + assert_eq!(discovered[0].lnk.sha256, blob_sha); + + // Materialize fetches + verifies + writes into the build tree. + let materialized = materialize_all(&discovered, &src_root, &build_dir, &cache).unwrap(); + assert_eq!(materialized.len(), 1); + + let target = build_dir.join("data/asset.bin"); + assert!( + target.exists(), + "materialized file should exist at {}", + target.display() + ); + let got = std::fs::read(&target).unwrap(); + assert_eq!(got, blob_bytes, "materialized bytes should match source"); + + // Second materialization is a cache hit — no network would be required. + // (We could shut down the server here to *prove* it, but the cleanest + // assertion is just that it succeeds and the bytes are still right.) + let materialized_again = + materialize_all(&discovered, &src_root, &build_dir, &cache).unwrap(); + assert_eq!(materialized_again.len(), 1); + assert_eq!(std::fs::read(&target).unwrap(), blob_bytes); + + server_handle.abort(); }, ) .await; @@ -162,45 +167,45 @@ async fn lnk_pipeline_e2e_fetches_verifies_and_materializes() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn lnk_pipeline_rejects_sha_mismatch() { with_timeout("lnk_pipeline_rejects_sha_mismatch", || async { - let blob_bytes = b"actual bytes from server".to_vec(); - let wrong_sha = sha256_of(b"different content"); // claims something else - - let (port, server_handle) = - spawn_test_server(vec![("x.bin".to_string(), blob_bytes.clone())]).await; - let url = format!("http://127.0.0.1:{port}/x.bin"); - - let work = tempfile::tempdir().unwrap(); - let src_root = work.path().join("src"); - let build_dir = work.path().join("build"); - let cache_dir = work.path().join("cache"); - - let lnk_path = src_root.join("x.bin.lnk"); - std::fs::create_dir_all(&src_root).unwrap(); - std::fs::write( - &lnk_path, - format!(r#"{{"v":1,"url":"{url}","sha256":"{wrong_sha}"}}"#), - ) - .unwrap(); - - let cache = DiskCache::open_at(&cache_dir).unwrap(); - let discovered = scan_for_lnk(&src_root).unwrap(); - assert_eq!(discovered.len(), 1); - - let result = materialize_all(&discovered, &src_root, &build_dir, &cache); - let err = result.unwrap_err().to_string(); - assert!( - err.contains("sha256 mismatch"), - "expected sha mismatch error, got: {err}" - ); - - // Build target should NOT exist after a failed verify. - let target = build_dir.join("x.bin"); - assert!( - !target.exists(), - "target should not be materialized on failed verify" - ); - - server_handle.abort(); + let blob_bytes = b"actual bytes from server".to_vec(); + let wrong_sha = sha256_of(b"different content"); // claims something else + + let (port, server_handle) = + spawn_test_server(vec![("x.bin".to_string(), blob_bytes.clone())]).await; + let url = format!("http://127.0.0.1:{port}/x.bin"); + + let work = tempdir(); + let src_root = work.path().join("src"); + let build_dir = work.path().join("build"); + let cache_dir = work.path().join("cache"); + + let lnk_path = src_root.join("x.bin.lnk"); + std::fs::create_dir_all(&src_root).unwrap(); + std::fs::write( + &lnk_path, + format!(r#"{{"v":1,"url":"{url}","sha256":"{wrong_sha}"}}"#), + ) + .unwrap(); + + let cache = DiskCache::open_at(&cache_dir).unwrap(); + let discovered = scan_for_lnk(&src_root).unwrap(); + assert_eq!(discovered.len(), 1); + + let result = materialize_all(&discovered, &src_root, &build_dir, &cache); + let err = result.unwrap_err().to_string(); + assert!( + err.contains("sha256 mismatch"), + "expected sha mismatch error, got: {err}" + ); + + // Build target should NOT exist after a failed verify. + let target = build_dir.join("x.bin"); + assert!( + !target.exists(), + "target should not be materialized on failed verify" + ); + + server_handle.abort(); }) .await; } @@ -208,39 +213,39 @@ async fn lnk_pipeline_rejects_sha_mismatch() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn lnk_pipeline_handles_404() { with_timeout("lnk_pipeline_handles_404", || async { - let (port, server_handle) = spawn_test_server(vec![]).await; - let url = format!("http://127.0.0.1:{port}/nope.bin"); - // 404 still produces *some* response body; sha matches that won't be - // ours. Easier: just refer to a non-existent route and let the - // download succeed (returning the 404 page) but verify will fail. - - let work = tempfile::tempdir().unwrap(); - let src_root = work.path().join("src"); - let build_dir = work.path().join("build"); - let cache_dir = work.path().join("cache"); - std::fs::create_dir_all(&src_root).unwrap(); - - // Sha that won't match the 404 page. - let bogus_sha = "0".repeat(64); - std::fs::write( - src_root.join("nope.bin.lnk"), - format!(r#"{{"v":1,"url":"{url}","sha256":"{bogus_sha}"}}"#), - ) - .unwrap(); - - let cache = DiskCache::open_at(&cache_dir).unwrap(); - let discovered = scan_for_lnk(&src_root).unwrap(); - assert_eq!(discovered.len(), 1); - - // Either the downloader bails on the non-2xx, or we bail on sha verify. - // Both are acceptable failure modes — the assertion is just "errors out". - let result = materialize_all(&discovered, &src_root, &build_dir, &cache); - assert!( - result.is_err(), - "expected error for unreachable/missing blob" - ); - - server_handle.abort(); + let (port, server_handle) = spawn_test_server(vec![]).await; + let url = format!("http://127.0.0.1:{port}/nope.bin"); + // 404 still produces *some* response body; sha matches that won't be + // ours. Easier: just refer to a non-existent route and let the + // download succeed (returning the 404 page) but verify will fail. + + let work = tempdir(); + let src_root = work.path().join("src"); + let build_dir = work.path().join("build"); + let cache_dir = work.path().join("cache"); + std::fs::create_dir_all(&src_root).unwrap(); + + // Sha that won't match the 404 page. + let bogus_sha = "0".repeat(64); + std::fs::write( + src_root.join("nope.bin.lnk"), + format!(r#"{{"v":1,"url":"{url}","sha256":"{bogus_sha}"}}"#), + ) + .unwrap(); + + let cache = DiskCache::open_at(&cache_dir).unwrap(); + let discovered = scan_for_lnk(&src_root).unwrap(); + assert_eq!(discovered.len(), 1); + + // Either the downloader bails on the non-2xx, or we bail on sha verify. + // Both are acceptable failure modes — the assertion is just "errors out". + let result = materialize_all(&discovered, &src_root, &build_dir, &cache); + assert!( + result.is_err(), + "expected error for unreachable/missing blob" + ); + + server_handle.abort(); }) .await; } @@ -250,38 +255,38 @@ async fn lnk_resolver_cache_hit_skips_network_on_second_call() { with_timeout( "lnk_resolver_cache_hit_skips_network_on_second_call", || async { - let blob_bytes = b"cache me".to_vec(); - let sha = sha256_of(&blob_bytes); - - let (port, server_handle) = - spawn_test_server(vec![("y.bin".to_string(), blob_bytes.clone())]).await; - let url = format!("http://127.0.0.1:{port}/y.bin"); - - let work = tempfile::tempdir().unwrap(); - let cache_dir = work.path().join("cache"); - let cache = DiskCache::open_at(&cache_dir).unwrap(); - - // First call: cache miss → download. - let lnk = fbuild_packages::LnkFile { - version: 1, - url: url.clone(), - sha256: sha.clone(), - size: None, - extract: fbuild_packages::ExtractMode::File, - }; - let r1 = fbuild_packages::lnk::resolve(&lnk, &cache).unwrap(); - assert_eq!(r1.sha256, sha); - let blob_path: PathBuf = r1.path.clone(); - assert!(blob_path.exists()); - - // Now shut down the server so we *prove* the second call is offline. - server_handle.abort(); - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - - // Second call: cache hit, no network. - let r2 = fbuild_packages::lnk::resolve(&lnk, &cache).unwrap(); - assert_eq!(r2.sha256, sha); - assert_eq!(r2.path, blob_path); + let blob_bytes = b"cache me".to_vec(); + let sha = sha256_of(&blob_bytes); + + let (port, server_handle) = + spawn_test_server(vec![("y.bin".to_string(), blob_bytes.clone())]).await; + let url = format!("http://127.0.0.1:{port}/y.bin"); + + let work = tempdir(); + let cache_dir = work.path().join("cache"); + let cache = DiskCache::open_at(&cache_dir).unwrap(); + + // First call: cache miss → download. + let lnk = fbuild_packages::LnkFile { + version: 1, + url: url.clone(), + sha256: sha.clone(), + size: None, + extract: fbuild_packages::ExtractMode::File, + }; + let r1 = fbuild_packages::lnk::resolve(&lnk, &cache).unwrap(); + assert_eq!(r1.sha256, sha); + let blob_path: PathBuf = r1.path.clone(); + assert!(blob_path.exists()); + + // Now shut down the server so we *prove* the second call is offline. + server_handle.abort(); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + + // Second call: cache hit, no network. + let r2 = fbuild_packages::lnk::resolve(&lnk, &cache).unwrap(); + assert_eq!(r2.sha256, sha); + assert_eq!(r2.path, blob_path); }, ) .await; diff --git a/crates/fbuild-paths/src/lib.rs b/crates/fbuild-paths/src/lib.rs index edbf78a4..6c398843 100644 --- a/crates/fbuild-paths/src/lib.rs +++ b/crates/fbuild-paths/src/lib.rs @@ -391,13 +391,13 @@ mod tests { #[test] fn find_firmware_returns_none_for_missing_dir() { - let tmp = std::env::temp_dir().join("fbuild_test_find_fw_none"); + let tmp = temp_subdir("fbuild_test_find_fw_none"); assert!(find_firmware(&tmp, "esp32dev", None).is_none()); } #[test] fn find_firmware_finds_bin_in_release_profile() { - let tmp = std::env::temp_dir().join("fbuild_test_find_fw_bin"); + let tmp = temp_subdir("fbuild_test_find_fw_bin"); let fw_dir = tmp .join(".fbuild") .join("build") @@ -419,7 +419,7 @@ mod tests { #[test] fn find_firmware_prefers_release_over_quick() { - let tmp = std::env::temp_dir().join("fbuild_test_find_fw_pref"); + let tmp = temp_subdir("fbuild_test_find_fw_pref"); let release_dir = tmp .join(".fbuild") .join("build") @@ -440,7 +440,7 @@ mod tests { #[test] fn find_firmware_specific_name() { - let tmp = std::env::temp_dir().join("fbuild_test_find_fw_specific"); + let tmp = temp_subdir("fbuild_test_find_fw_specific"); let fw_dir = tmp .join(".fbuild") .join("build") @@ -464,7 +464,7 @@ mod tests { /// firmware in that collapsed layout. See FastLED/fbuild#432. #[test] fn find_firmware_in_collapsed_layout_when_basename_matches_env() { - let tmp = std::env::temp_dir().join("fbuild_test_find_fw_collapsed"); + let tmp = temp_subdir("fbuild_test_find_fw_collapsed"); let _ = std::fs::remove_dir_all(&tmp); let project_dir = tmp.join(".build").join("pio").join("teensy40"); // Collapsed layout: `/.fbuild/build/release/` — @@ -489,7 +489,7 @@ mod tests { #[test] fn find_firmware_legacy_pio_build() { - let tmp = std::env::temp_dir().join("fbuild_test_find_fw_pio"); + let tmp = temp_subdir("fbuild_test_find_fw_pio"); let pio_dir = tmp.join(".pio").join("build").join("uno"); std::fs::create_dir_all(&pio_dir).unwrap(); std::fs::write(pio_dir.join("firmware.hex"), b"legacy").unwrap(); diff --git a/crates/fbuild-paths/src/running_process.rs b/crates/fbuild-paths/src/running_process.rs index 6d351c04..2b51fc9b 100644 --- a/crates/fbuild-paths/src/running_process.rs +++ b/crates/fbuild-paths/src/running_process.rs @@ -239,21 +239,30 @@ fn platform_service_definition_dir() -> PathBuf { #[cfg(target_os = "macos")] fn platform_service_definition_dir() -> PathBuf { - std::env::var_os("HOME") - .map(PathBuf::from) - .unwrap_or_else(std::env::temp_dir) - .join("Library") - .join("Application Support") - .join("running-process") - .join("services") + if let Some(home) = std::env::var_os("HOME") { + return PathBuf::from(home) + .join("Library") + .join("Application Support") + .join("running-process") + .join("services"); + } + fbuild_owned_service_definition_dir() } #[cfg(all(unix, not(target_os = "macos")))] fn platform_service_definition_dir() -> PathBuf { - std::env::var_os("XDG_CONFIG_HOME") + if let Some(config_home) = std::env::var_os("XDG_CONFIG_HOME") .map(PathBuf::from) .or_else(|| std::env::var_os("HOME").map(|home| PathBuf::from(home).join(".config"))) - .unwrap_or_else(std::env::temp_dir) + { + return config_home.join("running-process").join("services"); + } + fbuild_owned_service_definition_dir() +} + +#[cfg(any(target_os = "macos", all(unix, not(target_os = "macos"))))] +fn fbuild_owned_service_definition_dir() -> PathBuf { + crate::get_cache_root() .join("running-process") .join("services") } @@ -310,6 +319,24 @@ mod tests { ); } + #[cfg(all(unix, not(target_os = "macos")))] + #[test] + fn service_definition_dir_falls_back_to_fbuild_cache_root_without_home() { + let _env = ENV_LOCK.lock().unwrap(); + let cache_root = crate::temp_subdir(&format!( + "fbuild-service-def-cache-root-{}", + std::process::id() + )); + let _cache_guard = EnvVarGuard::set(FBUILD_CACHE_DIR_ENV, &cache_root); + let _xdg_guard = EnvVarGuard::remove("XDG_CONFIG_HOME"); + let _home_guard = EnvVarGuard::remove("HOME"); + + assert_eq!( + platform_service_definition_dir(), + cache_root.join("running-process").join("services") + ); + } + #[test] fn broker_requested_mode_is_still_direct_fallback_for_this_slice() { let mode = RunningProcessDaemonMode::BrokerRequested; @@ -341,8 +368,7 @@ mod tests { #[test] fn cache_roots_respect_fbuild_cache_dir_as_artifact_owner() { let _env = ENV_LOCK.lock().unwrap(); - let cache_root = - std::env::temp_dir().join(format!("fbuild-cache-roots-{}", std::process::id())); + let cache_root = crate::temp_subdir(&format!("fbuild-cache-roots-{}", std::process::id())); let _cache_guard = EnvVarGuard::set(FBUILD_CACHE_DIR_ENV, &cache_root); let runtime = PathBuf::from("/opt/fbuild/bin"); @@ -361,7 +387,7 @@ mod tests { fn cache_roots_keep_artifacts_stable_across_runtime_dirs() { let _env = ENV_LOCK.lock().unwrap(); let cache_root = - std::env::temp_dir().join(format!("fbuild-cache-roots-stable-{}", std::process::id())); + crate::temp_subdir(&format!("fbuild-cache-roots-stable-{}", std::process::id())); let _cache_guard = EnvVarGuard::set(FBUILD_CACHE_DIR_ENV, &cache_root); let runtime_v1 = PathBuf::from("/opt/fbuild-1/bin"); let runtime_v2 = PathBuf::from("/opt/fbuild-2/bin"); diff --git a/crates/fbuild-python/src/daemon.rs b/crates/fbuild-python/src/daemon.rs index f48dc34f..ab001f45 100644 --- a/crates/fbuild-python/src/daemon.rs +++ b/crates/fbuild-python/src/daemon.rs @@ -228,7 +228,11 @@ fn broker_refusal_is_fatal(kind: Option) -> bool { /// `AsyncDaemon::ensure_running` (via `pyo3_async_runtimes::tokio`). The /// process spawn must be resolved against the venv before this is called. /// FastLED/fbuild#817. -async fn ensure_running_async_impl(url: &str, spawn_target: Option, dev_mode: bool) -> bool { +async fn ensure_running_async_impl( + url: &str, + spawn_target: Option, + dev_mode: bool, +) -> bool { match ensure_running_via_broker_async(url).await { Ok(true) => return true, Ok(false) => {} @@ -361,8 +365,7 @@ impl Daemon { #[staticmethod] fn status(py: Python<'_>) -> PyResult { // FastLED/fbuild#817: sync wrapper around `status_async_impl`. - let rt = one_shot_runtime() - .map_err(pyo3::exceptions::PyRuntimeError::new_err)?; + let rt = one_shot_runtime().map_err(pyo3::exceptions::PyRuntimeError::new_err)?; let text = rt.block_on(status_async_impl())?; let json_module = py.import_bound("json")?; let result = json_module.call_method1("loads", (text,))?; @@ -435,9 +438,7 @@ impl AsyncDaemon { /// Returns `True` if the daemon acknowledged with a 2xx response. #[staticmethod] fn stop(py: Python<'_>) -> PyResult> { - pyo3_async_runtimes::tokio::future_into_py(py, async move { - Ok(stop_async_impl().await) - }) + pyo3_async_runtimes::tokio::future_into_py(py, async move { Ok(stop_async_impl().await) }) } } diff --git a/crates/fbuild-serial/src/boards.rs b/crates/fbuild-serial/src/boards.rs index 02397f92..8873821b 100644 --- a/crates/fbuild-serial/src/boards.rs +++ b/crates/fbuild-serial/src/boards.rs @@ -589,17 +589,15 @@ pub fn family_from_upload_hint(hint: &UploadHint) -> Option { "picotool" => return Some(NativeUsbCdcReset1200Bps), // PJRC — teensy_loader_cli / Teensy Loader GUI. HalfKay // bootloader entered via 1200-bps touch. - "teensy-gui" | "teensy_gui" | "teensy-cli" | "teensy_cli" - | "teensy_loader_cli" => { + "teensy-gui" | "teensy_gui" | "teensy-cli" | "teensy_cli" | "teensy_loader_cli" => { return Some(Teensy); } // SWD-side reset. The physical serial VCOM bridge in front // of the target MCU wants DTR=true/RTS=true (host-ready); // the reset is dispatched by the debug probe over SWD, // NOT DTR/RTS. - "cmsis-dap" | "cmsis_dap" | "jlink" | "j-link" | "stlink" - | "st-link" | "raspberrypi-swd" | "raspberrypi_swd" - | "atmel-ice" | "atmel_ice" | "openocd" => { + "cmsis-dap" | "cmsis_dap" | "jlink" | "j-link" | "stlink" | "st-link" + | "raspberrypi-swd" | "raspberrypi_swd" | "atmel-ice" | "atmel_ice" | "openocd" => { return Some(CdcAcmBridge); } _ => {} // fall through to touch-flag fallback @@ -681,9 +679,10 @@ pub fn family_for_port(name: &str) -> Option { let vid_pid_lookup = lookup_port_vid_pid(name); let kernel_class = crate::port_class::detect_port_kernel_class(name); - if let (Some((vid, pid, Some(vp_family))), Some(kc)) = - (vid_pid_lookup.as_ref().copied(), kernel_class.as_ref().copied()) - { + if let (Some((vid, pid, Some(vp_family))), Some(kc)) = ( + vid_pid_lookup.as_ref().copied(), + kernel_class.as_ref().copied(), + ) { warn_if_class_disagrees(name, vid, pid, vp_family, kc); } @@ -919,7 +918,10 @@ mod tests { assert_eq!(family_for_vid_pid(0x2341, 0x0001), Some(ArduinoAutoReset)); assert!(!vp_family_implies_cdc(ArduinoAutoReset)); // 0x2E8A — RP2040 native CDC → CDC ✓ - assert_eq!(family_for_vid_pid(0x2E8A, 0x000A), Some(NativeUsbCdcReset1200Bps)); + assert_eq!( + family_for_vid_pid(0x2E8A, 0x000A), + Some(NativeUsbCdcReset1200Bps) + ); assert!(vp_family_implies_cdc(NativeUsbCdcReset1200Bps)); } diff --git a/crates/fbuild-serial/src/port_class.rs b/crates/fbuild-serial/src/port_class.rs index f3448dbc..80a52fcd 100644 --- a/crates/fbuild-serial/src/port_class.rs +++ b/crates/fbuild-serial/src/port_class.rs @@ -131,7 +131,7 @@ pub fn detect_port_kernel_class(port_name: &str) -> Option { #[cfg(target_os = "linux")] mod linux { use super::PortKernelClass; - use std::path::{Path, PathBuf}; + use std::path::Path; pub(super) fn detect(port_name: &str) -> Option { detect_with_sysfs_root(port_name, Path::new("/sys")) @@ -150,9 +150,7 @@ mod linux { // e.g. -> .../bus/usb-serial/drivers/cdc_acm // -> .../bus/usb-serial/drivers/ftdi_sio // -> .../bus/usb-serial/drivers/cp210x - if let Some(driver_name) = - read_driver_symlink_name(sysfs_root, bare) - { + if let Some(driver_name) = read_driver_symlink_name(sysfs_root, bare) { return Some(classify_driver(&driver_name)); } @@ -166,7 +164,7 @@ mod linux { } fn read_driver_symlink_name(sysfs_root: &Path, port_stem: &str) -> Option { - let driver_link: PathBuf = sysfs_root + let driver_link = sysfs_root .join("class") .join("tty") .join(port_stem) @@ -220,7 +218,9 @@ mod linux { } #[cfg(test)] - pub(super) use port_name_stem as port_name_stem_for_tests; + pub(super) fn port_name_stem_for_tests(port_name: &str) -> Option<&str> { + port_name_stem(port_name) + } } #[cfg(target_os = "macos")] @@ -290,11 +290,7 @@ mod tests { // Build a fake `/sys/class/tty//device/driver` symlink // pointing at a fake driver dir under a tmp root, then ask // detect_with_sysfs_root to classify it. - fn build_fake_sysfs_tree( - sysfs_root: &std::path::Path, - port_stem: &str, - driver_name: &str, - ) { + fn build_fake_sysfs_tree(sysfs_root: &std::path::Path, port_stem: &str, driver_name: &str) { let device_dir = sysfs_root .join("class") .join("tty") @@ -380,7 +376,10 @@ mod tests { // either — the kernel didn't bind it via cdc_acm or // usbserial. let tmp = tempdir().unwrap(); - assert_eq!(linux::detect_with_sysfs_root("/dev/ttyS0", tmp.path()), None); + assert_eq!( + linux::detect_with_sysfs_root("/dev/ttyS0", tmp.path()), + None + ); } #[test] @@ -489,10 +488,7 @@ mod tests { None ); // Stray random name returns None too. - assert_eq!( - macos::classify_macos_devnode("/dev/cu.random-thing"), - None - ); + assert_eq!(macos::classify_macos_devnode("/dev/cu.random-thing"), None); } } diff --git a/crates/fbuild-serial/src/preemption.rs b/crates/fbuild-serial/src/preemption.rs index 1ecfbb00..f37527fa 100644 --- a/crates/fbuild-serial/src/preemption.rs +++ b/crates/fbuild-serial/src/preemption.rs @@ -42,13 +42,12 @@ impl PreemptionTracker { pub async fn preempt(&self, port: &str, reason: String, preempted_by: String) { // No `.await` inside this critical section — `std::sync::Mutex` - // is the right choice. `expect` is safe: the only path that - // could poison the lock is a panic while holding it, and we - // hold it only across a single `HashMap::insert`. + // is the right choice. If a previous holder panicked, keep the + // tracker usable and recover the inner map. let mut ports = self .preempted_ports .lock() - .expect("preemption mutex poisoned"); + .unwrap_or_else(|err| err.into_inner()); ports.insert( port.to_string(), PreemptionInfo { @@ -63,7 +62,7 @@ impl PreemptionTracker { let mut ports = self .preempted_ports .lock() - .expect("preemption mutex poisoned"); + .unwrap_or_else(|err| err.into_inner()); ports.remove(port); } @@ -71,7 +70,7 @@ impl PreemptionTracker { let ports = self .preempted_ports .lock() - .expect("preemption mutex poisoned"); + .unwrap_or_else(|err| err.into_inner()); ports.contains_key(port) } } diff --git a/crates/fbuild-test-support/Cargo.toml b/crates/fbuild-test-support/Cargo.toml index d605b5b6..121b02cb 100644 --- a/crates/fbuild-test-support/Cargo.toml +++ b/crates/fbuild-test-support/Cargo.toml @@ -16,6 +16,7 @@ tracing = { workspace = true } tracing-subscriber = { workspace = true } tokio = { workspace = true } fbuild-config = { path = "../fbuild-config" } +fbuild-core = { path = "../fbuild-core" } fbuild-packages = { path = "../fbuild-packages" } fbuild-paths = { path = "../fbuild-paths" } fbuild-header-scan = { path = "../fbuild-header-scan" } diff --git a/crates/fbuild-test-support/src/elf_probe.rs b/crates/fbuild-test-support/src/elf_probe.rs index 5be819b8..3c13e4ca 100644 --- a/crates/fbuild-test-support/src/elf_probe.rs +++ b/crates/fbuild-test-support/src/elf_probe.rs @@ -387,7 +387,8 @@ mod tests { #[test] fn open_returns_io_error_for_missing_file() { - let path = std::env::temp_dir().join("fbuild-elfprobe-no-such-file-xyz123.elf"); + let path = fbuild_paths::temp_subdir("fbuild-test-support-elf-probe") + .join("fbuild-elfprobe-no-such-file-xyz123.elf"); // Ensure it really doesn't exist. let _ = std::fs::remove_file(&path); match ElfProbe::open(&path) { diff --git a/crates/fbuild-test-support/src/mini_framework.rs b/crates/fbuild-test-support/src/mini_framework.rs index b120a716..53a88a12 100644 --- a/crates/fbuild-test-support/src/mini_framework.rs +++ b/crates/fbuild-test-support/src/mini_framework.rs @@ -433,8 +433,8 @@ mod tests { assert_eq!(std::fs::read_to_string(&main).unwrap(), "// sketch\n"); } - #[test] - fn walker_round_trip() { + #[tokio::test] + async fn walker_round_trip() { let mut fx = MiniFramework::new(); fx.add_library("SPI").done(); fx.sketch("#include \n"); @@ -449,10 +449,15 @@ mod tests { } let res = walk(&fx.project_seeds(), &search_paths); - let spi_h = std::fs::canonicalize(fx.libraries_dir().join("SPI").join("src").join("SPI.h")) - .unwrap(); + let spi_h = fbuild_core::path::canonicalize_existing( + fx.libraries_dir().join("SPI").join("src").join("SPI.h"), + ) + .await + .unwrap(); assert!( - res.reached.contains(&spi_h), + res.reached + .iter() + .any(|p| fbuild_core::path::NormalizedPath::new(p).key() == spi_h.key()), "walker did not reach SPI.h via fixture; reached={:?}", res.reached, ); diff --git a/docs/online-data.md b/docs/online-data.md index e7105f38..a77c8ae0 100644 --- a/docs/online-data.md +++ b/docs/online-data.md @@ -7,6 +7,7 @@ currently published: | Dataset | Path | Description | |---|---|---| | `usb-vid` | `data/usb-vid.json` | USB VID:PID → `{vendor, product}` (union of multiple sources) | +| `usb-vids.proto.zstd` | `data/usb-vids.proto.zstd` | Compact protobuf/zstd form of `usb-vid.json` used by fbuild runtime scans | | `usb-vid-conflicts` | `data/usb-vid-conflicts.json` | Per-key disagreements between USB-VID sources (observability) | | `pio-boards` | `data/pio-boards.json` | Full PlatformIO board catalog (vendor, mcu, frameworks, debug tools, etc.) | | `vendor_boards` | `data/vendor_boards.json` | Slim view of `pio-boards` — only `{vendor, name, mcu}` per board id, for cheap "what board is plugged in?" lookups | @@ -14,11 +15,20 @@ currently published: The format is **future-forward** — new datasets are added by writing a new JSON file under `data/`; `tools/build_manifest.py` auto-discovers them on the next workflow run. No client breakage when datasets are added. +Binary companion files such as `usb-vids.proto.zstd` are published under +`data/` too, but are consumed through explicit runtime constants rather than +manifest auto-discovery. The companion in-process USB resolver lives at `fbuild_core::usb` — see `crates/fbuild-core/src/usb/`. The branch is the **tier-2 fallback** when the bundled `usb-ids` crate doesn't know a VID:PID. +VID/PID product rows are USB-name metadata, not board-support proof. Board +existence remains governed by `crates/fbuild-config/assets/boards`; if a board +is absent there, a third-party SDK or board-package row is treated only as a +weak supplement that can fill product-name gaps after first-party, vendor, and +generic USB-ID sources have already won. + ## URLs Always start from the manifest — direct dataset URLs may change in the @@ -28,6 +38,8 @@ future, but the manifest's `datasets..url` field is the contract. `https://raw.githubusercontent.com/fastled/fbuild/online-data/manifest.json` - USB VID:PID dataset: `https://raw.githubusercontent.com/fastled/fbuild/online-data/data/usb-vid.json` +- Compact USB VID:PID protobuf/zstd runtime overlay: + `https://raw.githubusercontent.com/fastled/fbuild/online-data/data/usb-vids.proto.zstd` - USB-VID source-conflict log: `https://raw.githubusercontent.com/fastled/fbuild/online-data/data/usb-vid-conflicts.json` - PlatformIO full board catalog: @@ -35,8 +47,9 @@ future, but the manifest's `datasets..url` field is the contract. - PlatformIO slim vendor-name lookup (small, ~200 KB): `https://raw.githubusercontent.com/fastled/fbuild/online-data/data/vendor_boards.json` -The matching constants in code: `fbuild_core::usb::MANIFEST_URL` and -`fbuild_core::usb::USB_VID_JSON_URL`. +The matching constants in code: `fbuild_core::usb::MANIFEST_URL`, +`fbuild_core::usb::USB_VID_JSON_URL`, and +`fbuild_core::usb::USB_VIDS_PROTO_ZSTD_URL`. ## Branch shape @@ -46,6 +59,7 @@ online-data (orphan, NEVER merged into main) ├── manifest.json ├── data/ │ ├── usb-vid.json # alphabetically sorted, lowercase hex keys +│ ├── usb-vids.proto.zstd # compact runtime overlay consumed by fbuild │ └── usb-vid-conflicts.json # only keys where sources disagreed └── tools/ ├── README.md @@ -79,11 +93,13 @@ Per run: the run). 6. `uv run --no-project --script .online-data/tools/merge_sources.py …` over whichever sources arrived intact. The merger: - - takes the union, prefers `usb-ids-rs` > `linux-usb.org` > `usbids-github` - on conflict; + - takes the union in workflow argument order; first-party/vendor-owned + sources are ordered before generic USB-ID feeds, while third-party SDK + and board-package supplements are ordered after them so they only fill + missing rows; - sorts keys alphabetically (lowercase `vvvv:pppp`); - writes `data/usb-vid.json`, `data/usb-vid-conflicts.json`, - and the freshly-stamped `manifest.json`; + `data/usb-vids.proto.zstd`, and the freshly-stamped `manifest.json`; - **refuses to write** if the union has fewer than 1000 entries so a truncated upstream cannot blow away a healthy committed dataset. 7. If files actually changed, commit on `online-data`. @@ -102,6 +118,8 @@ Manual trigger: Actions → "Update data" → Run workflow. - **All upstream sources fail** → merger refuses to write → workflow finishes with no commit; existing committed data is preserved. - **Merger writes too-small output** → same as above (sanity floor). +- **Compact protobuf generation fails after a successful USB merge** → + workflow stops before publishing a mismatched `online-data` commit. - **Workflow itself fails before commit** → previous commit on `online-data` remains the live data. diff --git a/dylints/ban_env_var_set_after_import/src/lib.rs b/dylints/ban_env_var_set_after_import/src/lib.rs index 58ee6ad2..da25639a 100644 --- a/dylints/ban_env_var_set_after_import/src/lib.rs +++ b/dylints/ban_env_var_set_after_import/src/lib.rs @@ -8,7 +8,10 @@ extern crate rustc_span; use rustc_errors::DiagDecorator; use rustc_hir::{def::Res, Expr, ExprKind, HirId}; use rustc_lint::{LateContext, LateLintPass, LintContext}; -use rustc_span::{symbol::Symbol, FileName, RemapPathScopeComponents}; +use rustc_span::{ + symbol::{sym, Symbol}, + FileName, RemapPathScopeComponents, +}; dylint_linting::declare_late_lint! { /// ### What it does @@ -137,31 +140,29 @@ fn emit_lint(cx: &LateContext<'_>, span: rustc_span::Span, banned: &[&str]) { } fn owned_by_cfg_test_module(cx: &LateContext<'_>, hir_id: HirId) -> bool { - let mut current = cx.tcx.hir_get_parent_item(hir_id); - loop { - let attrs = cx.tcx.hir_attrs(current.into()); - for attr in attrs { - if attr_is_cfg_test(attr) { - return true; - } - } - let parent = cx.tcx.hir_get_parent_item(current.into()); - if parent == current { - return false; - } - current = parent; - } + std::iter::once(hir_id) + .chain(cx.tcx.hir_parent_id_iter(hir_id)) + .any(|id| { + cx.tcx.hir_attrs(id).iter().any(attr_is_cfg_test) || is_test_module_node(cx, id) + }) } -fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { - let Some(meta) = attr.meta() else { +fn is_test_module_node(cx: &LateContext<'_>, hir_id: HirId) -> bool { + let rustc_hir::Node::Item(item) = cx.tcx.hir_node(hir_id) else { return false; }; - let path = meta.path(); - if path.segments.len() != 1 || path.segments[0].ident.as_str() != "cfg" { + let rustc_hir::ItemKind::Mod(ident, _) = item.kind else { + return false; + }; + let name = ident.name.as_str(); + name == "tests" || name.ends_with("_tests") || name.ends_with("_test") +} + +fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { + if !attr.has_name(sym::cfg) { return false; } - let Some(list) = meta.meta_item_list() else { + let Some(list) = attr.meta_item_list() else { return false; }; list.iter().any(|nested| { diff --git a/dylints/ban_poison_panic/src/lib.rs b/dylints/ban_poison_panic/src/lib.rs index e1f30793..d1a02eff 100644 --- a/dylints/ban_poison_panic/src/lib.rs +++ b/dylints/ban_poison_panic/src/lib.rs @@ -9,7 +9,10 @@ use rustc_errors::DiagDecorator; use rustc_hir::{def::Res, Expr, ExprKind, HirId}; use rustc_lint::{LateContext, LateLintPass, LintContext}; use rustc_middle::ty; -use rustc_span::{symbol::Symbol, FileName, RemapPathScopeComponents}; +use rustc_span::{ + symbol::{sym, Symbol}, + FileName, RemapPathScopeComponents, +}; dylint_linting::declare_late_lint! { /// ### What it does @@ -160,31 +163,29 @@ fn emit_lint(cx: &LateContext<'_>, span: rustc_span::Span) { } fn owned_by_cfg_test_module(cx: &LateContext<'_>, hir_id: HirId) -> bool { - let mut current = cx.tcx.hir_get_parent_item(hir_id); - loop { - let attrs = cx.tcx.hir_attrs(current.into()); - for attr in attrs { - if attr_is_cfg_test(attr) { - return true; - } - } - let parent = cx.tcx.hir_get_parent_item(current.into()); - if parent == current { - return false; - } - current = parent; - } + std::iter::once(hir_id) + .chain(cx.tcx.hir_parent_id_iter(hir_id)) + .any(|id| { + cx.tcx.hir_attrs(id).iter().any(attr_is_cfg_test) || is_test_module_node(cx, id) + }) } -fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { - let Some(meta) = attr.meta() else { +fn is_test_module_node(cx: &LateContext<'_>, hir_id: HirId) -> bool { + let rustc_hir::Node::Item(item) = cx.tcx.hir_node(hir_id) else { return false; }; - let path = meta.path(); - if path.segments.len() != 1 || path.segments[0].ident.as_str() != "cfg" { + let rustc_hir::ItemKind::Mod(ident, _) = item.kind else { + return false; + }; + let name = ident.name.as_str(); + name == "tests" || name.ends_with("_tests") || name.ends_with("_test") +} + +fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { + if !attr.has_name(sym::cfg) { return false; } - let Some(list) = meta.meta_item_list() else { + let Some(list) = attr.meta_item_list() else { return false; }; list.iter().any(|nested| { diff --git a/dylints/ban_runtime_new_outside_main/src/lib.rs b/dylints/ban_runtime_new_outside_main/src/lib.rs index 3cd5fe66..8467b5b9 100644 --- a/dylints/ban_runtime_new_outside_main/src/lib.rs +++ b/dylints/ban_runtime_new_outside_main/src/lib.rs @@ -8,7 +8,10 @@ extern crate rustc_span; use rustc_errors::DiagDecorator; use rustc_hir::{def::Res, Expr, ExprKind, HirId}; use rustc_lint::{LateContext, LateLintPass, LintContext}; -use rustc_span::{symbol::Symbol, FileName, RemapPathScopeComponents}; +use rustc_span::{ + symbol::{sym, Symbol}, + FileName, RemapPathScopeComponents, +}; dylint_linting::declare_late_lint! { /// ### What it does @@ -130,31 +133,29 @@ fn emit_lint(cx: &LateContext<'_>, span: rustc_span::Span, banned: &[&str]) { } fn owned_by_cfg_test_module(cx: &LateContext<'_>, hir_id: HirId) -> bool { - let mut current = cx.tcx.hir_get_parent_item(hir_id); - loop { - let attrs = cx.tcx.hir_attrs(current.into()); - for attr in attrs { - if attr_is_cfg_test(attr) { - return true; - } - } - let parent = cx.tcx.hir_get_parent_item(current.into()); - if parent == current { - return false; - } - current = parent; - } + std::iter::once(hir_id) + .chain(cx.tcx.hir_parent_id_iter(hir_id)) + .any(|id| { + cx.tcx.hir_attrs(id).iter().any(attr_is_cfg_test) || is_test_module_node(cx, id) + }) } -fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { - let Some(meta) = attr.meta() else { +fn is_test_module_node(cx: &LateContext<'_>, hir_id: HirId) -> bool { + let rustc_hir::Node::Item(item) = cx.tcx.hir_node(hir_id) else { return false; }; - let path = meta.path(); - if path.segments.len() != 1 || path.segments[0].ident.as_str() != "cfg" { + let rustc_hir::ItemKind::Mod(ident, _) = item.kind else { + return false; + }; + let name = ident.name.as_str(); + name == "tests" || name.ends_with("_tests") || name.ends_with("_test") +} + +fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { + if !attr.has_name(sym::cfg) { return false; } - let Some(list) = meta.meta_item_list() else { + let Some(list) = attr.meta_item_list() else { return false; }; list.iter().any(|nested| { diff --git a/dylints/ban_std_fs_in_async/src/allowlist.txt b/dylints/ban_std_fs_in_async/src/allowlist.txt index 0ab04468..97fdfaaa 100644 --- a/dylints/ban_std_fs_in_async/src/allowlist.txt +++ b/dylints/ban_std_fs_in_async/src/allowlist.txt @@ -13,3 +13,10 @@ # axum workers — the opposite of what this lint exists to encourage. crates/fbuild-daemon/src/handlers/operations/common.rs crates/fbuild-daemon/src/handlers/emulator/avr8js_npm.rs + +# Synchronous daemon lifecycle helpers. The broker backend is hosted on a +# dedicated std::thread, not a tokio worker. StatusManager intentionally writes +# synchronously via write_atomic_sync so daemon status updates are safe from +# current-thread test runtimes (FastLED/fbuild#865). +crates/fbuild-daemon/src/broker/backend.rs +crates/fbuild-daemon/src/status_manager.rs diff --git a/dylints/ban_std_sync_mutex_in_async/src/lib.rs b/dylints/ban_std_sync_mutex_in_async/src/lib.rs index be1b6608..18243227 100644 --- a/dylints/ban_std_sync_mutex_in_async/src/lib.rs +++ b/dylints/ban_std_sync_mutex_in_async/src/lib.rs @@ -8,7 +8,10 @@ extern crate rustc_span; use rustc_errors::DiagDecorator; use rustc_hir::{def::Res, AmbigArg, Expr, ExprKind, HirId, Ty, TyKind}; use rustc_lint::{LateContext, LateLintPass, LintContext}; -use rustc_span::{symbol::Symbol, FileName, RemapPathScopeComponents}; +use rustc_span::{ + symbol::{sym, Symbol}, + FileName, RemapPathScopeComponents, +}; dylint_linting::declare_late_lint! { /// ### What it does @@ -128,34 +131,31 @@ impl<'tcx> LateLintPass<'tcx> for BanStdSyncMutexInAsync { } } -/// Walk up the HIR owner chain looking for a module annotated with -/// `#[cfg(test)]`. Mirrors `ban_unwrap_in_production`. +/// Walk up the HIR parent chain looking for `#[cfg(test)]`. fn owned_by_cfg_test_module(cx: &LateContext<'_>, hir_id: HirId) -> bool { - let mut current = cx.tcx.hir_get_parent_item(hir_id); - loop { - let attrs = cx.tcx.hir_attrs(current.into()); - for attr in attrs { - if attr_is_cfg_test(attr) { - return true; - } - } - let parent = cx.tcx.hir_get_parent_item(current.into()); - if parent == current { - return false; - } - current = parent; - } + std::iter::once(hir_id) + .chain(cx.tcx.hir_parent_id_iter(hir_id)) + .any(|id| { + cx.tcx.hir_attrs(id).iter().any(attr_is_cfg_test) || is_test_module_node(cx, id) + }) } -fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { - let Some(meta) = attr.meta() else { +fn is_test_module_node(cx: &LateContext<'_>, hir_id: HirId) -> bool { + let rustc_hir::Node::Item(item) = cx.tcx.hir_node(hir_id) else { return false; }; - let path = meta.path(); - if path.segments.len() != 1 || path.segments[0].ident.as_str() != "cfg" { + let rustc_hir::ItemKind::Mod(ident, _) = item.kind else { + return false; + }; + let name = ident.name.as_str(); + name == "tests" || name.ends_with("_tests") || name.ends_with("_test") +} + +fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { + if !attr.has_name(sym::cfg) { return false; } - let Some(list) = meta.meta_item_list() else { + let Some(list) = attr.meta_item_list() else { return false; }; list.iter().any(|nested| { diff --git a/dylints/ban_unrooted_tempdir/src/lib.rs b/dylints/ban_unrooted_tempdir/src/lib.rs index f296bcb9..e57da95e 100644 --- a/dylints/ban_unrooted_tempdir/src/lib.rs +++ b/dylints/ban_unrooted_tempdir/src/lib.rs @@ -70,6 +70,9 @@ impl<'tcx> LateLintPass<'tcx> for BanUnrootedTempdir { if is_allowlisted(cx, expr.span) { return; } + if is_unit_test_module_scope(cx, expr.hir_id) { + return; + } if let ExprKind::Path(qpath) = expr.kind { let res = cx.qpath_res(&qpath, expr.hir_id); @@ -85,6 +88,23 @@ impl<'tcx> LateLintPass<'tcx> for BanUnrootedTempdir { } } +fn is_unit_test_module_scope(cx: &LateContext<'_>, hir_id: rustc_hir::HirId) -> bool { + std::iter::once(hir_id) + .chain(cx.tcx.hir_parent_id_iter(hir_id)) + .any(|id| is_test_module_node(cx, id)) +} + +fn is_test_module_node(cx: &LateContext<'_>, hir_id: rustc_hir::HirId) -> bool { + let rustc_hir::Node::Item(item) = cx.tcx.hir_node(hir_id) else { + return false; + }; + let rustc_hir::ItemKind::Mod(ident, _) = item.kind else { + return false; + }; + let name = ident.name.as_str(); + name == "tests" || name.ends_with("_tests") || name.ends_with("_test") +} + fn emit_lint(cx: &LateContext<'_>, span: rustc_span::Span, banned: &[&str]) { let joined = banned.join("::"); cx.opt_span_lint( diff --git a/dylints/ban_unwrap_in_production/src/lib.rs b/dylints/ban_unwrap_in_production/src/lib.rs index 69bb613c..434194fd 100644 --- a/dylints/ban_unwrap_in_production/src/lib.rs +++ b/dylints/ban_unwrap_in_production/src/lib.rs @@ -8,7 +8,10 @@ extern crate rustc_span; use rustc_errors::DiagDecorator; use rustc_hir::{Expr, ExprKind, HirId}; use rustc_lint::{LateContext, LateLintPass, LintContext}; -use rustc_span::{symbol::Symbol, FileName, RemapPathScopeComponents}; +use rustc_span::{ + symbol::{sym, Symbol}, + FileName, RemapPathScopeComponents, +}; dylint_linting::declare_late_lint! { /// ### What it does @@ -125,37 +128,31 @@ impl<'tcx> LateLintPass<'tcx> for BanUnwrapInProduction { } } -/// Walk up the HIR owner chain looking for a module annotated with -/// `#[cfg(test)]`. We deliberately only look at module items — -/// `#[cfg(test)]` on a function is the right wrong-shape to nudge -/// the developer into `mod tests { ... }`. +/// Walk up the HIR parent chain looking for `#[cfg(test)]`. fn owned_by_cfg_test_module(cx: &LateContext<'_>, hir_id: HirId) -> bool { - let mut current = cx.tcx.hir_get_parent_item(hir_id); - loop { - // `is_in_test_module` via attribute scan: - let attrs = cx.tcx.hir_attrs(current.into()); - for attr in attrs { - if attr_is_cfg_test(attr) { - return true; - } - } - let parent = cx.tcx.hir_get_parent_item(current.into()); - if parent == current { - return false; - } - current = parent; - } + std::iter::once(hir_id) + .chain(cx.tcx.hir_parent_id_iter(hir_id)) + .any(|id| { + cx.tcx.hir_attrs(id).iter().any(attr_is_cfg_test) || is_test_module_node(cx, id) + }) } -fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { - let Some(meta) = attr.meta() else { +fn is_test_module_node(cx: &LateContext<'_>, hir_id: HirId) -> bool { + let rustc_hir::Node::Item(item) = cx.tcx.hir_node(hir_id) else { return false; }; - let path = meta.path(); - if path.segments.len() != 1 || path.segments[0].ident.as_str() != "cfg" { + let rustc_hir::ItemKind::Mod(ident, _) = item.kind else { + return false; + }; + let name = ident.name.as_str(); + name == "tests" || name.ends_with("_tests") || name.ends_with("_test") +} + +fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { + if !attr.has_name(sym::cfg) { return false; } - let Some(list) = meta.meta_item_list() else { + let Some(list) = attr.meta_item_list() else { return false; }; list.iter().any(|nested| { diff --git a/dylints/require_multi_thread_flavor_when_spawning/src/lib.rs b/dylints/require_multi_thread_flavor_when_spawning/src/lib.rs index d756f73b..86950e8b 100644 --- a/dylints/require_multi_thread_flavor_when_spawning/src/lib.rs +++ b/dylints/require_multi_thread_flavor_when_spawning/src/lib.rs @@ -12,7 +12,7 @@ use rustc_hir::{ def::Res, def_id::LocalDefId, intravisit::{walk_expr, Visitor}, - Attribute, BodyId, Expr, ExprKind, FnDecl, + Attribute, Body, Expr, ExprKind, FnDecl, }; use rustc_lint::{LateContext, LateLintPass, LintContext}; use rustc_middle::hir::nested_filter; @@ -76,7 +76,7 @@ impl<'tcx> LateLintPass<'tcx> for RequireMultiThreadFlavorWhenSpawning { cx: &LateContext<'tcx>, kind: rustc_hir::intravisit::FnKind<'tcx>, _decl: &'tcx FnDecl<'tcx>, - body_id: BodyId, + body: &'tcx Body<'tcx>, span: Span, def_id: LocalDefId, ) { @@ -104,7 +104,6 @@ impl<'tcx> LateLintPass<'tcx> for RequireMultiThreadFlavorWhenSpawning { } // Walk the body looking for `tokio::spawn(...)` calls. - let body = cx.tcx.hir_body(body_id); let mut visitor = SpawnFinder { cx, found_at: None, @@ -135,18 +134,15 @@ fn is_async_fn(kind: &rustc_hir::intravisit::FnKind<'_>) -> bool { /// `#[::tokio::test]`, and `#[tokio::test(...)]`. fn find_tokio_test_attr(attrs: &[Attribute]) -> Option<&Attribute> { for attr in attrs { - let meta = attr.meta()?; - let path = meta.path(); - let segments: Vec<&str> = path - .segments - .iter() - .map(|s| s.ident.as_str()) - .collect::>(); - let last = segments.last().copied().unwrap_or(""); - if last != "test" { + let path = attr.path(); + if !path + .last() + .map(|symbol| symbol.as_str() == "test") + .unwrap_or(false) + { continue; } - if segments.iter().any(|s| *s == "tokio") { + if path.iter().any(|symbol| symbol.as_str() == "tokio") { return Some(attr); } } @@ -154,10 +150,7 @@ fn find_tokio_test_attr(attrs: &[Attribute]) -> Option<&Attribute> { } fn attr_has_multi_thread_flavor(attr: &Attribute) -> bool { - let Some(meta) = attr.meta() else { - return false; - }; - let Some(list) = meta.meta_item_list() else { + let Some(list) = attr.meta_item_list() else { return false; }; for nested in list { diff --git a/dylints/require_oncelock_install_before_use/src/lib.rs b/dylints/require_oncelock_install_before_use/src/lib.rs index ec1bd3d5..232c40f2 100644 --- a/dylints/require_oncelock_install_before_use/src/lib.rs +++ b/dylints/require_oncelock_install_before_use/src/lib.rs @@ -8,7 +8,10 @@ extern crate rustc_span; use rustc_errors::DiagDecorator; use rustc_hir::{def::Res, Expr, ExprKind, HirId}; use rustc_lint::{LateContext, LateLintPass, LintContext}; -use rustc_span::{symbol::Symbol, FileName, RemapPathScopeComponents}; +use rustc_span::{ + symbol::{sym, Symbol}, + FileName, RemapPathScopeComponents, +}; dylint_linting::declare_late_lint! { /// ### What it does @@ -148,31 +151,29 @@ fn emit_lint(cx: &LateContext<'_>, span: rustc_span::Span, banned: &[&str]) { } fn owned_by_cfg_test_module(cx: &LateContext<'_>, hir_id: HirId) -> bool { - let mut current = cx.tcx.hir_get_parent_item(hir_id); - loop { - let attrs = cx.tcx.hir_attrs(current.into()); - for attr in attrs { - if attr_is_cfg_test(attr) { - return true; - } - } - let parent = cx.tcx.hir_get_parent_item(current.into()); - if parent == current { - return false; - } - current = parent; - } + std::iter::once(hir_id) + .chain(cx.tcx.hir_parent_id_iter(hir_id)) + .any(|id| { + cx.tcx.hir_attrs(id).iter().any(attr_is_cfg_test) || is_test_module_node(cx, id) + }) } -fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { - let Some(meta) = attr.meta() else { +fn is_test_module_node(cx: &LateContext<'_>, hir_id: HirId) -> bool { + let rustc_hir::Node::Item(item) = cx.tcx.hir_node(hir_id) else { return false; }; - let path = meta.path(); - if path.segments.len() != 1 || path.segments[0].ident.as_str() != "cfg" { + let rustc_hir::ItemKind::Mod(ident, _) = item.kind else { + return false; + }; + let name = ident.name.as_str(); + name == "tests" || name.ends_with("_tests") || name.ends_with("_test") +} + +fn attr_is_cfg_test(attr: &rustc_hir::Attribute) -> bool { + if !attr.has_name(sym::cfg) { return false; } - let Some(list) = meta.meta_item_list() else { + let Some(list) = attr.meta_item_list() else { return false; }; list.iter().any(|nested| { diff --git a/online-data-tools/README.md b/online-data-tools/README.md index 1719abac..d8f7823f 100644 --- a/online-data-tools/README.md +++ b/online-data-tools/README.md @@ -9,6 +9,7 @@ is committed to orphan branches: | Script | Reads from | Writes to | | ------------------- | ------------------------------------------- | -------------------------------------- | | `build_sqlite.py` | `online-data/data/*.json` | `www/.db` | +| `build_usb_vid_proto.py` | `online-data/data/usb-vid.json` | `online-data/data/usb-vids.proto.zstd` | | `rotate_www_dbs.py` | `www/*.db` | `www/` (deletes >2-day-old `.db`s) | | `build_www_manifest.py` | day-stable filenames | `www/manifest.json` | | `fetch_espressif_usb_pids.py` | `espressif/usb-pids` official PID registry | merge-compatible `/tmp/espressif-usb-pids.json` | @@ -34,6 +35,13 @@ the convention is documented in [issue #718](https://github.com/FastLED/fbuild/i ## USB VID:PID Supplements +Source authority is intentional. First-party vendor registries and local +FastLED board data are stronger than generic USB-ID feeds; third-party SDK or +board-package rows are weak supplements that merge after those sources and +only fill gaps. A USB VID/PID row improves product-name resolution, but if a +board is not present under `crates/fbuild-config/assets/boards`, it may not be +an fbuild-supported board. + The Espressif supplement ingests the official `espressif/usb-pids` registry: - `allocated-pids.txt` for customer/product allocations under VID `0x303a` @@ -186,6 +194,7 @@ rows improve VID/PID resolution but do not prove fbuild board support. ```bash uv run --no-project --with pytest pytest online-data-tools/test_build_sqlite.py -v +uv run --no-project --with pytest --with zstandard pytest online-data-tools/test_usb_vid_proto.py -v uv run --no-project --with pytest pytest online-data-tools/test_espressif_usb_pids.py -v uv run --no-project --with pytest pytest online-data-tools/test_raspberrypi_usb_pids.py -v uv run --no-project --with pytest pytest online-data-tools/test_nordic_usb_pids.py -v diff --git a/online-data-tools/build_usb_vid_proto.py b/online-data-tools/build_usb_vid_proto.py new file mode 100644 index 00000000..27383df4 --- /dev/null +++ b/online-data-tools/build_usb_vid_proto.py @@ -0,0 +1,203 @@ +#!/usr/bin/env -S uv run --no-project --with zstandard --script +# /// script +# requires-python = ">=3.10" +# dependencies = ["zstandard"] +# /// +"""Build the compact USB VID:PID protobuf overlay consumed by fbuild. + +Input is the merged `usb-vid.json` dataset: + + { + "303a": { + "vendor": "Espressif Systems", + "products": [["1001", "USB JTAG/serial debug unit"]] + } + } + +Output is `usb-vids.proto.zstd`, a zstd-compressed protobuf using the +wire schema in `crates/fbuild-core/src/usb/data.rs`. The encoder is kept +small and explicit so the online-data workflow can publish the runtime +artifact without needing a generated protobuf toolchain. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Iterable, Mapping, Sequence + +import zstandard as zstd + + +MIN_PRODUCTS = 1000 +PROTO_ZSTD_FILENAME = "usb-vids.proto.zstd" + + +@dataclass(frozen=True) +class BuildStats: + vendors: int + products: int + + +def _varint(value: int) -> bytes: + if value < 0: + raise ValueError("protobuf varint cannot encode negative values") + out = bytearray() + while value >= 0x80: + out.append((value & 0x7F) | 0x80) + value >>= 7 + out.append(value) + return bytes(out) + + +def _key(field_number: int, wire_type: int) -> bytes: + return _varint((field_number << 3) | wire_type) + + +def _field_varint(field_number: int, value: int) -> bytes: + return _key(field_number, 0) + _varint(value) + + +def _field_bytes(field_number: int, value: bytes) -> bytes: + return _key(field_number, 2) + _varint(len(value)) + value + + +def _field_string(field_number: int, value: str) -> bytes: + return _field_bytes(field_number, value.encode("utf-8")) + + +def _encode_product(pid: int, name: str) -> bytes: + return _field_varint(1, pid) + _field_string(2, name) + + +def _encode_vendor(vid: int, name: str, products: Sequence[tuple[int, str]]) -> bytes: + out = bytearray() + out.extend(_field_varint(1, vid)) + out.extend(_field_string(2, name)) + for pid, product_name in products: + out.extend(_field_bytes(3, _encode_product(pid, product_name))) + return bytes(out) + + +_HEX_U16 = re.compile(r"^(?:0x)?([0-9a-fA-F]{1,4})$") + + +def parse_u16_hex(value: object) -> int | None: + match = _HEX_U16.match(str(value).strip()) + if not match: + return None + return int(match.group(1), 16) + + +def _product_rows(raw_products: object) -> Iterable[tuple[object, object]]: + if isinstance(raw_products, Mapping): + return raw_products.items() + if not isinstance(raw_products, list): + return () + rows: list[tuple[object, object]] = [] + for row in raw_products: + if isinstance(row, (list, tuple)) and len(row) == 2: + rows.append((row[0], row[1])) + return rows + + +def normalize_usb_vid(raw: object) -> list[tuple[int, str, list[tuple[int, str]]]]: + """Return sorted `(vid, vendor, [(pid, product), ...])` rows.""" + if not isinstance(raw, Mapping): + raise ValueError("usb-vid.json top-level value must be an object") + + vendors: list[tuple[int, str, list[tuple[int, str]]]] = [] + for vid_raw, entry in raw.items(): + vid = parse_u16_hex(vid_raw) + if vid is None or not isinstance(entry, Mapping): + continue + + vendor = str(entry.get("vendor", "")).strip() + if not vendor: + continue + + products_by_pid: dict[int, str] = {} + for pid_raw, product_raw in _product_rows(entry.get("products", [])): + pid = parse_u16_hex(pid_raw) + product = str(product_raw).strip() + if pid is None or not product: + continue + products_by_pid[pid] = product + + if products_by_pid: + vendors.append((vid, vendor, sorted(products_by_pid.items()))) + + vendors.sort(key=lambda row: row[0]) + return vendors + + +def encode_database(raw: object) -> tuple[bytes, BuildStats]: + vendors = normalize_usb_vid(raw) + out = bytearray() + product_count = 0 + for vid, vendor, products in vendors: + product_count += len(products) + out.extend(_field_bytes(1, _encode_vendor(vid, vendor, products))) + return bytes(out), BuildStats(vendors=len(vendors), products=product_count) + + +def compress_proto(raw_proto: bytes, *, level: int = 19) -> bytes: + return zstd.ZstdCompressor(level=level).compress(raw_proto) + + +def load_json(path: Path) -> object: + return json.loads(path.read_text(encoding="utf-8")) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--upstream", required=True, type=Path, help="Merged usb-vid.json") + parser.add_argument( + "--out", + required=True, + type=Path, + help=f"Output path, usually data/{PROTO_ZSTD_FILENAME}", + ) + parser.add_argument( + "--min-products", + type=int, + default=MIN_PRODUCTS, + help="Refuse to write fewer product rows than this floor.", + ) + parser.add_argument( + "--compression-level", + type=int, + default=19, + help="zstd compression level.", + ) + args = parser.parse_args() + + try: + proto, stats = encode_database(load_json(args.upstream)) + except (OSError, json.JSONDecodeError, ValueError) as e: + print(f"error: failed to read {args.upstream}: {e}", file=sys.stderr) + return 2 + + if stats.products < args.min_products: + print( + f"error: encoded only {stats.products} product rows " + f"(< floor of {args.min_products}); refusing to write", + file=sys.stderr, + ) + return 3 + + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_bytes(compress_proto(proto, level=args.compression_level)) + print( + f"wrote {args.out} ({stats.vendors} vendors, {stats.products} products)", + file=sys.stderr, + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/online-data-tools/test_usb_vid_proto.py b/online-data-tools/test_usb_vid_proto.py new file mode 100644 index 00000000..0ed06132 --- /dev/null +++ b/online-data-tools/test_usb_vid_proto.py @@ -0,0 +1,175 @@ +#!/usr/bin/env -S uv run --no-project --with pytest --with zstandard --script +# /// script +# requires-python = ">=3.10" +# dependencies = ["pytest", "zstandard"] +# /// +"""Tests for the usb-vid.json -> usb-vids.proto.zstd runtime artifact.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +import zstandard as zstd + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import build_usb_vid_proto # noqa: E402 + + +SAMPLE_USB_VID = { + "303a": { + "vendor": "Espressif Systems", + "products": [["1001", "USB JTAG/serial debug unit"], ["0002", "ESP32-S2"]], + }, + "10C4": { + "vendor": "Silicon Labs", + "products": {"EA60": "CP210x UART Bridge"}, + }, + "dead": {"vendor": "", "products": [["beef", "skipped blank vendor"]]}, + "beef": {"vendor": "No Products Inc", "products": []}, + "cafe": {"vendor": "Invalid Product Inc", "products": [["zzzz", "bad pid"]]}, +} + + +def _read_varint(raw: bytes, pos: int) -> tuple[int, int]: + shift = 0 + value = 0 + while True: + byte = raw[pos] + pos += 1 + value |= (byte & 0x7F) << shift + if byte < 0x80: + return value, pos + shift += 7 + + +def _read_len(raw: bytes, pos: int) -> tuple[bytes, int]: + size, pos = _read_varint(raw, pos) + end = pos + size + return raw[pos:end], end + + +def _skip(raw: bytes, pos: int, wire_type: int) -> int: + if wire_type == 0: + _, pos = _read_varint(raw, pos) + return pos + if wire_type == 2: + _, pos = _read_len(raw, pos) + return pos + raise AssertionError(f"unsupported wire type {wire_type}") + + +def _decode_product(raw: bytes) -> tuple[int, str]: + pos = 0 + pid = -1 + name = "" + while pos < len(raw): + key, pos = _read_varint(raw, pos) + field = key >> 3 + wire = key & 0x07 + if field == 1 and wire == 0: + pid, pos = _read_varint(raw, pos) + elif field == 2 and wire == 2: + value, pos = _read_len(raw, pos) + name = value.decode("utf-8") + else: + pos = _skip(raw, pos, wire) + return pid, name + + +def _decode_vendor(raw: bytes) -> tuple[int, str, list[tuple[int, str]]]: + pos = 0 + vid = -1 + name = "" + products: list[tuple[int, str]] = [] + while pos < len(raw): + key, pos = _read_varint(raw, pos) + field = key >> 3 + wire = key & 0x07 + if field == 1 and wire == 0: + vid, pos = _read_varint(raw, pos) + elif field == 2 and wire == 2: + value, pos = _read_len(raw, pos) + name = value.decode("utf-8") + elif field == 3 and wire == 2: + value, pos = _read_len(raw, pos) + products.append(_decode_product(value)) + else: + pos = _skip(raw, pos, wire) + return vid, name, products + + +def _decode_database(raw: bytes) -> list[tuple[int, str, list[tuple[int, str]]]]: + pos = 0 + vendors: list[tuple[int, str, list[tuple[int, str]]]] = [] + while pos < len(raw): + key, pos = _read_varint(raw, pos) + field = key >> 3 + wire = key & 0x07 + if field == 1 and wire == 2: + value, pos = _read_len(raw, pos) + vendors.append(_decode_vendor(value)) + else: + pos = _skip(raw, pos, wire) + return vendors + + +def test_encode_database_matches_runtime_schema() -> None: + proto, stats = build_usb_vid_proto.encode_database(SAMPLE_USB_VID) + + assert stats == build_usb_vid_proto.BuildStats(vendors=2, products=3) + assert _decode_database(proto) == [ + (0x10C4, "Silicon Labs", [(0xEA60, "CP210x UART Bridge")]), + ( + 0x303A, + "Espressif Systems", + [(0x0002, "ESP32-S2"), (0x1001, "USB JTAG/serial debug unit")], + ), + ] + + +def test_compressed_proto_round_trip() -> None: + proto, _ = build_usb_vid_proto.encode_database(SAMPLE_USB_VID) + blob = build_usb_vid_proto.compress_proto(proto) + raw = zstd.ZstdDecompressor().decompress(blob) + assert _decode_database(raw)[0][1] == "Silicon Labs" + + +def test_main_emits_proto_zstd(tmp_path: Path) -> None: + src = tmp_path / "usb-vid.json" + src.write_text(json.dumps(SAMPLE_USB_VID), encoding="utf-8") + out = tmp_path / "usb-vids.proto.zstd" + sys.argv = [ + "build_usb_vid_proto.py", + "--upstream", + str(src), + "--out", + str(out), + "--min-products", + "1", + ] + assert build_usb_vid_proto.main() == 0 + assert _decode_database(zstd.ZstdDecompressor().decompress(out.read_bytes())) + + +def test_main_rejects_too_few_products(tmp_path: Path) -> None: + src = tmp_path / "usb-vid.json" + src.write_text(json.dumps(SAMPLE_USB_VID), encoding="utf-8") + out = tmp_path / "usb-vids.proto.zstd" + sys.argv = [ + "build_usb_vid_proto.py", + "--upstream", + str(src), + "--out", + str(out), + "--min-products", + "4", + ] + assert build_usb_vid_proto.main() == 3 + assert not out.exists() + + +if __name__ == "__main__": + sys.exit(pytest.main([__file__, "-v"])) From a6dacef06e6deaae15ac2341de93aa489681a426 Mon Sep 17 00:00:00 2001 From: zackees Date: Wed, 1 Jul 2026 06:53:57 -0700 Subject: [PATCH 2/4] Fill USB VID gaps from board data --- .github/workflows/update-data.yml | 16 +++ crates/fbuild-cli/src/cli/port_scan.rs | 111 ++++++++++++++-- crates/fbuild-core/src/usb/data.rs | 21 +++- crates/fbuild-core/src/usb/mod.rs | 4 +- docs/online-data.md | 13 +- online-data-tools/README.md | 22 +++- .../extract_fastled_board_usb_pids.py | 118 ++++++++++++++++++ .../test_fastled_board_usb_pids.py | 102 +++++++++++++++ 8 files changed, 383 insertions(+), 24 deletions(-) create mode 100644 online-data-tools/extract_fastled_board_usb_pids.py create mode 100644 online-data-tools/test_fastled_board_usb_pids.py diff --git a/.github/workflows/update-data.yml b/.github/workflows/update-data.yml index 6352ce4a..e906935f 100644 --- a/.github/workflows/update-data.yml +++ b/.github/workflows/update-data.yml @@ -341,6 +341,16 @@ jobs: --out /tmp/seeed-supplemental-usb-pids.json wc -l /tmp/seeed-supplemental-usb-pids.json + - name: Extract FastLED board USB PID rows (USB-VID repo-board supplement) + id: extract-fastled-board-pids + continue-on-error: true + run: | + uv run --no-project --script \ + "${{ github.workspace }}/online-data-tools/extract_fastled_board_usb_pids.py" \ + --boards-dir "${{ github.workspace }}/crates/fbuild-config/assets/boards/json" \ + --out /tmp/fastled-board-usb-pids.json + wc -l /tmp/fastled-board-usb-pids.json + - name: Dump PlatformIO board catalog → /tmp/all_boards.json id: dump-pio continue-on-error: true @@ -440,6 +450,12 @@ jobs: # Energy Micro / Silicon Labs debug-interface row fills in. args+=(--json "silabs-usb-pids=/tmp/silabs-usb-pids.json") fi + if [ "${{ steps.extract-fastled-board-pids.outcome }}" = "success" ] && [ -s /tmp/fastled-board-usb-pids.json ]; then + # Local board JSON rows are repo-scope supplements: they fill + # missing product names for boards fbuild actually carries, but + # do not replace stronger vendor/generic USB-ID source rows. + args+=(--json "fastled-board-usb-pids=/tmp/fastled-board-usb-pids.json") + fi if [ "${{ steps.fetch-renesas-pids.outcome }}" = "success" ] && [ -s /tmp/renesas-usb-pids.json ]; then # ArduinoCore-renesas board-package rows are deliberately weak: # they fill missing Arduino RA board product names after generic diff --git a/crates/fbuild-cli/src/cli/port_scan.rs b/crates/fbuild-cli/src/cli/port_scan.rs index 96811a96..d828a9ec 100644 --- a/crates/fbuild-cli/src/cli/port_scan.rs +++ b/crates/fbuild-cli/src/cli/port_scan.rs @@ -74,18 +74,42 @@ fn run_scan(offline: bool) -> Result<()> { /// resolver degrades to tier-1 (embedded vendor archive). The cache is /// kept fresh on a 7-day cadence — older copies are refetched. fn populate_online_overlay() { + populate_online_overlay_from_urls( + fbuild_core::usb::USB_VIDS_PROTO_ZSTD_URL, + fbuild_core::usb::USB_VID_JSON_URL, + ); +} + +fn populate_online_overlay_from_urls(proto_url: &str, json_url: &str) { let Some(cache_path) = overlay_cache_path() else { return; }; if !cache_is_fresh(&cache_path) { - if let Err(e) = fetch_overlay_to(&cache_path) { + if let Err(e) = fetch_overlay_to(&cache_path, proto_url) { + tracing::debug!( + error = %e, + "port scan: protobuf overlay fetch failed; trying JSON overlay" + ); + } + } + if fbuild_core::usb::try_install_online_cache_proto_zstd(&cache_path) { + return; + } + + let Some(json_cache_path) = overlay_json_cache_path() else { + return; + }; + if !cache_is_fresh(&json_cache_path) { + if let Err(e) = fetch_overlay_to(&json_cache_path, json_url) { tracing::debug!( error = %e, - "port scan: overlay fetch failed — degrading to tier-1 only" + "port scan: JSON overlay fetch failed" ); } } - fbuild_core::usb::install_online_cache_proto_zstd(&cache_path); + if !fbuild_core::usb::try_install_online_cache(&json_cache_path) { + tracing::debug!("port scan: overlay unavailable; degrading to tier-1 only"); + } } fn overlay_cache_path() -> Option { @@ -95,6 +119,13 @@ fn overlay_cache_path() -> Option { Some(dir.join("usb-vids.proto.zstd")) } +fn overlay_json_cache_path() -> Option { + let root = fbuild_paths::get_cache_root(); + let dir = root.join("usb"); + std::fs::create_dir_all(&dir).ok()?; + Some(dir.join("usb-vid.json")) +} + /// 7-day cache TTL — fbuild's online-data branch refreshes nightly; /// a weekly local refresh gives us most of the benefit with minimal /// cold-start network cost. CI / offline boxes still get useful @@ -114,22 +145,23 @@ fn cache_is_fresh(path: &std::path::Path) -> bool { age.as_secs() < OVERLAY_TTL_SECS } -fn fetch_overlay_to(path: &std::path::Path) -> std::result::Result<(), String> { +fn fetch_overlay_to(path: &std::path::Path, url: &str) -> std::result::Result<(), String> { // reqwest::blocking spins its own internal runtime and rejects // being called from inside an outer tokio runtime (the CLI // dispatcher uses `#[tokio::main]`). Run the fetch on a dedicated // OS thread so reqwest's runtime sees a clean async-free context. let path = path.to_path_buf(); - std::thread::spawn(move || fetch_overlay_to_inner(&path)) + let url = url.to_string(); + std::thread::spawn(move || fetch_overlay_to_inner(&path, &url)) .join() .map_err(|_| "fetch thread panicked".to_string())? } -fn fetch_overlay_to_inner(path: &std::path::Path) -> std::result::Result<(), String> { +fn fetch_overlay_to_inner(path: &std::path::Path, url: &str) -> std::result::Result<(), String> { // FastLED/fbuild#844: route the OS-thread blocking client through the // shared bridge so all reqwest construction has one source of truth. let client = fbuild_core::http::blocking_client(Duration::from_secs(15)); - fetch_overlay_to_inner_with_client(path, &client, fbuild_core::usb::USB_VIDS_PROTO_ZSTD_URL) + fetch_overlay_to_inner_with_client(path, &client, url) } fn fetch_overlay_to_inner_with_client( @@ -403,8 +435,10 @@ mod tests { let _guard = EnvVarGuard::set("FBUILD_CACHE_DIR", tmp.path()); let path = overlay_cache_path().expect("cache path"); + let json_path = overlay_json_cache_path().expect("json cache path"); assert_eq!(path, tmp.path().join("usb").join("usb-vids.proto.zstd")); + assert_eq!(json_path, tmp.path().join("usb").join("usb-vid.json")); assert!(tmp.path().join("usb").is_dir()); } @@ -441,6 +475,69 @@ mod tests { assert!(!path.with_extension("proto.zstd.tmp").exists()); } + #[test] + fn populate_overlay_falls_back_to_json_when_proto_is_missing() { + let _env = ENV_LOCK.lock().unwrap(); + let tmp = tempfile::tempdir().unwrap(); + let _guard = EnvVarGuard::set("FBUILD_CACHE_DIR", tmp.path()); + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let addr = listener.local_addr().unwrap(); + let json = r#"{"feed":{"vendor":"Feedface Inc","products":[["c0de","Coded Widget"]]}}"#; + let server_json = json.as_bytes().to_vec(); + let handle = std::thread::spawn(move || { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + let mut request_count = 0_u8; + while request_count < 2 { + match listener.accept() { + Ok((mut stream, _)) => { + let mut request = [0_u8; 1024]; + let _ = stream.read(&mut request).unwrap(); + request_count += 1; + if request_count == 1 { + stream + .write_all( + b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .unwrap(); + } else { + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_json.len() + ); + stream.write_all(response.as_bytes()).unwrap(); + stream.write_all(&server_json).unwrap(); + } + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for overlay requests" + ); + std::thread::sleep(Duration::from_millis(10)); + } + Err(e) => panic!("accept failed: {e}"), + } + } + request_count + }); + + populate_online_overlay_from_urls( + &format!("http://{addr}/usb-vids.proto.zstd"), + &format!("http://{addr}/usb-vid.json"), + ); + + assert_eq!(handle.join().unwrap(), 2); + assert_eq!( + std::fs::read_to_string(tmp.path().join("usb").join("usb-vid.json")).unwrap(), + json + ); + let info = fbuild_core::usb::resolve(0xFEED, 0xC0DE); + assert_eq!(info.vendor, "Feedface Inc"); + assert_eq!(info.product, "Coded Widget"); + } + #[test] fn empty_port_list_renders_canonical_message() { assert_eq!(render_scan(&[]), "no serial ports visible\n"); diff --git a/crates/fbuild-core/src/usb/data.rs b/crates/fbuild-core/src/usb/data.rs index a95c1084..758ab0fc 100644 --- a/crates/fbuild-core/src/usb/data.rs +++ b/crates/fbuild-core/src/usb/data.rs @@ -115,18 +115,24 @@ struct VendorEntry { /// installed overlay. Silently no-ops on any IO or parse error so the /// resolver never crashes on a stale / partial cache file. pub fn install_online_cache(path: &Path) { + let _ = try_install_online_cache(path); +} + +/// Same as [`install_online_cache`], but reports whether an overlay was +/// successfully parsed and installed. +pub fn try_install_online_cache(path: &Path) -> bool { let raw = match std::fs::read_to_string(path) { Ok(s) => s, Err(e) => { tracing::debug!(?path, error = %e, "usb online overlay: read failed"); - return; + return false; } }; let parsed: HashMap = match serde_json::from_str(&raw) { Ok(m) => m, Err(e) => { tracing::warn!(?path, error = %e, "usb online overlay: parse failed"); - return; + return false; } }; // Flatten the on-disk per-VID nested shape into the O(1) flat @@ -152,6 +158,7 @@ pub fn install_online_cache(path: &Path) { let count = packed.len(); install_online_cache_map(packed); tracing::debug!(path = %path.display(), entries = count, "usb online overlay installed"); + true } /// Install the overlay from the current `usb-vids.proto.zstd` cache file. @@ -159,11 +166,17 @@ pub fn install_online_cache(path: &Path) { /// resolution always degrades to the embedded vendor archive instead of /// failing port enumeration. pub fn install_online_cache_proto_zstd(path: &Path) { + let _ = try_install_online_cache_proto_zstd(path); +} + +/// Same as [`install_online_cache_proto_zstd`], but reports whether an +/// overlay was successfully decoded and installed. +pub fn try_install_online_cache_proto_zstd(path: &Path) -> bool { let raw = match std::fs::read(path) { Ok(bytes) => bytes, Err(e) => { tracing::debug!(?path, error = %e, "usb online overlay: read failed"); - return; + return false; } }; match decode_proto_zstd_bytes(&raw) { @@ -175,6 +188,7 @@ pub fn install_online_cache_proto_zstd(path: &Path) { entries = count, "usb online protobuf overlay installed" ); + true } Err(e) => { tracing::warn!( @@ -182,6 +196,7 @@ pub fn install_online_cache_proto_zstd(path: &Path) { error = %e, "usb online protobuf overlay decode failed" ); + false } } } diff --git a/crates/fbuild-core/src/usb/mod.rs b/crates/fbuild-core/src/usb/mod.rs index 11a273a9..80bedaf9 100644 --- a/crates/fbuild-core/src/usb/mod.rs +++ b/crates/fbuild-core/src/usb/mod.rs @@ -31,8 +31,8 @@ pub mod embedded; pub mod resolver; pub use data::{ - install_online_cache, install_online_cache_proto_zstd, MANIFEST_URL, USB_VIDS_PROTO_ZSTD_URL, - USB_VID_JSON_URL, + install_online_cache, install_online_cache_proto_zstd, try_install_online_cache, + try_install_online_cache_proto_zstd, MANIFEST_URL, USB_VIDS_PROTO_ZSTD_URL, USB_VID_JSON_URL, }; pub use embedded::vendor_name as embedded_vendor_name; pub use resolver::{pretty, resolve, resolve_bundled, try_resolve, UsbInfo}; diff --git a/docs/online-data.md b/docs/online-data.md index a77c8ae0..074354d0 100644 --- a/docs/online-data.md +++ b/docs/online-data.md @@ -25,9 +25,10 @@ the bundled `usb-ids` crate doesn't know a VID:PID. VID/PID product rows are USB-name metadata, not board-support proof. Board existence remains governed by `crates/fbuild-config/assets/boards`; if a board -is absent there, a third-party SDK or board-package row is treated only as a -weak supplement that can fill product-name gaps after first-party, vendor, and -generic USB-ID sources have already won. +is absent there, it may not be an fbuild-supported board. Local FastLED board +VID/PID rows fill product-name gaps for checked-in boards after stronger USB +owner and generic sources have won. Third-party SDK or board-package rows are +weaker supplements after that. ## URLs @@ -94,9 +95,9 @@ Per run: 6. `uv run --no-project --script .online-data/tools/merge_sources.py …` over whichever sources arrived intact. The merger: - takes the union in workflow argument order; first-party/vendor-owned - sources are ordered before generic USB-ID feeds, while third-party SDK - and board-package supplements are ordered after them so they only fill - missing rows; + sources are ordered before generic USB-ID feeds, local FastLED board rows + fill checked-in board gaps after those sources, and third-party SDK / + board-package supplements come last so they only fill remaining rows; - sorts keys alphabetically (lowercase `vvvv:pppp`); - writes `data/usb-vid.json`, `data/usb-vid-conflicts.json`, `data/usb-vids.proto.zstd`, and the freshly-stamped `manifest.json`; diff --git a/online-data-tools/README.md b/online-data-tools/README.md index d8f7823f..a78b9318 100644 --- a/online-data-tools/README.md +++ b/online-data-tools/README.md @@ -27,6 +27,7 @@ is committed to orphan branches: | `fetch_nxp_usb_pids.py` | NXP mfgtools/UUU config table | merge-compatible `/tmp/nxp-usb-pids.json` | | `fetch_silabs_usb_pids.py` | Linux CP210x driver + SiliconLabsSoftware OpenOCD udev rule | merge-compatible `/tmp/silabs-usb-pids.json` | | `fetch_renesas_usb_pids.py` | ArduinoCore-renesas `boards.txt` weak supplement | merge-compatible `/tmp/renesas-usb-pids.json` | +| `extract_fastled_board_usb_pids.py` | Local `crates/fbuild-config/assets/boards/json` board VID/PID metadata | merge-compatible `/tmp/fastled-board-usb-pids.json` | The merger scripts on the `online-data` orphan branch (`merge_sources.py`, `merge_pio_boards.py`, `build_manifest.py`, @@ -35,12 +36,14 @@ the convention is documented in [issue #718](https://github.com/FastLED/fbuild/i ## USB VID:PID Supplements -Source authority is intentional. First-party vendor registries and local -FastLED board data are stronger than generic USB-ID feeds; third-party SDK or -board-package rows are weak supplements that merge after those sources and -only fill gaps. A USB VID/PID row improves product-name resolution, but if a -board is not present under `crates/fbuild-config/assets/boards`, it may not be -an fbuild-supported board. +Source authority is intentional. First-party vendor registries are strongest, +generic USB-ID feeds provide broad baseline names, and local FastLED board data +is a repo-scope supplement that fills product-name gaps for boards under +`crates/fbuild-config/assets/boards/json`. Third-party SDK or board-package +rows are weaker supplements that merge after those sources and only fill gaps. +A USB VID/PID row improves product-name resolution, but if a board is not +present under `crates/fbuild-config/assets/boards`, it may not be an +fbuild-supported board. The Espressif supplement ingests the official `espressif/usb-pids` registry: @@ -99,6 +102,13 @@ Adafruit first-party product names win, but entries still describe USB products rather than fbuild board support; support is governed by `crates/fbuild-config/assets/boards`. +The FastLED board supplement extracts `build.vid` / `build.pid` from the +checked-in board JSON files under `crates/fbuild-config/assets/boards/json`. +It is intentionally ordered after vendor-owned and generic USB-ID sources: +local board names fill gaps for boards fbuild actually carries, but board JSON +`vendor` fields are not always the USB VID owner and should not replace +stronger USB product tables. Boards outside this repo are not inferred here. + The SparkFun supplement has explicit source tiers for VID `0x1b4f`. `--tier first-party` parses SparkFun-maintained Arduino board package files, product-repo board files, and SparkFun product descriptors such as UF2 diff --git a/online-data-tools/extract_fastled_board_usb_pids.py b/online-data-tools/extract_fastled_board_usb_pids.py new file mode 100644 index 00000000..b27f7bae --- /dev/null +++ b/online-data-tools/extract_fastled_board_usb_pids.py @@ -0,0 +1,118 @@ +#!/usr/bin/env -S uv run --no-project --script +# /// script +# requires-python = ">=3.10" +# /// +"""Extract USB VID:PID rows from fbuild's checked-in board JSON files. + +This is a repo-scope supplement for the USB resolver. It only considers board +definitions under `crates/fbuild-config/assets/boards/json` and emits the flat +JSON shape consumed by `online-data/tools/merge_sources.py`: + + { + "239a:811b": { + "vendor": "Adafruit", + "product": "Adafruit Feather ESP32-S3 (2MB PSRAM)" + } + } + +The workflow orders this source after stronger vendor/generic USB tables, so +it fills product-name gaps for fbuild-supported boards without replacing +better USB-owner data. +""" + +from __future__ import annotations + +import argparse +import json +import re +from collections import OrderedDict +from pathlib import Path + + +def _hex4(value: object) -> str | None: + if value is None: + return None + if isinstance(value, int): + number = value + else: + text = str(value).strip().lower() + if text.startswith("0x"): + text = text[2:] + if not text: + return None + try: + number = int(text, 16) + except ValueError: + return None + if not 0 <= number <= 0xFFFF: + return None + return f"{number:04x}" + + +def _clean(text: object) -> str: + return re.sub(r"\s+", " ", str(text)).strip() + + +def _collapse(values: list[str]) -> str: + unique = sorted({value for value in (_clean(value) for value in values) if value}) + return " / ".join(unique) + + +def extract_board_usb_pids(boards_dir: Path) -> dict[str, dict[str, str]]: + rows: dict[str, dict[str, list[str]]] = {} + for path in sorted(boards_dir.glob("*.json")): + try: + data = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + continue + if not isinstance(data, dict): + continue + build = data.get("build") + if not isinstance(build, dict): + continue + vid = _hex4(build.get("vid")) + pid = _hex4(build.get("pid")) + if vid is None or pid is None: + continue + + product = _clean(data.get("name") or data.get("id") or path.stem) + if not product: + continue + vendor = _clean(data.get("vendor")) or f"Unknown vendor 0x{vid.upper()}" + key = f"{vid}:{pid}" + bucket = rows.setdefault(key, {"vendors": [], "products": []}) + bucket["vendors"].append(vendor) + bucket["products"].append(product) + + out: dict[str, dict[str, str]] = {} + for key, values in rows.items(): + out[key] = { + "vendor": _collapse(values["vendors"]), + "product": _collapse(values["products"]), + } + return OrderedDict(sorted(out.items())) + + +def write_json(path: Path, rows: dict[str, dict[str, str]]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + with path.open("w", encoding="utf-8", newline="\n") as f: + json.dump(rows, f, indent=2, ensure_ascii=False, sort_keys=True) + f.write("\n") + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--boards-dir", + type=Path, + default=Path("crates/fbuild-config/assets/boards/json"), + help="Directory containing fbuild board JSON files", + ) + parser.add_argument("--out", type=Path, required=True) + args = parser.parse_args() + + write_json(args.out, extract_board_usb_pids(args.boards_dir)) + + +if __name__ == "__main__": + main() diff --git a/online-data-tools/test_fastled_board_usb_pids.py b/online-data-tools/test_fastled_board_usb_pids.py new file mode 100644 index 00000000..7e4c0e0d --- /dev/null +++ b/online-data-tools/test_fastled_board_usb_pids.py @@ -0,0 +1,102 @@ +#!/usr/bin/env -S uv run --no-project --with pytest --script +# /// script +# requires-python = ">=3.10" +# dependencies = ["pytest"] +# /// +"""Tests for extract_fastled_board_usb_pids.py.""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE)) +import extract_fastled_board_usb_pids # noqa: E402 + + +def _write_board( + path: Path, + *, + name: str, + vendor: str, + vid: object | None, + pid: object | None, +) -> None: + build = {} + if vid is not None: + build["vid"] = vid + if pid is not None: + build["pid"] = pid + path.write_text( + json.dumps( + { + "build": build, + "id": path.stem, + "name": name, + "vendor": vendor, + } + ), + encoding="utf-8", + ) + + +def test_extract_board_usb_pids_normalizes_and_collapses_duplicates(tmp_path: Path) -> None: + _write_board( + tmp_path / "alpha.json", + name=" Alpha Board ", + vendor="Vendor A", + vid="0x239A", + pid="0x811B", + ) + _write_board( + tmp_path / "beta.json", + name="Beta Board", + vendor="Vendor B", + vid="239a", + pid="811b", + ) + _write_board( + tmp_path / "gamma.json", + name="Gamma Board", + vendor="Vendor C", + vid=0x2E8A, + pid=0x00C0, + ) + _write_board( + tmp_path / "missing_pid.json", + name="Missing PID", + vendor="Vendor D", + vid="0x1209", + pid=None, + ) + (tmp_path / "bad.json").write_text("{", encoding="utf-8") + + rows = extract_fastled_board_usb_pids.extract_board_usb_pids(tmp_path) + + assert rows == { + "239a:811b": { + "vendor": "Vendor A / Vendor B", + "product": "Alpha Board / Beta Board", + }, + "2e8a:00c0": { + "vendor": "Vendor C", + "product": "Gamma Board", + }, + } + assert list(rows) == sorted(rows) + + +def test_write_json_emits_merge_sources_shape(tmp_path: Path) -> None: + rows = { + "feed:c0de": { + "vendor": "Feedface Inc", + "product": "Coded Widget", + } + } + out = tmp_path / "fastled-board-usb-pids.json" + + extract_fastled_board_usb_pids.write_json(out, rows) + + assert json.loads(out.read_text(encoding="utf-8")) == rows From 8d03d07d32113846984c4c7974310464721dc50d Mon Sep 17 00:00:00 2001 From: zackees Date: Wed, 1 Jul 2026 07:34:44 -0700 Subject: [PATCH 3/4] Load USB overlay in daemon paths Move the proto-first / JSON-fallback overlay download + install helper into `fbuild_core::usb::populate_online_cache_from_paths(_and_urls)` so both `fbuild port scan` and daemon-backed device paths share the same cold-cache hydration path. The helper still runs the reqwest blocking client on a dedicated OS thread (safe from Tokio) and uses the shared `fbuild_core::http::blocking_client` bridge. `fbuild-cli` now delegates to the shared helper; the local fetch/cache-freshness scaffolding is deleted and the port_scan test that exercised the local helper is replaced by a proto-404-then-JSON-200 integration test (kept in the CLI crate) plus a new `populate_online_cache_falls_back_to_json_when_proto_is_missing` test in `fbuild-core::usb::data`. `fbuild-daemon` startup now spawn_blocking's a best-effort call to `populate_online_cache_from_paths` using `/usb/{usb-vids.proto.zstd, usb-vid.json}` before device refresh/list can be served, so `fbuild device list/status/deploy` return full vendor + product names instead of tier-1 vendor-only + `Device 0xPPPP` placeholder. Any failure logs at debug/warn and never blocks daemon bootstrap. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/fbuild-cli/src/cli/port_scan.rs | 128 ++---------------- crates/fbuild-core/src/usb/data.rs | 174 +++++++++++++++++++++++++ crates/fbuild-core/src/usb/mod.rs | 6 +- crates/fbuild-daemon/src/main.rs | 35 +++++ 4 files changed, 221 insertions(+), 122 deletions(-) diff --git a/crates/fbuild-cli/src/cli/port_scan.rs b/crates/fbuild-cli/src/cli/port_scan.rs index d828a9ec..7cb9bf20 100644 --- a/crates/fbuild-cli/src/cli/port_scan.rs +++ b/crates/fbuild-cli/src/cli/port_scan.rs @@ -22,7 +22,6 @@ use clap::Subcommand; use fbuild_core::{FbuildError, Result}; -use std::time::Duration; use crate::output; @@ -81,33 +80,18 @@ fn populate_online_overlay() { } fn populate_online_overlay_from_urls(proto_url: &str, json_url: &str) { - let Some(cache_path) = overlay_cache_path() else { + let Some(proto_cache_path) = overlay_cache_path() else { return; }; - if !cache_is_fresh(&cache_path) { - if let Err(e) = fetch_overlay_to(&cache_path, proto_url) { - tracing::debug!( - error = %e, - "port scan: protobuf overlay fetch failed; trying JSON overlay" - ); - } - } - if fbuild_core::usb::try_install_online_cache_proto_zstd(&cache_path) { - return; - } - let Some(json_cache_path) = overlay_json_cache_path() else { return; }; - if !cache_is_fresh(&json_cache_path) { - if let Err(e) = fetch_overlay_to(&json_cache_path, json_url) { - tracing::debug!( - error = %e, - "port scan: JSON overlay fetch failed" - ); - } - } - if !fbuild_core::usb::try_install_online_cache(&json_cache_path) { + if !fbuild_core::usb::populate_online_cache_from_paths_and_urls( + &proto_cache_path, + &json_cache_path, + proto_url, + json_url, + ) { tracing::debug!("port scan: overlay unavailable; degrading to tier-1 only"); } } @@ -126,70 +110,6 @@ fn overlay_json_cache_path() -> Option { Some(dir.join("usb-vid.json")) } -/// 7-day cache TTL — fbuild's online-data branch refreshes nightly; -/// a weekly local refresh gives us most of the benefit with minimal -/// cold-start network cost. CI / offline boxes still get useful -/// results from the cached copy. -const OVERLAY_TTL_SECS: u64 = 7 * 24 * 60 * 60; - -fn cache_is_fresh(path: &std::path::Path) -> bool { - let Ok(meta) = std::fs::metadata(path) else { - return false; - }; - let Ok(modified) = meta.modified() else { - return false; - }; - let Ok(age) = modified.elapsed() else { - return false; - }; - age.as_secs() < OVERLAY_TTL_SECS -} - -fn fetch_overlay_to(path: &std::path::Path, url: &str) -> std::result::Result<(), String> { - // reqwest::blocking spins its own internal runtime and rejects - // being called from inside an outer tokio runtime (the CLI - // dispatcher uses `#[tokio::main]`). Run the fetch on a dedicated - // OS thread so reqwest's runtime sees a clean async-free context. - let path = path.to_path_buf(); - let url = url.to_string(); - std::thread::spawn(move || fetch_overlay_to_inner(&path, &url)) - .join() - .map_err(|_| "fetch thread panicked".to_string())? -} - -fn fetch_overlay_to_inner(path: &std::path::Path, url: &str) -> std::result::Result<(), String> { - // FastLED/fbuild#844: route the OS-thread blocking client through the - // shared bridge so all reqwest construction has one source of truth. - let client = fbuild_core::http::blocking_client(Duration::from_secs(15)); - fetch_overlay_to_inner_with_client(path, &client, url) -} - -fn fetch_overlay_to_inner_with_client( - path: &std::path::Path, - client: &reqwest::blocking::Client, - url: &str, -) -> std::result::Result<(), String> { - let response = client - .get(url) - .send() - .map_err(|e| format!("http get: {e}"))?; - if !response.status().is_success() { - return Err(format!("http status {}", response.status())); - } - let body = response.bytes().map_err(|e| format!("body read: {e}"))?; - // Atomic write via a `.tmp` sibling + rename — partial writes from - // a Ctrl+C mid-fetch don't poison the cache. - let tmp = path.with_extension("proto.zstd.tmp"); - std::fs::write(&tmp, &body).map_err(|e| format!("tmp write: {e}"))?; - std::fs::rename(&tmp, path).map_err(|e| format!("rename: {e}"))?; - tracing::debug!( - path = %path.display(), - size = body.len(), - "port scan: overlay cache refreshed" - ); - Ok(()) -} - // ─── pure, testable formatter ──────────────────────────────────────── /// Render the entire `fbuild port scan` output for a port list. Pure @@ -384,6 +304,7 @@ mod tests { use std::io::{Read, Write}; use std::net::TcpListener; use std::sync::Mutex; + use std::time::Duration; static ENV_LOCK: Mutex<()> = Mutex::new(()); @@ -442,39 +363,6 @@ mod tests { assert!(tmp.path().join("usb").is_dir()); } - #[test] - fn fetch_overlay_writes_cache_file_atomically() { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let addr = listener.local_addr().unwrap(); - let expected = b"fake proto zstd bytes".to_vec(); - let server_expected = expected.clone(); - let handle = std::thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - let mut request = [0_u8; 1024]; - let _ = stream.read(&mut request).unwrap(); - let response = format!( - "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - server_expected.len() - ); - stream.write_all(response.as_bytes()).unwrap(); - stream.write_all(&server_expected).unwrap(); - }); - - let tmp = tempfile::tempdir().unwrap(); - let path = tmp.path().join("usb-vids.proto.zstd"); - let client = reqwest::blocking::Client::builder() - .timeout(Duration::from_secs(5)) - .build() - .unwrap(); - - fetch_overlay_to_inner_with_client(&path, &client, &format!("http://{addr}/data")) - .expect("fetch should write cache"); - - handle.join().unwrap(); - assert_eq!(std::fs::read(&path).unwrap(), expected); - assert!(!path.with_extension("proto.zstd.tmp").exists()); - } - #[test] fn populate_overlay_falls_back_to_json_when_proto_is_missing() { let _env = ENV_LOCK.lock().unwrap(); diff --git a/crates/fbuild-core/src/usb/data.rs b/crates/fbuild-core/src/usb/data.rs index 758ab0fc..9c1e3c47 100644 --- a/crates/fbuild-core/src/usb/data.rs +++ b/crates/fbuild-core/src/usb/data.rs @@ -60,6 +60,7 @@ use serde::Deserialize; use std::collections::HashMap; use std::path::Path; use std::sync::RwLock; +use std::time::Duration; /// Legacy URL of the dataset index produced by the `online-data` branch's /// nightly workflow. @@ -77,6 +78,11 @@ pub const USB_VIDS_PROTO_ZSTD_URL: &str = static ONLINE_MAP: RwLock>> = RwLock::new(None); +/// 7-day cache TTL. The `online-data` branch refreshes nightly; a weekly +/// local refresh gives useful freshness without adding network cost to every +/// serial-port operation. +pub const ONLINE_CACHE_TTL_SECS: u64 = 7 * 24 * 60 * 60; + #[derive(Clone, PartialEq, Message)] struct UsbVidDatabase { #[prost(message, repeated, tag = "1")] @@ -201,6 +207,102 @@ pub fn try_install_online_cache_proto_zstd(path: &Path) -> bool { } } +/// Populate and install the runtime USB overlay from cache paths. +/// +/// The compact protobuf/zstd artifact is preferred. If that fetch or decode +/// fails, the legacy JSON dataset is fetched/installed as a compatibility +/// fallback. This is intentionally path-driven so callers can use +/// `fbuild-paths` without creating a dependency cycle in `fbuild-core`. +pub fn populate_online_cache_from_paths(proto_cache_path: &Path, json_cache_path: &Path) -> bool { + populate_online_cache_from_paths_and_urls( + proto_cache_path, + json_cache_path, + USB_VIDS_PROTO_ZSTD_URL, + USB_VID_JSON_URL, + ) +} + +/// Same as [`populate_online_cache_from_paths`], with injectable URLs for +/// tests and local mirrors. +pub fn populate_online_cache_from_paths_and_urls( + proto_cache_path: &Path, + json_cache_path: &Path, + proto_url: &str, + json_url: &str, +) -> bool { + if !cache_is_fresh(proto_cache_path) { + if let Err(e) = fetch_overlay_to(proto_cache_path, proto_url) { + tracing::debug!( + error = %e, + "usb online protobuf overlay fetch failed; trying JSON overlay" + ); + } + } + if try_install_online_cache_proto_zstd(proto_cache_path) { + return true; + } + + if !cache_is_fresh(json_cache_path) { + if let Err(e) = fetch_overlay_to(json_cache_path, json_url) { + tracing::debug!(error = %e, "usb online JSON overlay fetch failed"); + } + } + try_install_online_cache(json_cache_path) +} + +fn cache_is_fresh(path: &Path) -> bool { + let Ok(meta) = std::fs::metadata(path) else { + return false; + }; + let Ok(modified) = meta.modified() else { + return false; + }; + let Ok(age) = modified.elapsed() else { + return false; + }; + age.as_secs() < ONLINE_CACHE_TTL_SECS +} + +fn fetch_overlay_to(path: &Path, url: &str) -> Result<(), String> { + let path = path.to_path_buf(); + let url = url.to_string(); + std::thread::spawn(move || fetch_overlay_to_inner(&path, &url)) + .join() + .map_err(|_| "fetch thread panicked".to_string())? +} + +fn fetch_overlay_to_inner(path: &Path, url: &str) -> Result<(), String> { + let client = crate::http::blocking_client(Duration::from_secs(15)); + fetch_overlay_to_inner_with_client(path, &client, url) +} + +fn fetch_overlay_to_inner_with_client( + path: &Path, + client: &reqwest::blocking::Client, + url: &str, +) -> Result<(), String> { + let response = client + .get(url) + .send() + .map_err(|e| format!("http get: {e}"))?; + if !response.status().is_success() { + return Err(format!("http status {}", response.status())); + } + let body = response.bytes().map_err(|e| format!("body read: {e}"))?; + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).map_err(|e| format!("mkdir: {e}"))?; + } + let tmp = path.with_extension("tmp"); + std::fs::write(&tmp, &body).map_err(|e| format!("tmp write: {e}"))?; + std::fs::rename(&tmp, path).map_err(|e| format!("rename: {e}"))?; + tracing::debug!( + path = %path.display(), + size = body.len(), + "usb online overlay cache refreshed" + ); + Ok(()) +} + fn decode_proto_zstd_bytes(raw: &[u8]) -> Result, String> { let mut decoded = Vec::with_capacity(raw.len() * 4); zstd::stream::copy_decode(raw, &mut decoded).map_err(|e| format!("zstd: {e}"))?; @@ -387,6 +489,78 @@ mod tests { assert!(lookup(0x1234, 0x5678).is_none()); } + #[test] + fn populate_online_cache_falls_back_to_json_when_proto_is_missing() { + use std::io::{Read, Write}; + use std::net::TcpListener; + + let _guard = OVERLAY_LOCK.lock().unwrap(); + clear_online_cache_for_tests(); + + let tmp = tempfile::tempdir().unwrap(); + let proto_path = tmp.path().join("usb-vids.proto.zstd"); + let json_path = tmp.path().join("usb-vid.json"); + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + listener.set_nonblocking(true).unwrap(); + let addr = listener.local_addr().unwrap(); + let json = r#"{"feed":{"vendor":"Feedface Inc","products":[["c0de","Coded Widget"]]}}"#; + let server_json = json.as_bytes().to_vec(); + let handle = std::thread::spawn(move || { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + let mut request_count = 0_u8; + while request_count < 2 { + match listener.accept() { + Ok((mut stream, _)) => { + let mut buf = [0_u8; 1024]; + let _ = stream.read(&mut buf).unwrap(); + request_count += 1; + if request_count == 1 { + stream + .write_all( + b"HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\nConnection: close\r\n\r\n", + ) + .unwrap(); + } else { + let response = format!( + "HTTP/1.1 200 OK\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + server_json.len() + ); + stream.write_all(response.as_bytes()).unwrap(); + stream.write_all(&server_json).unwrap(); + } + } + Err(e) if e.kind() == std::io::ErrorKind::WouldBlock => { + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for overlay requests" + ); + std::thread::sleep(Duration::from_millis(10)); + } + Err(e) => panic!("accept failed: {e}"), + } + } + request_count + }); + + let installed = populate_online_cache_from_paths_and_urls( + &proto_path, + &json_path, + &format!("http://{addr}/usb-vids.proto.zstd"), + &format!("http://{addr}/usb-vid.json"), + ); + + assert_eq!(handle.join().unwrap(), 2); + assert!(installed, "JSON fallback should install the overlay"); + assert_eq!(std::fs::read_to_string(&json_path).unwrap(), json); + + let info = lookup(0xFEED, 0xC0DE).expect("json fallback overlay lookup"); + assert_eq!(info.vendor, "Feedface Inc"); + assert_eq!(info.product, "Coded Widget"); + + clear_online_cache_for_tests(); + } + #[test] fn install_online_cache_vendor_without_products_is_skipped() { let _guard = OVERLAY_LOCK.lock().unwrap(); diff --git a/crates/fbuild-core/src/usb/mod.rs b/crates/fbuild-core/src/usb/mod.rs index 80bedaf9..5244f4ec 100644 --- a/crates/fbuild-core/src/usb/mod.rs +++ b/crates/fbuild-core/src/usb/mod.rs @@ -31,8 +31,10 @@ pub mod embedded; pub mod resolver; pub use data::{ - install_online_cache, install_online_cache_proto_zstd, try_install_online_cache, - try_install_online_cache_proto_zstd, MANIFEST_URL, USB_VIDS_PROTO_ZSTD_URL, USB_VID_JSON_URL, + install_online_cache, install_online_cache_proto_zstd, populate_online_cache_from_paths, + populate_online_cache_from_paths_and_urls, try_install_online_cache, + try_install_online_cache_proto_zstd, MANIFEST_URL, ONLINE_CACHE_TTL_SECS, + USB_VIDS_PROTO_ZSTD_URL, USB_VID_JSON_URL, }; pub use embedded::vendor_name as embedded_vendor_name; pub use resolver::{pretty, resolve, resolve_bundled, try_resolve, UsbInfo}; diff --git a/crates/fbuild-daemon/src/main.rs b/crates/fbuild-daemon/src/main.rs index 9cc0ec45..847d00ef 100644 --- a/crates/fbuild-daemon/src/main.rs +++ b/crates/fbuild-daemon/src/main.rs @@ -77,6 +77,14 @@ async fn main() { tracing::info!("fbuild daemon starting on port {}", port); + // Populate the tier-2 USB VID:PID overlay so `device list/status/deploy` + // return full vendor + product names instead of the tier-1 vendor-only + // + synthetic `Device 0xPPPP` placeholder. Best-effort: the resolver + // silently degrades to the embedded vendor archive if the fetch or + // decode fails. Runs on a blocking thread so a slow network doesn't + // stall daemon bootstrap. + tokio::task::spawn_blocking(populate_usb_overlay_best_effort); + // FastLED/fbuild#800 (Phase 4 stage 2 of #789): start the embedded // zccache service inside this tokio runtime and install the global // handle BEFORE any compile work begins. The wrapper-binary path is @@ -439,6 +447,33 @@ async fn main() { std::process::exit(0); } +/// Populate the runtime USB VID:PID overlay from the shared cache root. +/// +/// The daemon's `/api/devices/*` handlers call `fbuild_core::usb::resolve` +/// to render vendor/product names. Without this best-effort startup step +/// the resolver only sees the compile-time embedded vendor archive +/// (tier-1), so unknown PIDs render as `Device 0xPPPP`. Any I/O, network, +/// or decode failure is swallowed — the resolver falls back to tier-1. +fn populate_usb_overlay_best_effort() { + let root = fbuild_paths::get_cache_root(); + let dir = root.join("usb"); + if let Err(e) = std::fs::create_dir_all(&dir) { + tracing::debug!( + path = %dir.display(), + error = %e, + "usb overlay: cache dir create failed; skipping tier-2 population" + ); + return; + } + let proto_path = dir.join("usb-vids.proto.zstd"); + let json_path = dir.join("usb-vid.json"); + if fbuild_core::usb::populate_online_cache_from_paths(&proto_path, &json_path) { + tracing::info!("usb overlay: tier-2 VID:PID map installed"); + } else { + tracing::warn!("usb overlay: tier-2 unavailable; falling back to embedded vendor archive"); + } +} + /// Read the daemon PID file. Returns `Some(pid)` only if the file exists, /// the contents parse as a u32, AND the referenced process is no longer /// alive (i.e. it's safe to clean up). Returns `None` if the file is From ab58ace68cf0b15d6b755f8f3765bcc08e5234f7 Mon Sep 17 00:00:00 2001 From: zackees Date: Wed, 1 Jul 2026 07:59:41 -0700 Subject: [PATCH 4/4] Drop redundant mkdir from daemon USB overlay hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `ban_std_fs_in_async` fires on `std::fs::create_dir_all` inside `crates/fbuild-daemon/src/**`, even though the whole helper is invoked through `tokio::task::spawn_blocking`. The mkdir was defensive only — `fbuild_core::usb::populate_online_cache_from_paths` already creates the parent directory inside its fetch step before writing the cache file, so the daemon-side mkdir is redundant. Drop the `std::fs` call entirely rather than allowlist it or force an async rewrite; the resolver still degrades to the embedded vendor archive if both fetches fail and the cache dir stays empty. Co-Authored-By: Claude Opus 4.7 (1M context) --- crates/fbuild-daemon/src/main.rs | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/crates/fbuild-daemon/src/main.rs b/crates/fbuild-daemon/src/main.rs index 847d00ef..d32b9cb1 100644 --- a/crates/fbuild-daemon/src/main.rs +++ b/crates/fbuild-daemon/src/main.rs @@ -454,17 +454,13 @@ async fn main() { /// the resolver only sees the compile-time embedded vendor archive /// (tier-1), so unknown PIDs render as `Device 0xPPPP`. Any I/O, network, /// or decode failure is swallowed — the resolver falls back to tier-1. +/// +/// The `/usb/` directory is created lazily inside the shared +/// `fbuild_core::usb::populate_online_cache_from_paths` helper (its fetch +/// step does its own parent-dir `create_dir_all`), so we don't touch +/// `std::fs` from the daemon crate here. fn populate_usb_overlay_best_effort() { - let root = fbuild_paths::get_cache_root(); - let dir = root.join("usb"); - if let Err(e) = std::fs::create_dir_all(&dir) { - tracing::debug!( - path = %dir.display(), - error = %e, - "usb overlay: cache dir create failed; skipping tier-2 population" - ); - return; - } + let dir = fbuild_paths::get_cache_root().join("usb"); let proto_path = dir.join("usb-vids.proto.zstd"); let json_path = dir.join("usb-vid.json"); if fbuild_core::usb::populate_online_cache_from_paths(&proto_path, &json_path) {