From b3f0f9f9002c62b36764d6d2125a210e5507a414 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 17:22:54 -0700 Subject: [PATCH 01/16] fix(installer): register the SMBus broker during Windows install The 0.2.1 installer ran PawnIO's setup but skipped the HypercolorSmBus broker, and the broker is what loads PawnIO modules on behalf of the unelevated daemon. Without it CpuTempReader::new() fails its probe read, so cpu_temp_celsius stays null forever: motherboard and DRAM SMBus lighting never appear, and any display face bound to cpu_temp renders a flat zero. Verified on a clean 0.2.1 install: PawnIO service running, modules deployed, HypercolorSmBus absent, /api/v1/system/sensors reporting cpu_temp_celsius null next to a working gpu_temp_celsius. Install time is the only moment Hypercolor holds administrator rights. Deferring broker setup meant either an out-of-nowhere UAC prompt later or silently broken hardware, and in practice it was the latter, because the only remaining path was a Hardware Support panel gated behind motherboard detection. Run the existing hardware-support orchestrator from the postinstall hook instead, which installs PawnIO, stages the verified module blobs, then registers and starts the broker in one elevated pass. Every path handed to the orchestrator sits under $INSTDIR, so the broker installer's rejection of user-writable service paths still holds; that guard, not the deferral, is what keeps a LocalSystem service from loading user-rewritable code. A failed pass no longer implies a failed install: USB and network lighting work without any of this, and the details log now says so. Co-Authored-By: Nova (Claude Opus 5) --- crates/hypercolor-app/installer-hooks.nsh | 50 +++++++++++++------ .../hypercolor-app/tests/packaging_tests.rs | 46 +++++++++++++++++ 2 files changed, 82 insertions(+), 14 deletions(-) diff --git a/crates/hypercolor-app/installer-hooks.nsh b/crates/hypercolor-app/installer-hooks.nsh index 90146a737..5ea92f8ef 100644 --- a/crates/hypercolor-app/installer-hooks.nsh +++ b/crates/hypercolor-app/installer-hooks.nsh @@ -2,9 +2,9 @@ ; ; Tauri's templated NSIS installer handles the file/registry steps, ; but it has no knowledge of our hardware access stack (PawnIO kernel -; driver + Windows Firewall exception), and no concept of cleaning that -; stack up on uninstall. These hooks fill that gap while keeping -; privileged SMBus broker setup as an explicit administrator action. +; driver + SMBus broker service + Windows Firewall exception), and no +; concept of cleaning that stack up on uninstall. These hooks fill +; that gap. ; ; Wired in via bundle.windows.nsis.installerHooks in ; tauri.windows.bundle.conf.json. The installer runs elevated @@ -12,23 +12,45 @@ ; all inherit the rights they need. !macro NSIS_HOOK_POSTINSTALL - ; PawnIO kernel driver — silent install of the bundled installer. - ; Skips if PawnIO is already present (the script's Resolve-PawnIoHome - ; short-circuits the setup.exe run). Modules are copied into PawnIO's - ; install dir so the broker can find them via pawnio_module_dirs(). + ; Hardware access stack: PawnIO kernel driver plus the HypercolorSmBus + ; broker, installed in one elevated pass. The orchestrator installs + ; PawnIO (short-circuiting when already present), copies the verified + ; module blobs into PawnIO's install dir, then registers and starts the + ; broker. + ; + ; Both halves are load-bearing. The broker is what loads PawnIO modules + ; on behalf of the unelevated daemon, so skipping it leaves CPU package + ; temperature and motherboard/DRAM SMBus lighting permanently dark with + ; no error the user can act on. + ; + ; Install time is the only moment Hypercolor holds administrator rights. + ; An unelevated app cannot register a LocalSystem service, so deferring + ; this means either an out-of-nowhere UAC prompt later or — as shipped in + ; 0.2.1 — silently broken hardware. Every path handed to the orchestrator + ; sits under $INSTDIR (Program Files under perMachine), which satisfies + ; the broker installer's own rejection of user-writable service paths. + ; + ; -ReinstallService keeps upgrades idempotent: the broker installer + ; refuses to clobber an existing registration without it. ; ; The bundled script propagates Windows installer exit code 3010 ; ("reboot required") when the kernel driver needs a restart to finish ; binding to SCM. We stash that into $R0 so we can prompt the user to ; restart at the end of postinstall. - DetailPrint "Installing PawnIO hardware access (this may take a moment)..." - nsExec::ExecToLog 'powershell.exe -NoLogo -NoProfile -ExecutionPolicy Bypass -File "$INSTDIR\tools\install-bundled-pawnio.ps1" -AssetRoot "$INSTDIR\tools\pawnio" -Silent' + DetailPrint "Installing Hypercolor hardware access (this may take a moment)..." + nsExec::ExecToLog 'powershell.exe -NoLogo -NoProfile -ExecutionPolicy Bypass -File "$INSTDIR\tools\install-windows-hardware-support.ps1" -AssetRoot "$INSTDIR\tools\pawnio" -BrokerExe "$INSTDIR\tools\hypercolor-smbus-service.exe" -ModuleDestination "$INSTDIR\tools\pawnio\modules" -Silent -ReinstallService' Pop $R0 - DetailPrint " PawnIO install exit code: $R0" + DetailPrint " Hardware access exit code: $R0" - ; HypercolorSmBus broker service is no longer auto-installed by default. - ; Installing and starting a privileged broker remains an explicit - ; administrator action outside the main installer. + ; A failed hardware-access pass must not fail the install — Hypercolor + ; still drives every USB and network device without it. Say so plainly + ; in the details log so a support request has something to quote. + ${If} $R0 <> 0 + ${AndIf} $R0 <> 3010 + DetailPrint " Hardware access setup did not complete. USB and network" + DetailPrint " lighting still work; motherboard SMBus lighting and CPU" + DetailPrint " temperature need Settings > Discovery > Hardware Support." + ${EndIf} ; Windows Firewall — pre-grant the daemon so mDNS discovery and any ; future inbound traffic don't trigger the "Allow on public networks?" @@ -51,7 +73,7 @@ ; of letting them launch Hypercolor into a broken hardware-access ; state. ${If} $R0 = 3010 - MessageBox MB_YESNO|MB_ICONQUESTION "Hypercolor installed successfully, but the PawnIO kernel driver needs a Windows restart before hardware lighting can come online. Restart now?" IDNO no_reboot_now + MessageBox MB_YESNO|MB_ICONQUESTION "Hypercolor installed successfully, but the PawnIO kernel driver needs a Windows restart before motherboard lighting and CPU temperature can come online. Restart now?" IDNO no_reboot_now Reboot no_reboot_now: ${EndIf} diff --git a/crates/hypercolor-app/tests/packaging_tests.rs b/crates/hypercolor-app/tests/packaging_tests.rs index 529d0cced..78413e431 100644 --- a/crates/hypercolor-app/tests/packaging_tests.rs +++ b/crates/hypercolor-app/tests/packaging_tests.rs @@ -19,6 +19,7 @@ const PACKAGE_DEB_SH: &str = include_str!("../../../scripts/package-deb.sh"); const VERIFY_DEB_SH: &str = include_str!("../../../scripts/verify-deb-package.sh"); const STAGE_APP_BUNDLE_PS1: &str = include_str!("../../../scripts/stage-app-bundle-assets.ps1"); const STAGE_APP_BUNDLE_SH: &str = include_str!("../../../scripts/stage-app-bundle-assets.sh"); +const INSTALLER_HOOKS_NSH: &str = include_str!("../installer-hooks.nsh"); const WINDOWS_TOOL_SCRIPTS: &[&str] = &[ "install-windows-service.ps1", @@ -130,6 +131,51 @@ fn app_bundle_staging_includes_pawnio_runtime_payloads() { } } +/// 0.2.1 shipped PawnIO without the broker that loads its modules, so CPU +/// package temperature and motherboard SMBus lighting were dead on every +/// Windows install with nothing prompting for the rights to fix it. +#[test] +fn installer_hook_provisions_the_whole_hardware_access_stack() { + assert!( + INSTALLER_HOOKS_NSH.contains("install-windows-hardware-support.ps1"), + "postinstall must run the orchestrator that installs PawnIO *and* the SMBus broker" + ); + assert!( + INSTALLER_HOOKS_NSH + .contains(r#"-BrokerExe "$INSTDIR\tools\hypercolor-smbus-service.exe""#), + "the broker must be registered from its Program Files path" + ); + assert!( + INSTALLER_HOOKS_NSH.contains("-ReinstallService"), + "upgrades must not trip the broker installer's existing-registration guard" + ); +} + +/// The broker installer rejects service binaries and PawnIO directories under +/// per-user profile paths, since a LocalSystem service must not load code the +/// user can rewrite. Everything the hook hands it therefore lives in $INSTDIR. +#[test] +fn installer_hook_keeps_privileged_paths_administrator_owned() { + for flag in ["-AssetRoot", "-BrokerExe", "-ModuleDestination"] { + let value = INSTALLER_HOOKS_NSH + .split_once(&format!("{flag} \"")) + .map(|(_, rest)| rest) + .and_then(|rest| rest.split_once('"')) + .map(|(value, _)| value) + .unwrap_or_else(|| panic!("installer hook should pass {flag}")); + assert!( + value.starts_with("$INSTDIR\\"), + "{flag} must stay under $INSTDIR, got {value}" + ); + } +} + +#[test] +fn installer_hook_cleans_up_the_broker_on_uninstall() { + assert!(INSTALLER_HOOKS_NSH.contains("sc.exe stop HypercolorSmBus")); + assert!(INSTALLER_HOOKS_NSH.contains("sc.exe delete HypercolorSmBus")); +} + #[test] fn brand_build_pipeline_mirrors_nsis_assets_to_tauri_icons() { assert!(BRAND_BUILD_PY.contains("crates\" / \"hypercolor-app\" / \"icons")); From 0c4c037117ab578df770ebcd1e2ca6de7b06b10a Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 17:33:17 -0700 Subject: [PATCH 02/16] fix(windows): splat hardware-support arguments by hashtable Splatting an array into a PowerShell script binds every element positionally, so @("-AssetRoot", $path, ...) handed the child script the literal string "-AssetRoot" as its first parameter, shifted every real value one slot left, and dropped -Silent, -Force, -Reinstall and -Start entirely. Only native executables parse "-Name value" pairs out of a splatted array; scripts need a hashtable. The other splat sites in scripts/ all target real executables and are fine as they are. The blast radius was the whole Windows hardware-access story. The orchestrator is what the app's Settings > Discovery > "Install support" button runs, and it threw before reaching the broker every single time, which is why an install could never be repaired from inside the app. PawnIO itself survived only because the NSIS hook called its installer directly with real named parameters. Verified by running the fixed orchestrator elevated against a stock 0.2.1 install: PawnIO reported ready, HypercolorSmBus registered and started, and the running daemon picked up two previously invisible ASUS Aura DRAM devices over SMBus without needing a restart. Co-Authored-By: Nova (Claude Opus 5) --- .../hypercolor-app/tests/packaging_tests.rs | 23 +++++++++++++ scripts/install-windows-hardware-support.ps1 | 32 +++++++++---------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/crates/hypercolor-app/tests/packaging_tests.rs b/crates/hypercolor-app/tests/packaging_tests.rs index 78413e431..4d7eb4f0c 100644 --- a/crates/hypercolor-app/tests/packaging_tests.rs +++ b/crates/hypercolor-app/tests/packaging_tests.rs @@ -20,6 +20,8 @@ const VERIFY_DEB_SH: &str = include_str!("../../../scripts/verify-deb-package.sh const STAGE_APP_BUNDLE_PS1: &str = include_str!("../../../scripts/stage-app-bundle-assets.ps1"); const STAGE_APP_BUNDLE_SH: &str = include_str!("../../../scripts/stage-app-bundle-assets.sh"); const INSTALLER_HOOKS_NSH: &str = include_str!("../installer-hooks.nsh"); +const INSTALL_WINDOWS_HARDWARE_SUPPORT_PS1: &str = + include_str!("../../../scripts/install-windows-hardware-support.ps1"); const WINDOWS_TOOL_SCRIPTS: &[&str] = &[ "install-windows-service.ps1", @@ -170,6 +172,27 @@ fn installer_hook_keeps_privileged_paths_administrator_owned() { } } +/// Splatting an array into a PowerShell *script* binds every element +/// positionally, so `@("-AssetRoot", $path)` puts the literal string +/// "-AssetRoot" in the child's first parameter and drops every switch on the +/// floor. Only native executables parse "-Name value" pairs out of a splatted +/// array; scripts need a hashtable. This silently broke both the installer +/// path and the app's "Install support" button. +#[test] +fn hardware_support_orchestrator_splats_by_hashtable_not_array() { + for splat in ["$pawnIoArgs", "$serviceArgs"] { + let opener = format!("{splat} = @"); + let (_, rest) = INSTALL_WINDOWS_HARDWARE_SUPPORT_PS1 + .split_once(opener.as_str()) + .unwrap_or_else(|| panic!("orchestrator should build {splat}")); + assert!( + rest.starts_with('{'), + "{splat} must be a hashtable; an array splat binds positionally \ + and silently mis-assigns every named parameter" + ); + } +} + #[test] fn installer_hook_cleans_up_the_broker_on_uninstall() { assert!(INSTALLER_HOOKS_NSH.contains("sc.exe stop HypercolorSmBus")); diff --git a/scripts/install-windows-hardware-support.ps1 b/scripts/install-windows-hardware-support.ps1 index 2f514f3f4..323e669b8 100644 --- a/scripts/install-windows-hardware-support.ps1 +++ b/scripts/install-windows-hardware-support.ps1 @@ -83,32 +83,32 @@ foreach ($path in @($pawnIoInstaller, $serviceInstaller, $BrokerExe)) { } } -$pawnIoArgs = @( - "-AssetRoot", - $AssetRoot, - "-ModuleDestination", - $ModuleDestination -) +# Hashtable splatting, not array splatting. Splatting an array into a +# PowerShell script binds every element positionally, so "-AssetRoot" lands +# in $AssetRoot as a literal value and switches never bind at all. Only +# native executables parse "-Name value" pairs out of a splatted array. +$pawnIoArgs = @{ + AssetRoot = $AssetRoot + ModuleDestination = $ModuleDestination +} if ($ForcePawnIo) { - $pawnIoArgs += "-Force" + $pawnIoArgs["Force"] = $true } if ($Silent) { - $pawnIoArgs += "-Silent" + $pawnIoArgs["Silent"] = $true } & $pawnIoInstaller @pawnIoArgs $pawnIoExit = $LASTEXITCODE -$serviceArgs = @( - "-BrokerExe", - $BrokerExe, - "-StartupType", - "Automatic" -) +$serviceArgs = @{ + BrokerExe = $BrokerExe + StartupType = "Automatic" +} if ($ReinstallService) { - $serviceArgs += "-Reinstall" + $serviceArgs["Reinstall"] = $true } if (-not $NoStartService) { - $serviceArgs += "-Start" + $serviceArgs["Start"] = $true } & $serviceInstaller @serviceArgs From 3ef1e1a9673f7f4255f618af3c204e6afc8514ae Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 17:34:26 -0700 Subject: [PATCH 03/16] fix(sensors): re-probe the Windows CPU temperature reader WindowsSensorExtras opened the PawnIO reader once at construction and cached the failure for the daemon's lifetime, so cpu_temp_celsius stayed null until someone restarted the daemon. That is precisely backwards from how the broker actually arrives: the installer registers it while an upgrade's daemon is already running, PawnIO's kernel driver can need a reboot before it binds to SCM, and Settings can install hardware support at any moment. Confirmed on this host, where starting the broker brought SMBus DRAM devices online immediately while CPU temperature stayed dark. Retry on a 20 second backoff while the reader is absent, and drop the reader on a failed read so a stopped or crashed broker heals once SCM restarts it instead of staying dead. The all-sources-absent early return is gone: it would have made the retry unreachable on exactly the hosts that need it, and merge_snapshot already no-ops when every source is missing. Co-Authored-By: Nova (Claude Opus 5) --- crates/hypercolor-core/src/input/sensor.rs | 77 ++++++++++++++++++---- 1 file changed, 63 insertions(+), 14 deletions(-) diff --git a/crates/hypercolor-core/src/input/sensor.rs b/crates/hypercolor-core/src/input/sensor.rs index eb0371056..083520ba7 100644 --- a/crates/hypercolor-core/src/input/sensor.rs +++ b/crates/hypercolor-core/src/input/sensor.rs @@ -17,6 +17,13 @@ use sysinfo::{ const DEFAULT_SENSOR_POLL_INTERVAL: Duration = Duration::from_secs(2); const BYTES_PER_MEGABYTE: f64 = 1_000_000.0; +/// Backoff between attempts to open the Windows PawnIO CPU temperature +/// reader while it is unavailable. Long enough that a permanently +/// PawnIO-less host pays almost nothing, short enough that installing +/// hardware support feels immediate rather than requiring a restart. +#[cfg(target_os = "windows")] +const CPU_TEMP_REPROBE_INTERVAL: Duration = Duration::from_secs(20); + /// Background poller that publishes latest-value system telemetry snapshots. pub struct SensorPoller { interval: Duration, @@ -143,7 +150,7 @@ struct SystemSampler { components: Components, nvidia: Option, #[cfg(target_os = "windows")] - windows: Option, + windows: WindowsSensorExtras, } impl SystemSampler { @@ -208,9 +215,7 @@ impl SystemSampler { } #[cfg(target_os = "windows")] - if let Some(windows) = self.windows.as_mut() { - windows.merge_snapshot(&mut snapshot); - } + self.windows.merge_snapshot(&mut snapshot); snapshot } @@ -361,6 +366,8 @@ struct WindowsSensorExtras { /// Set after we've emitted at least one warn about a PawnIO CPU temp /// failure, so the per-poll error doesn't spam the log every 2 seconds. cpu_temp_logged_error: bool, + /// When to next attempt opening the PawnIO reader, while it is absent. + cpu_temp_next_probe: Option, libre_hardware: Option, open_hardware: Option, acpi_zones: Option, @@ -369,7 +376,7 @@ struct WindowsSensorExtras { #[cfg(target_os = "windows")] impl WindowsSensorExtras { - fn new() -> Option { + fn new() -> Self { // First-class CPU temp source: PawnIO MSR/SMN reads. Fails cleanly // if PawnIO isn't installed yet — the user will see CPU temp once // they accept the hardware-support prompt (which they need for @@ -410,25 +417,60 @@ impl WindowsSensorExtras { .map_err(|err| debug!("ROOT\\WMI namespace not present for ACPI thermal zones: {err}")) .ok(); - if cpu_temp.is_none() - && libre_hardware.is_none() - && open_hardware.is_none() - && acpi_zones.is_none() - { - return None; - } + // Deliberately no early return when every source is absent: the + // PawnIO reader is re-probed from merge_snapshot, and bailing here + // would make that unreachable on the exact hosts that need it. + let cpu_temp_next_probe = cpu_temp + .is_none() + .then(|| std::time::Instant::now() + CPU_TEMP_REPROBE_INTERVAL); - Some(Self { + Self { cpu_temp, cpu_temp_logged_error: false, + cpu_temp_next_probe, libre_hardware, open_hardware, acpi_zones, acpi_zones_enabled: true, - }) + } + } + + /// Retry opening the PawnIO reader once the backoff expires. + /// + /// The broker routinely arrives after the daemon does: the installer + /// registers it while an upgrade's daemon is already running, PawnIO's + /// kernel driver can need a reboot to bind to SCM, and Settings can + /// install hardware support at any point. Probing once at startup left + /// CPU temperature dead until someone thought to restart the daemon. + fn reprobe_cpu_temp_if_due(&mut self) { + let Some(next_probe) = self.cpu_temp_next_probe else { + return; + }; + if std::time::Instant::now() < next_probe { + return; + } + + match hypercolor_windows_pawnio::CpuTempReader::new() { + Ok(reader) => { + tracing::info!( + vendor = ?reader.vendor(), + "PawnIO CPU temperature reader came online" + ); + self.cpu_temp = Some(reader); + self.cpu_temp_logged_error = false; + self.cpu_temp_next_probe = None; + } + Err(err) => { + debug!("PawnIO CPU temperature reader still unavailable ({err})"); + self.cpu_temp_next_probe = + Some(std::time::Instant::now() + CPU_TEMP_REPROBE_INTERVAL); + } + } } fn merge_snapshot(&mut self, snapshot: &mut SystemSnapshot) { + self.reprobe_cpu_temp_if_due(); + // 1. PawnIO MSR/SMN — first-class CPU temp source. Authoritative // when it works; we won't override it with the WMI fallbacks below. if let Some(reader) = self.cpu_temp.as_mut() { @@ -459,6 +501,13 @@ impl WindowsSensorExtras { tracing::warn!("PawnIO CPU temp read failed: {err}"); self.cpu_temp_logged_error = true; } + // Drop the reader so the re-probe path can rebuild it. + // A stopped or crashed broker is otherwise permanent for + // the daemon's lifetime, and SCM restarting the broker + // underneath us should heal on its own. + self.cpu_temp = None; + self.cpu_temp_next_probe = + Some(std::time::Instant::now() + CPU_TEMP_REPROBE_INTERVAL); } } } From f9ef61b6617421e9be5d703cfe1ce2db01720633 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 17:34:26 -0700 Subject: [PATCH 04/16] style(tests): reflow a packaging assertion to satisfy rustfmt Whitespace only. Co-Authored-By: Nova (Claude Opus 5) --- crates/hypercolor-app/tests/packaging_tests.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/hypercolor-app/tests/packaging_tests.rs b/crates/hypercolor-app/tests/packaging_tests.rs index 4d7eb4f0c..d1dd8d818 100644 --- a/crates/hypercolor-app/tests/packaging_tests.rs +++ b/crates/hypercolor-app/tests/packaging_tests.rs @@ -143,8 +143,7 @@ fn installer_hook_provisions_the_whole_hardware_access_stack() { "postinstall must run the orchestrator that installs PawnIO *and* the SMBus broker" ); assert!( - INSTALLER_HOOKS_NSH - .contains(r#"-BrokerExe "$INSTDIR\tools\hypercolor-smbus-service.exe""#), + INSTALLER_HOOKS_NSH.contains(r#"-BrokerExe "$INSTDIR\tools\hypercolor-smbus-service.exe""#), "the broker must be registered from its Program Files path" ); assert!( From 076594d4df0e452971ef6da044a2c8bb76e06dc8 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 17:52:27 -0700 Subject: [PATCH 05/16] feat(windows): add a Desktop Duplication capture crate Screen capture on Windows had no backend at all. The only implementation was WaylandScreenCaptureInput, and build_input_manager registered a screen source solely under cfg(target_os = "linux"), so toggling capture.enabled on Windows did nothing whatsoever and every screen-reactive effect rendered against an empty input. Desktop Duplication is the right primitive here. It is the compositor's own presented output, it costs nothing while the screen is static, and unlike Windows.Graphics.Capture it draws no yellow border and shows no picker. It also needs no permission grant: Windows has no equivalent of the XDG portal handshake or macOS TCC prompt, so a screen-reactive effect can simply work with nothing to consent to. This lands the FFI layer only; wiring into the input manager follows. It is a separate crate because the workspace forbids unsafe_code and COM interop cannot, matching hypercolor-windows-pawnio and hypercolor-windows-gpu-interop. Readback subsamples by an integer stride instead of copying native resolution. Ambient lighting reduces the frame to a coarse sector grid, so point sampling is indistinguishable in the output while letting the loop skip most mapped staging rows, which matters when a 4K desktop is 33 MB per frame of write-combined memory. Frames whose LastPresentTime is zero carry only a pointer move and are dropped before readback, and access loss from mode changes, full-screen takeover, or the UAC secure desktop rebuilds the duplication interface in place. Verified on a 3840x2160 display: the live smoke tests captured real frames at 1280x720, tightly packed and fully opaque, across repeated acquisitions. The tests degrade to skips on hosts without a duplicatable output so the Linux-only CI stays green. Co-Authored-By: Nova (Claude Opus 5) --- Cargo.lock | 9 + crates/hypercolor-windows-capture/Cargo.toml | 29 ++ .../src/duplication.rs | 422 ++++++++++++++++++ crates/hypercolor-windows-capture/src/lib.rs | 32 ++ .../hypercolor-windows-capture/src/shared.rs | 105 +++++ .../hypercolor-windows-capture/src/stubs.rs | 45 ++ .../tests/duplication_tests.rs | 137 ++++++ .../tests/subsample_tests.rs | 54 +++ 8 files changed, 833 insertions(+) create mode 100644 crates/hypercolor-windows-capture/Cargo.toml create mode 100644 crates/hypercolor-windows-capture/src/duplication.rs create mode 100644 crates/hypercolor-windows-capture/src/lib.rs create mode 100644 crates/hypercolor-windows-capture/src/shared.rs create mode 100644 crates/hypercolor-windows-capture/src/stubs.rs create mode 100644 crates/hypercolor-windows-capture/tests/duplication_tests.rs create mode 100644 crates/hypercolor-windows-capture/tests/subsample_tests.rs diff --git a/Cargo.lock b/Cargo.lock index f055069f7..758a8ac4e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5120,6 +5120,15 @@ dependencies = [ "uuid", ] +[[package]] +name = "hypercolor-windows-capture" +version = "0.2.1" +dependencies = [ + "thiserror 2.0.18", + "tracing", + "windows 0.62.2", +] + [[package]] name = "hypercolor-windows-gpu-interop" version = "0.2.1" diff --git a/crates/hypercolor-windows-capture/Cargo.toml b/crates/hypercolor-windows-capture/Cargo.toml new file mode 100644 index 000000000..17b9e0c5b --- /dev/null +++ b/crates/hypercolor-windows-capture/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "hypercolor-windows-capture" +description = "Windows DXGI Desktop Duplication screen capture for Hypercolor" +version.workspace = true +edition.workspace = true +authors.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +[lints.rust] +unsafe_code = "allow" + +[lints.clippy] +undocumented_unsafe_blocks = "deny" +unwrap_used = "deny" + +[dependencies] +thiserror = { workspace = true } +tracing = { workspace = true } + +[target.'cfg(target_os = "windows")'.dependencies] +windows = { version = "0.62", default-features = false, features = [ + "Win32_Foundation", + "Win32_Graphics_Direct3D", + "Win32_Graphics_Direct3D11", + "Win32_Graphics_Dxgi", + "Win32_Graphics_Dxgi_Common", +] } diff --git a/crates/hypercolor-windows-capture/src/duplication.rs b/crates/hypercolor-windows-capture/src/duplication.rs new file mode 100644 index 000000000..8e2499f02 --- /dev/null +++ b/crates/hypercolor-windows-capture/src/duplication.rs @@ -0,0 +1,422 @@ +//! DXGI Desktop Duplication capture loop. + +use std::time::Duration; + +use tracing::{debug, warn}; +use windows::Win32::Foundation::{E_ACCESSDENIED, HMODULE}; +use windows::Win32::Graphics::Direct3D::{D3D_DRIVER_TYPE_UNKNOWN, D3D_FEATURE_LEVEL_11_0}; +use windows::Win32::Graphics::Direct3D11::{ + D3D11_CPU_ACCESS_READ, D3D11_CREATE_DEVICE_BGRA_SUPPORT, D3D11_MAP_READ, + D3D11_MAPPED_SUBRESOURCE, D3D11_SDK_VERSION, D3D11_TEXTURE2D_DESC, D3D11_USAGE_STAGING, + D3D11CreateDevice, ID3D11Device, ID3D11DeviceContext, ID3D11Texture2D, +}; +use windows::Win32::Graphics::Dxgi::{ + CreateDXGIFactory1, DXGI_ERROR_ACCESS_LOST, DXGI_ERROR_NOT_FOUND, DXGI_ERROR_UNSUPPORTED, + DXGI_ERROR_WAIT_TIMEOUT, DXGI_OUTDUPL_FRAME_INFO, IDXGIAdapter1, IDXGIFactory1, IDXGIOutput, + IDXGIOutput1, IDXGIOutputDuplication, IDXGIResource, +}; +use windows::core::Interface; + +use crate::shared::{CaptureError, CaptureResult, Frame, subsample_stride, subsampled_extent}; + +/// Bytes per pixel in both the duplicated surface and our RGBA output. +const BYTES_PER_PIXEL: usize = 4; + +/// Enumerate every output across every adapter, in adapter-then-output order. +fn enumerate_outputs() -> CaptureResult> { + // SAFETY: CreateDXGIFactory1 is a plain COM factory call with no + // preconditions; the returned interface is checked by `?`. + let factory: IDXGIFactory1 = unsafe { CreateDXGIFactory1() } + .map_err(|source| CaptureError::windows("create DXGI factory", source))?; + + let mut outputs = Vec::new(); + for adapter_index in 0.. { + // SAFETY: EnumAdapters1 reports DXGI_ERROR_NOT_FOUND past the last + // adapter, which is the documented loop terminator. + let adapter: IDXGIAdapter1 = match unsafe { factory.EnumAdapters1(adapter_index) } { + Ok(adapter) => adapter, + Err(error) if error.code() == DXGI_ERROR_NOT_FOUND => break, + Err(source) => { + return Err(CaptureError::windows("enumerate DXGI adapters", source)); + } + }; + + for output_index in 0.. { + // SAFETY: same NOT_FOUND termination contract as adapters. + match unsafe { adapter.EnumOutputs(output_index) } { + Ok(output) => outputs.push((adapter.clone(), output)), + Err(error) if error.code() == DXGI_ERROR_NOT_FOUND => break, + Err(source) => { + return Err(CaptureError::windows("enumerate DXGI outputs", source)); + } + } + } + } + + Ok(outputs) +} + +/// How many display outputs are attached. +pub(crate) fn output_count() -> CaptureResult { + enumerate_outputs().map(|outputs| outputs.len()) +} + +/// A live Desktop Duplication session for one display output. +pub struct DesktopDuplicator { + monitor: usize, + max_width: u32, + device: ID3D11Device, + context: ID3D11DeviceContext, + output: IDXGIOutput1, + duplication: IDXGIOutputDuplication, + /// CPU-readable copy target, rebuilt when the desktop dimensions change. + staging: Option<(ID3D11Texture2D, u32, u32)>, + /// Reused RGBA output buffer. + rgba: Vec, + /// Set while a duplicated frame is held and must be released before the + /// next acquire. DXGI rejects back-to-back acquires without a release. + frame_held: bool, + output_width: u32, + output_height: u32, +} + +impl DesktopDuplicator { + /// Open Desktop Duplication for `monitor`, subsampling to `max_width`. + /// + /// # Errors + /// + /// Returns [`CaptureError::MonitorNotFound`] when the index is out of + /// range, [`CaptureError::AlreadyDuplicating`] when another process holds + /// the duplication interface, or [`CaptureError::Windows`] for any other + /// D3D11/DXGI failure. + pub fn new(monitor: usize, max_width: u32) -> CaptureResult { + let outputs = enumerate_outputs()?; + let available = outputs.len(); + let (adapter, output) = + outputs + .into_iter() + .nth(monitor) + .ok_or(CaptureError::MonitorNotFound { + requested: monitor, + available, + })?; + + let (device, context) = create_device(&adapter)?; + let output = output + .cast::() + .map_err(|source| CaptureError::windows("query IDXGIOutput1", source))?; + let duplication = duplicate_output(&output, &device)?; + let (output_width, output_height) = duplication_extent(&duplication); + + Ok(Self { + monitor, + max_width, + device, + context, + output, + duplication, + staging: None, + rgba: Vec::new(), + frame_held: false, + output_width, + output_height, + }) + } + + /// Which monitor index this duplicator is bound to. + #[must_use] + pub const fn monitor(&self) -> usize { + self.monitor + } + + /// Native (pre-subsample) desktop dimensions. + #[must_use] + pub const fn native_extent(&self) -> (u32, u32) { + (self.output_width, self.output_height) + } + + /// Change the subsample target for subsequent frames. + pub const fn set_max_width(&mut self, max_width: u32) { + self.max_width = max_width; + } + + /// Wait up to `timeout` for the next desktop frame. + /// + /// Returns `Ok(None)` when nothing new arrived, which is the common and + /// cheap case: DXGI reports a timeout whenever the desktop is static, and + /// a mouse-only update carries no new desktop image worth re-analyzing. + /// + /// # Errors + /// + /// Returns [`CaptureError::Windows`] when acquiring, copying, or mapping + /// the frame fails for a reason that is not recoverable in place. Access + /// loss is handled internally by rebuilding the duplication interface. + pub fn next_frame(&mut self, timeout: Duration) -> CaptureResult>> { + self.release_frame(); + + let timeout_ms = u32::try_from(timeout.as_millis()).unwrap_or(u32::MAX); + let mut frame_info = DXGI_OUTDUPL_FRAME_INFO::default(); + let mut resource: Option = None; + + // SAFETY: both out-params are owned locals living past the call, and + // the duplication interface is kept alive by `self`. + let acquire = unsafe { + self.duplication + .AcquireNextFrame(timeout_ms, &mut frame_info, &mut resource) + }; + + if let Err(error) = acquire { + return match error.code() { + DXGI_ERROR_WAIT_TIMEOUT => Ok(None), + DXGI_ERROR_ACCESS_LOST => { + // Mode change, a full-screen app taking over, or the + // secure desktop during a UAC prompt. All are transient + // and all are fixed by re-duplicating the output. + debug!("desktop duplication access lost; rebuilding session"); + self.rebuild()?; + Ok(None) + } + _ => Err(CaptureError::windows("acquire duplicated frame", error)), + }; + } + self.frame_held = true; + + // LastPresentTime stays zero when only the pointer moved. There is no + // new desktop image behind that, so re-running the sector grid would + // burn a readback to produce identical colors. + if frame_info.LastPresentTime == 0 { + self.release_frame(); + return Ok(None); + } + + let Some(resource) = resource else { + self.release_frame(); + return Ok(None); + }; + + let texture = resource + .cast::() + .map_err(|source| CaptureError::windows("query duplicated texture", source))?; + + let result = self.read_back(&texture); + self.release_frame(); + let (width, height) = result?; + + Ok(Some(Frame { + width, + height, + rgba: &self.rgba[..(width as usize * height as usize * BYTES_PER_PIXEL)], + })) + } + + /// Copy the duplicated texture into staging, then subsample into `rgba`. + fn read_back(&mut self, texture: &ID3D11Texture2D) -> CaptureResult<(u32, u32)> { + let mut desc = D3D11_TEXTURE2D_DESC::default(); + // SAFETY: GetDesc fills a caller-owned struct and cannot fail. + unsafe { texture.GetDesc(&mut desc) }; + + let staging = self.ensure_staging(&desc)?; + + // SAFETY: both textures are same-desc 2D textures on this device; + // CopyResource is the documented duplication readback path. + unsafe { self.context.CopyResource(&staging, texture) }; + + let mut mapped = D3D11_MAPPED_SUBRESOURCE::default(); + // SAFETY: staging was created USAGE_STAGING | CPU_ACCESS_READ, so + // subresource 0 is mappable for reads. + unsafe { + self.context + .Map(&staging, 0, D3D11_MAP_READ, 0, Some(&mut mapped)) + } + .map_err(|source| CaptureError::windows("map staging texture", source))?; + + let extent = self.copy_mapped_rows(&mapped, desc.Width, desc.Height); + + // SAFETY: pairs with the Map above on the same subresource. + unsafe { self.context.Unmap(&staging, 0) }; + + Ok(extent) + } + + /// Subsample BGRA staging rows into the packed RGBA output buffer. + fn copy_mapped_rows( + &mut self, + mapped: &D3D11_MAPPED_SUBRESOURCE, + width: u32, + height: u32, + ) -> (u32, u32) { + let stride = subsample_stride(width, self.max_width); + let out_width = subsampled_extent(width, stride); + let out_height = subsampled_extent(height, stride); + let row_pitch = mapped.RowPitch as usize; + let source_len = row_pitch * height as usize; + + self.rgba.resize( + out_width as usize * out_height as usize * BYTES_PER_PIXEL, + 0, + ); + + // SAFETY: Map handed back a buffer of at least RowPitch * Height + // bytes for this subresource, and it stays valid until Unmap. + let source = unsafe { std::slice::from_raw_parts(mapped.pData.cast::(), source_len) }; + + for out_y in 0..out_height as usize { + let src_row_start = (out_y * stride as usize) * row_pitch; + let dst_row_start = out_y * out_width as usize * BYTES_PER_PIXEL; + for out_x in 0..out_width as usize { + let src = src_row_start + (out_x * stride as usize) * BYTES_PER_PIXEL; + let dst = dst_row_start + out_x * BYTES_PER_PIXEL; + // Desktop Duplication hands back BGRA; the pipeline wants RGBA. + self.rgba[dst] = source[src + 2]; + self.rgba[dst + 1] = source[src + 1]; + self.rgba[dst + 2] = source[src]; + self.rgba[dst + 3] = 0xFF; + } + } + + (out_width, out_height) + } + + /// Return a staging texture matching `desc`, rebuilding on size change. + /// + /// Hands back a clone rather than a borrow: COM interfaces are refcounted + /// so the clone is an AddRef, and it frees `self` for the copy and map + /// calls that immediately follow. + fn ensure_staging(&mut self, desc: &D3D11_TEXTURE2D_DESC) -> CaptureResult { + let matches = self + .staging + .as_ref() + .is_some_and(|(_, width, height)| *width == desc.Width && *height == desc.Height); + + if !matches { + let staging_desc = D3D11_TEXTURE2D_DESC { + Usage: D3D11_USAGE_STAGING, + BindFlags: 0, + CPUAccessFlags: u32::try_from(D3D11_CPU_ACCESS_READ.0).unwrap_or_default(), + MiscFlags: 0, + ..*desc + }; + + let mut texture: Option = None; + // SAFETY: staging_desc is a valid staging description and the + // out-param outlives the call. + unsafe { + self.device + .CreateTexture2D(&staging_desc, None, Some(&mut texture)) + } + .map_err(|source| CaptureError::windows("create staging texture", source))?; + + let texture = texture.ok_or_else(|| { + CaptureError::windows( + "create staging texture", + "CreateTexture2D returned no texture", + ) + })?; + self.staging = Some((texture, desc.Width, desc.Height)); + self.output_width = desc.Width; + self.output_height = desc.Height; + } + + self.staging + .as_ref() + .map(|(texture, _, _)| texture.clone()) + .ok_or_else(|| { + CaptureError::windows( + "resolve staging texture", + "staging texture missing after creation", + ) + }) + } + + /// Drop the duplication interface and open a fresh one. + fn rebuild(&mut self) -> CaptureResult<()> { + self.release_frame(); + self.staging = None; + self.duplication = duplicate_output(&self.output, &self.device)?; + let (width, height) = duplication_extent(&self.duplication); + self.output_width = width; + self.output_height = height; + Ok(()) + } + + /// Release a held frame if there is one. Safe to call unconditionally. + fn release_frame(&mut self) { + if !self.frame_held { + return; + } + self.frame_held = false; + // SAFETY: paired with a successful AcquireNextFrame. + if let Err(error) = unsafe { self.duplication.ReleaseFrame() } { + // Access loss here is normal during mode changes and is repaired + // on the next acquire, so this stays a debug line. + debug!(%error, "releasing duplicated frame failed"); + } + } +} + +impl Drop for DesktopDuplicator { + fn drop(&mut self) { + self.release_frame(); + } +} + +/// Create a D3D11 device on `adapter`. +fn create_device(adapter: &IDXGIAdapter1) -> CaptureResult<(ID3D11Device, ID3D11DeviceContext)> { + let mut device: Option = None; + let mut context: Option = None; + let feature_levels = [D3D_FEATURE_LEVEL_11_0]; + + // SAFETY: the adapter outlives the call, DRIVER_TYPE_UNKNOWN is required + // when passing an explicit adapter, and both out-params are owned locals. + unsafe { + D3D11CreateDevice( + adapter, + D3D_DRIVER_TYPE_UNKNOWN, + HMODULE::default(), + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + Some(&feature_levels), + D3D11_SDK_VERSION, + Some(&mut device), + None, + Some(&mut context), + ) + } + .map_err(|source| CaptureError::windows("create D3D11 device", source))?; + + match (device, context) { + (Some(device), Some(context)) => Ok((device, context)), + _ => Err(CaptureError::windows( + "create D3D11 device", + "D3D11CreateDevice returned no device", + )), + } +} + +/// Open the duplication interface, mapping the two well-known refusals. +fn duplicate_output( + output: &IDXGIOutput1, + device: &ID3D11Device, +) -> CaptureResult { + // SAFETY: both interfaces outlive the call. + unsafe { output.DuplicateOutput(device) }.map_err(|source| match source.code() { + // Documented as "already duplicating"; Windows allows one per output. + E_ACCESSDENIED => CaptureError::AlreadyDuplicating, + // Raised on hybrid-graphics hosts when the desktop is not on this + // adapter's output. Callers surface it as "capture unavailable". + DXGI_ERROR_UNSUPPORTED => { + CaptureError::windows("duplicate output (desktop is not on this adapter)", source) + } + _ => CaptureError::windows("duplicate output", source), + }) +} + +/// Read the duplicated desktop dimensions, defaulting to zero on failure. +fn duplication_extent(duplication: &IDXGIOutputDuplication) -> (u32, u32) { + // SAFETY: GetDesc reads cached descriptor state and cannot fail. + let desc = unsafe { duplication.GetDesc() }; + let mode = desc.ModeDesc; + if mode.Width == 0 || mode.Height == 0 { + warn!("desktop duplication reported a zero-sized mode"); + } + (mode.Width, mode.Height) +} diff --git a/crates/hypercolor-windows-capture/src/lib.rs b/crates/hypercolor-windows-capture/src/lib.rs new file mode 100644 index 000000000..0f457f5c7 --- /dev/null +++ b/crates/hypercolor-windows-capture/src/lib.rs @@ -0,0 +1,32 @@ +//! Windows desktop screen capture for Hypercolor, via DXGI Desktop Duplication. +//! +//! Desktop Duplication is the right primitive for ambient lighting: it is the +//! compositor's own presented output, it costs nothing when the screen is +//! static, and unlike Windows.Graphics.Capture it draws no yellow border and +//! shows no picker. It also needs no permission grant of any kind — there is +//! no Windows equivalent of the XDG portal handshake or macOS TCC prompt, so a +//! screen-reactive effect can simply work. +//! +//! This crate is deliberately thin: it owns the D3D11 device, the duplication +//! interface, and the staging readback, and hands out RGBA frames. Everything +//! downstream (sector grids, letterbox detection, smoothing) lives in +//! `hypercolor-core` and stays backend-agnostic. It exists as its own crate +//! because the workspace forbids `unsafe_code` and COM interop cannot. + +#[cfg(target_os = "windows")] +mod duplication; + +#[cfg(target_os = "windows")] +pub use duplication::DesktopDuplicator; + +mod shared; + +pub use shared::{ + CaptureError, CaptureResult, Frame, monitor_count, subsample_stride, subsampled_extent, +}; + +#[cfg(not(target_os = "windows"))] +mod stubs; + +#[cfg(not(target_os = "windows"))] +pub use stubs::DesktopDuplicator; diff --git a/crates/hypercolor-windows-capture/src/shared.rs b/crates/hypercolor-windows-capture/src/shared.rs new file mode 100644 index 000000000..95020c679 --- /dev/null +++ b/crates/hypercolor-windows-capture/src/shared.rs @@ -0,0 +1,105 @@ +//! Platform-neutral surface: errors, frame view, and the subsample math. + +use thiserror::Error; + +/// Screen capture result type. +pub type CaptureResult = Result; + +/// Screen capture failures. +#[derive(Debug, Error)] +pub enum CaptureError { + /// Desktop Duplication is a Windows-only API. + #[error("desktop screen capture is only available on Windows")] + UnsupportedPlatform, + + /// No display output matched the requested monitor index. + #[error("monitor {requested} not found ({available} attached)")] + MonitorNotFound { + /// Zero-based monitor index that was requested. + requested: usize, + /// How many outputs enumeration actually found. + available: usize, + }, + + /// Another process already holds the duplication interface for this output. + /// + /// Windows permits exactly one duplication per output per process, and a + /// handful of tools (some capture utilities, other ambient-lighting apps) + /// hold theirs for their whole lifetime. + #[error("another application is already duplicating this display")] + AlreadyDuplicating, + + /// A Windows API call failed. + /// + /// Carries a rendered message rather than the `windows` error type: with + /// `default-features = false` that type does not implement + /// `std::error::Error`, and the HRESULT text is the part worth keeping. + #[error("{context}: {message}")] + Windows { + /// What we were attempting. + context: &'static str, + /// Rendered HRESULT description. + message: String, + }, +} + +impl CaptureError { + /// Build a [`CaptureError::Windows`] from anything printable. + pub(crate) fn windows(context: &'static str, message: impl std::fmt::Display) -> Self { + Self::Windows { + context, + message: message.to_string(), + } + } +} + +/// A borrowed RGBA frame produced by the capture backend. +/// +/// The pixel buffer is owned by the duplicator and reused between frames, so +/// consumers must copy anything they need to keep. +#[derive(Debug)] +pub struct Frame<'a> { + /// Frame width in pixels, after subsampling. + pub width: u32, + /// Frame height in pixels, after subsampling. + pub height: u32, + /// Tightly packed RGBA8 pixels, `width * height * 4` bytes. + pub rgba: &'a [u8], +} + +/// Number of attached display outputs, or zero when capture is unavailable. +#[must_use] +pub fn monitor_count() -> usize { + #[cfg(target_os = "windows")] + { + crate::duplication::output_count().unwrap_or(0) + } + #[cfg(not(target_os = "windows"))] + { + 0 + } +} + +/// Integer subsample stride that brings `source` at or under `target`. +/// +/// Ambient lighting reduces the frame to a coarse sector grid, so point +/// sampling every Nth pixel is indistinguishable from a box filter in the +/// output while letting readback skip most of the mapped staging rows. That +/// matters: a 4K desktop is 33 MB per frame, and mapped staging memory is +/// write-combined, so untouched rows are the cheapest possible rows. +#[must_use] +pub fn subsample_stride(source: u32, target: u32) -> u32 { + if target == 0 || source <= target { + return 1; + } + source.div_ceil(target).max(1) +} + +/// Dimension after applying `stride` to `source`. +#[must_use] +pub const fn subsampled_extent(source: u32, stride: u32) -> u32 { + if stride <= 1 { + return source; + } + source.div_ceil(stride) +} diff --git a/crates/hypercolor-windows-capture/src/stubs.rs b/crates/hypercolor-windows-capture/src/stubs.rs new file mode 100644 index 000000000..1768ed55c --- /dev/null +++ b/crates/hypercolor-windows-capture/src/stubs.rs @@ -0,0 +1,45 @@ +//! Non-Windows stand-in so downstream crates compile unconditionally. + +use std::time::Duration; + +use crate::shared::{CaptureError, CaptureResult, Frame}; + +/// Desktop Duplication placeholder for platforms without the API. +pub struct DesktopDuplicator { + _private: (), +} + +impl DesktopDuplicator { + /// Always fails: Desktop Duplication is Windows-only. + /// + /// # Errors + /// + /// Always returns [`CaptureError::UnsupportedPlatform`]. + pub const fn new(_monitor: usize, _max_width: u32) -> CaptureResult { + Err(CaptureError::UnsupportedPlatform) + } + + /// Monitor index this duplicator would be bound to. + #[must_use] + pub const fn monitor(&self) -> usize { + 0 + } + + /// Native desktop dimensions. + #[must_use] + pub const fn native_extent(&self) -> (u32, u32) { + (0, 0) + } + + /// Change the subsample target for subsequent frames. + pub const fn set_max_width(&mut self, _max_width: u32) {} + + /// Always fails: Desktop Duplication is Windows-only. + /// + /// # Errors + /// + /// Always returns [`CaptureError::UnsupportedPlatform`]. + pub const fn next_frame(&mut self, _timeout: Duration) -> CaptureResult>> { + Err(CaptureError::UnsupportedPlatform) + } +} diff --git a/crates/hypercolor-windows-capture/tests/duplication_tests.rs b/crates/hypercolor-windows-capture/tests/duplication_tests.rs new file mode 100644 index 000000000..cb0da42d4 --- /dev/null +++ b/crates/hypercolor-windows-capture/tests/duplication_tests.rs @@ -0,0 +1,137 @@ +//! Live Desktop Duplication smoke tests. +//! +//! These drive the real API against whatever display the test host has. CI is +//! Linux-only, and a headless or RDP session has no duplicatable output, so +//! every test degrades to a skip rather than a failure when capture is +//! unavailable. When a display *is* present they assert real pixel geometry, +//! which is the part that catches a broken readback. + +#![cfg(target_os = "windows")] + +use std::time::{Duration, Instant}; + +use hypercolor_windows_capture::{CaptureError, DesktopDuplicator, monitor_count}; + +/// Subsample target used across the tests, matching the daemon's default. +const MAX_WIDTH: u32 = 1280; + +/// Open a duplicator, or return `None` when this host cannot capture. +fn duplicator_or_skip() -> Option { + if monitor_count() == 0 { + eprintln!("skipping: no display outputs attached"); + return None; + } + + match DesktopDuplicator::new(0, MAX_WIDTH) { + Ok(duplicator) => Some(duplicator), + Err(CaptureError::AlreadyDuplicating) => { + eprintln!("skipping: another process holds the duplication interface"); + None + } + Err(error) => { + eprintln!("skipping: duplication unavailable ({error})"); + None + } + } +} + +/// Pump until a frame arrives or the budget expires. A static desktop +/// legitimately produces nothing, so absence is not a failure. +fn wait_for_frame( + duplicator: &mut DesktopDuplicator, + budget: Duration, +) -> Option<(u32, u32, usize)> { + let deadline = Instant::now() + budget; + while Instant::now() < deadline { + match duplicator.next_frame(Duration::from_millis(120)) { + Ok(Some(frame)) => return Some((frame.width, frame.height, frame.rgba.len())), + Ok(None) => {} + Err(error) => { + eprintln!("capture error while waiting for a frame: {error}"); + return None; + } + } + } + None +} + +#[test] +fn monitor_count_is_sane() { + // Purely a smoke check that enumeration does not panic or explode; a + // headless agent host legitimately reports zero. + let count = monitor_count(); + assert!(count < 64, "implausible monitor count: {count}"); +} + +#[test] +fn opening_a_missing_monitor_reports_not_found() { + let Err(error) = DesktopDuplicator::new(9_999, MAX_WIDTH) else { + panic!("monitor index 9999 should not resolve to a real output"); + }; + + match error { + CaptureError::MonitorNotFound { requested, .. } => assert_eq!(requested, 9_999), + // A host with no D3D11 at all fails earlier, which is still correct. + other => eprintln!("no monitors to enumerate against ({other})"), + } +} + +#[test] +fn captures_a_frame_with_consistent_geometry() { + let Some(mut duplicator) = duplicator_or_skip() else { + return; + }; + + let Some((width, height, len)) = wait_for_frame(&mut duplicator, Duration::from_secs(3)) else { + eprintln!("skipping assertions: desktop produced no frame within the budget"); + return; + }; + + assert!(width > 0 && height > 0, "frame must have real extent"); + assert!( + width <= MAX_WIDTH, + "subsampling should hold width at or under {MAX_WIDTH}, got {width}" + ); + assert_eq!( + len, + width as usize * height as usize * 4, + "buffer must be tightly packed RGBA" + ); + + let (native_width, native_height) = duplicator.native_extent(); + assert!( + native_width >= width && native_height >= height, + "subsampled frame must not exceed the native desktop" + ); + eprintln!("captured {width}x{height} from a {native_width}x{native_height} desktop"); +} + +#[test] +fn alpha_is_opaque_across_repeated_acquisitions() { + let Some(mut duplicator) = duplicator_or_skip() else { + return; + }; + + // Repeated acquisitions exercise the release-before-acquire contract: + // DXGI refuses a second acquire while the first frame is still held, so a + // missing ReleaseFrame would surface here rather than in production. + let mut checked = 0_u32; + let deadline = Instant::now() + Duration::from_secs(5); + while checked < 2 && Instant::now() < deadline { + match duplicator.next_frame(Duration::from_millis(150)) { + Ok(Some(frame)) => { + assert!( + frame.rgba.chunks_exact(4).all(|pixel| pixel[3] == 0xFF), + "every captured pixel must be fully opaque" + ); + checked += 1; + } + Ok(None) => {} + Err(error) => panic!("repeated acquisition failed: {error}"), + } + } + + if checked < 2 { + eprintln!("desktop was static; only {checked} frame(s) verified"); + } +} diff --git a/crates/hypercolor-windows-capture/tests/subsample_tests.rs b/crates/hypercolor-windows-capture/tests/subsample_tests.rs new file mode 100644 index 000000000..2ccebd8d1 --- /dev/null +++ b/crates/hypercolor-windows-capture/tests/subsample_tests.rs @@ -0,0 +1,54 @@ +//! Subsample geometry, the part of the readback that has to be exactly right. +//! +//! An off-by-one here silently truncates the bottom or right edge of the +//! desktop, which an ambient rig shows as a dead strip of LEDs rather than as +//! an error. + +use hypercolor_windows_capture::{subsample_stride, subsampled_extent}; + +#[test] +fn stride_is_one_when_source_already_fits() { + assert_eq!(subsample_stride(1280, 1280), 1); + assert_eq!(subsample_stride(800, 1280), 1); +} + +#[test] +fn stride_brings_every_common_resolution_under_target() { + for (source, target) in [ + (3840, 1280), + (2560, 1280), + (1920, 1280), + (7680, 1280), + (5120, 1280), + (3440, 1280), + ] { + let stride = subsample_stride(source, target); + assert!( + subsampled_extent(source, stride) <= target, + "{source} at stride {stride} should fit under {target}" + ); + } +} + +#[test] +fn stride_never_divides_by_zero() { + assert_eq!(subsample_stride(1920, 0), 1); + assert_eq!(subsample_stride(0, 1280), 1); +} + +#[test] +fn extent_rounds_up_so_a_trailing_partial_step_still_samples() { + // Truncating division would drop the last row of an odd-height desktop. + assert_eq!(subsampled_extent(1080, 2), 540); + assert_eq!(subsampled_extent(1081, 2), 541); + assert_eq!(subsampled_extent(1920, 1), 1920); +} + +#[test] +fn a_4k_desktop_lands_on_the_documented_geometry() { + // The shape the live capture test observes on a 3840x2160 display. + let stride = subsample_stride(3840, 1280); + assert_eq!(stride, 3); + assert_eq!(subsampled_extent(3840, stride), 1280); + assert_eq!(subsampled_extent(2160, stride), 720); +} From 0b4ca71adfd52200dd7456b43ddb3f3a9be12a8f Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 18:07:02 -0700 Subject: [PATCH 06/16] feat(windows): wire screen capture into the input manager Registers WindowsScreenCaptureInput so Desktop Duplication actually reaches the render pipeline. A worker thread owns the capture session and the analysis pipeline and publishes ScreenData snapshots, mirroring the Wayland source's shape; the render loop only clones the latest snapshot. The duplication interface opens on demand and is released the moment capture goes idle, because Windows allows one duplication per output per process and other ambient-lighting tools want it too. capture.enabled now defaults to true on Windows and stays false elsewhere. This is a privacy-relevant default, so the reasoning matters: Desktop Duplication has no permission prompt, no source picker, and no capture indicator, meaning there is nothing for a user to consent to and nothing an opt-in toggle would be protecting. The XDG portal opens a picker and macOS raises a TCC prompt, and forcing either at daemon start would be an ambush, so both stay opt-in. Enabling only grants permission: capture stays closed until a screen-reactive effect creates demand, so the common case is still no capture at all. Multi-monitor selection reuses the existing free-form capture.source field, which the portal leaves at "auto" on Linux. Windows addresses outputs directly and accepts "monitor:N", "display:N", or a bare index, falling back to the primary display for anything else so an unknown value is never an error. Verified: clippy -D warnings clean on types, core, the capture crate and the daemon; 36 types config tests, 43 core screen tests, and 9 capture crate tests pass, including a live capture off a 3840x2160 display. Co-Authored-By: Nova (Claude Opus 5) --- Cargo.lock | 1 + crates/hypercolor-core/Cargo.toml | 1 + .../hypercolor-core/src/input/screen/mod.rs | 29 ++ .../src/input/screen/windows.rs | 366 ++++++++++++++++++ crates/hypercolor-core/tests/screen_tests.rs | 34 ++ .../hypercolor-daemon/src/startup/services.rs | 18 +- crates/hypercolor-types/src/config.rs | 18 +- crates/hypercolor-types/tests/config_tests.rs | 14 +- docs/specs/14-screen-capture.md | 92 +++-- 9 files changed, 527 insertions(+), 46 deletions(-) create mode 100644 crates/hypercolor-core/src/input/screen/windows.rs diff --git a/Cargo.lock b/Cargo.lock index 758a8ac4e..722dd9c71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4701,6 +4701,7 @@ dependencies = [ "hypercolor-linux-gpu-interop", "hypercolor-macos-gpu-interop", "hypercolor-types", + "hypercolor-windows-capture", "hypercolor-windows-gpu-interop", "hypercolor-windows-pawnio", "image", diff --git a/crates/hypercolor-core/Cargo.toml b/crates/hypercolor-core/Cargo.toml index 66037c9c8..4ef37daa3 100644 --- a/crates/hypercolor-core/Cargo.toml +++ b/crates/hypercolor-core/Cargo.toml @@ -99,6 +99,7 @@ raw-window-handle = { version = "0.6", optional = true } tao = { version = "0.34", default-features = false, features = ["rwh_06"], optional = true } wmi = { workspace = true } hypercolor-windows-pawnio = { path = "../hypercolor-windows-pawnio" } +hypercolor-windows-capture = { path = "../hypercolor-windows-capture" } [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } diff --git a/crates/hypercolor-core/src/input/screen/mod.rs b/crates/hypercolor-core/src/input/screen/mod.rs index 6e787701e..a3f59a2c1 100644 --- a/crates/hypercolor-core/src/input/screen/mod.rs +++ b/crates/hypercolor-core/src/input/screen/mod.rs @@ -20,12 +20,16 @@ pub mod smooth; pub mod tune; #[cfg(target_os = "linux")] pub mod wayland; +#[cfg(target_os = "windows")] +pub mod windows; pub use sector::{LetterboxBars, SectorGrid}; pub use smooth::TemporalSmoother; pub use tune::ColorTuning; #[cfg(target_os = "linux")] pub use wayland::WaylandScreenCaptureInput; +#[cfg(target_os = "windows")] +pub use windows::WindowsScreenCaptureInput; use crate::input::traits::{InputData, InputSource, ScreenData}; use crate::types::canvas::{ @@ -68,6 +72,11 @@ pub struct CaptureConfig { /// XDG portal restore token from a previous session, if any. pub restore_token: Option, + + /// Zero-based display output to capture, for backends that address + /// monitors directly. The XDG portal picks its own source, so this is + /// only consulted on Windows. + pub monitor: usize, } impl Default for CaptureConfig { @@ -82,10 +91,30 @@ impl Default for CaptureConfig { letterbox_enabled: true, tuning: ColorTuning::default(), restore_token: None, + monitor: 0, } } } +/// Parse a configured capture source into a zero-based monitor index. +/// +/// `capture.source` is a free-form string shared across backends. The XDG +/// portal picks its own source and leaves the value at "auto", so this only +/// matters on Windows, which addresses display outputs directly. Accepts +/// "monitor:N", "display:N", or a bare number; anything else, "auto" +/// included, means the primary display. +#[must_use] +pub fn monitor_index_from_source(source: &str) -> usize { + let trimmed = source.trim(); + trimmed + .strip_prefix("monitor:") + .or_else(|| trimmed.strip_prefix("display:")) + .unwrap_or(trimmed) + .trim() + .parse::() + .unwrap_or(0) +} + // ── ScreenCaptureInput ──────────────────────────────────────────────────── /// Screen capture input source implementing [`InputSource`]. diff --git a/crates/hypercolor-core/src/input/screen/windows.rs b/crates/hypercolor-core/src/input/screen/windows.rs new file mode 100644 index 000000000..db296e9e5 --- /dev/null +++ b/crates/hypercolor-core/src/input/screen/windows.rs @@ -0,0 +1,366 @@ +//! Windows screen capture source backed by DXGI Desktop Duplication. +//! +//! Shaped like [`super::wayland::WaylandScreenCaptureInput`]: a worker thread +//! owns the capture session and the analysis pipeline, and the render loop +//! only clones the latest processed [`ScreenData`]. It is markedly simpler +//! than the Wayland source because Windows has nothing to negotiate — no +//! portal handshake, no source picker, no restore token, and no permission +//! grant of any kind. +//! +//! The duplication interface is opened lazily when capture goes active and +//! dropped the moment it goes idle. Windows allows one duplication per output +//! per process, and other ambient-lighting tools want the same interface, so +//! holding it while no effect needs it would be antisocial. + +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, mpsc}; +use std::thread; +use std::time::Duration; + +use anyhow::anyhow; +use hypercolor_windows_capture::{CaptureError, DesktopDuplicator}; +use tracing::{debug, info, warn}; + +use crate::input::screen::{CaptureConfig, ScreenCaptureInput}; +use crate::input::traits::{InputData, InputSource, ScreenData}; + +/// Width the capture backend subsamples to before analysis. +/// +/// Matches the resolution the Wayland source negotiates from PipeWire, so +/// both platforms feed the sector grid comparable input and cost. +const CAPTURE_TARGET_WIDTH: u32 = 1280; + +/// How long a worker waits on DXGI before checking its command channel. +/// +/// Bounded well under a second so a stop or deactivate lands promptly even +/// while the desktop is perfectly static and producing no frames at all. +const FRAME_WAIT: Duration = Duration::from_millis(100); + +/// Backoff after a failed attempt to open the duplication interface. +/// +/// The common cause is another application holding it, which resolves when +/// that application exits, so retrying quietly beats surfacing an error the +/// user cannot act on. +const REOPEN_BACKOFF: Duration = Duration::from_secs(2); + +/// Settings shared between the input source handle and the capture worker. +struct SharedSettings { + config: Mutex, + generation: AtomicU64, +} + +impl SharedSettings { + fn snapshot(&self) -> CaptureConfig { + self.config + .lock() + .map_or_else(|_| CaptureConfig::default(), |config| config.clone()) + } +} + +/// Windows-only live screen capture input source. +pub struct WindowsScreenCaptureInput { + settings: Arc, + running: bool, + capture_active: bool, + latest_snapshot: Arc>>, + worker: Option, +} + +struct CaptureWorker { + command_tx: mpsc::Sender, + join_handle: thread::JoinHandle<()>, + cancel: Arc, +} + +enum WorkerCommand { + SetActive(bool), + Stop, +} + +impl WindowsScreenCaptureInput { + /// Create a new Windows screen capture source. + #[must_use] + pub fn new(config: CaptureConfig) -> Self { + Self { + settings: Arc::new(SharedSettings { + config: Mutex::new(config), + generation: AtomicU64::new(0), + }), + running: false, + capture_active: false, + latest_snapshot: Arc::new(Mutex::new(None)), + worker: None, + } + } + + fn spawn_worker(&mut self) -> anyhow::Result<()> { + if self.worker.is_some() { + return Ok(()); + } + + let (command_tx, command_rx) = mpsc::channel(); + let settings = Arc::clone(&self.settings); + let latest_snapshot = Arc::clone(&self.latest_snapshot); + let cancel = Arc::new(AtomicBool::new(false)); + let worker_cancel = Arc::clone(&cancel); + + let join_handle = thread::Builder::new() + .name("hypercolor-screen-capture".to_owned()) + .spawn(move || { + run_worker(&settings, &latest_snapshot, &command_rx, &worker_cancel); + }) + .map_err(|error| anyhow!("failed to spawn screen capture worker: {error}"))?; + + self.worker = Some(CaptureWorker { + command_tx, + join_handle, + cancel, + }); + Ok(()) + } + + fn shutdown_worker(&mut self) { + let Some(worker) = self.worker.take() else { + return; + }; + + worker.cancel.store(true, Ordering::Release); + let _ = worker.command_tx.send(WorkerCommand::Stop); + if worker.join_handle.join().is_err() { + warn!("screen capture worker panicked during shutdown"); + } + } + + fn set_capture_active_state(&mut self, active: bool) -> anyhow::Result<()> { + if self.capture_active == active { + return Ok(()); + } + self.capture_active = active; + + if !self.running { + return Ok(()); + } + + if active { + self.spawn_worker()?; + } + + if let Some(worker) = self.worker.as_ref() { + let _ = worker.command_tx.send(WorkerCommand::SetActive(active)); + } + + if !active && let Ok(mut latest) = self.latest_snapshot.lock() { + *latest = None; + } + + Ok(()) + } + + /// Publish new settings and bump the generation the worker polls. + fn reconfigure(&mut self, config: CaptureConfig) { + if let Ok(mut current) = self.settings.config.lock() { + if *current == config { + return; + } + *current = config; + } + self.settings.generation.fetch_add(1, Ordering::Release); + } +} + +impl InputSource for WindowsScreenCaptureInput { + fn name(&self) -> &'static str { + "windows_screen_capture" + } + + fn start(&mut self) -> anyhow::Result<()> { + if self.running { + return Ok(()); + } + self.running = true; + + if self.capture_active { + self.spawn_worker()?; + if let Some(worker) = self.worker.as_ref() { + let _ = worker.command_tx.send(WorkerCommand::SetActive(true)); + } + } else { + debug!( + "Windows screen capture armed but idle until a screen-reactive effect requests capture" + ); + } + + Ok(()) + } + + fn stop(&mut self) { + self.running = false; + self.capture_active = false; + self.shutdown_worker(); + + if let Ok(mut latest) = self.latest_snapshot.lock() { + *latest = None; + } + } + + fn sample(&mut self) -> anyhow::Result { + if !self.running || !self.capture_active { + return Ok(InputData::None); + } + + let latest = self + .latest_snapshot + .lock() + .map_err(|_| anyhow!("windows screen capture snapshot mutex poisoned"))?; + + Ok(latest.clone().map_or(InputData::None, InputData::Screen)) + } + + fn is_running(&self) -> bool { + self.running + } + + fn is_screen_source(&self) -> bool { + true + } + + fn set_screen_capture_active(&mut self, active: bool) -> anyhow::Result<()> { + self.set_capture_active_state(active) + } + + fn reconfigure_screen_capture(&mut self, config: &CaptureConfig) -> anyhow::Result<()> { + self.reconfigure(config.clone()); + Ok(()) + } +} + +impl Drop for WindowsScreenCaptureInput { + fn drop(&mut self) { + self.shutdown_worker(); + } +} + +/// Worker loop: own the duplication session, analyze frames, publish results. +fn run_worker( + settings: &Arc, + latest_snapshot: &Arc>>, + command_rx: &mpsc::Receiver, + cancel: &Arc, +) { + let mut config = settings.snapshot(); + let mut generation = settings.generation.load(Ordering::Acquire); + let mut analyzer = ScreenCaptureInput::new(config.clone()); + let mut duplicator: Option = None; + let mut active = false; + let mut open_failure_logged = false; + + loop { + if cancel.load(Ordering::Acquire) { + break; + } + + match drain_commands(command_rx, &mut active) { + ControlFlow::Stop => break, + ControlFlow::Continue => {} + } + + if !active { + // Release the duplication interface so other applications can use + // it while no screen-reactive effect is running. + duplicator = None; + open_failure_logged = false; + match command_rx.recv_timeout(FRAME_WAIT) { + Ok(WorkerCommand::SetActive(next)) => active = next, + Ok(WorkerCommand::Stop) | Err(mpsc::RecvTimeoutError::Disconnected) => break, + Err(mpsc::RecvTimeoutError::Timeout) => {} + } + continue; + } + + let latest_generation = settings.generation.load(Ordering::Acquire); + if latest_generation != generation { + generation = latest_generation; + config = settings.snapshot(); + analyzer = ScreenCaptureInput::new(config.clone()); + if let Some(duplicator) = duplicator.as_mut() { + duplicator.set_max_width(CAPTURE_TARGET_WIDTH); + } + } + + let session = match duplicator.as_mut() { + Some(session) => session, + None => match DesktopDuplicator::new(config.monitor, CAPTURE_TARGET_WIDTH) { + Ok(session) => { + let (width, height) = session.native_extent(); + info!( + monitor = config.monitor, + width, height, "Windows screen capture online" + ); + open_failure_logged = false; + duplicator.insert(session) + } + Err(error) => { + if !open_failure_logged { + log_open_failure(&error); + open_failure_logged = true; + } + thread::sleep(REOPEN_BACKOFF); + continue; + } + }, + }; + + match session.next_frame(FRAME_WAIT) { + Ok(Some(frame)) => { + analyzer.push_frame(frame.rgba, frame.width, frame.height); + if let Ok(InputData::Screen(snapshot)) = analyzer.sample() + && let Ok(mut latest) = latest_snapshot.lock() + { + *latest = Some(snapshot); + } + } + // Static desktop or pointer-only update: nothing new to analyze. + Ok(None) => {} + Err(error) => { + warn!(%error, "Windows screen capture frame failed; reopening session"); + duplicator = None; + thread::sleep(REOPEN_BACKOFF); + } + } + } + + if let Ok(mut latest) = latest_snapshot.lock() { + *latest = None; + } + debug!("Windows screen capture worker stopped"); +} + +enum ControlFlow { + Continue, + Stop, +} + +/// Apply every queued command without blocking. +fn drain_commands(command_rx: &mpsc::Receiver, active: &mut bool) -> ControlFlow { + loop { + match command_rx.try_recv() { + Ok(WorkerCommand::SetActive(next)) => *active = next, + Ok(WorkerCommand::Stop) | Err(mpsc::TryRecvError::Disconnected) => { + return ControlFlow::Stop; + } + Err(mpsc::TryRecvError::Empty) => return ControlFlow::Continue, + } + } +} + +/// Log an open failure at a level that matches how actionable it is. +fn log_open_failure(error: &CaptureError) { + match error { + // Expected and self-healing: another RGB or capture tool holds the + // single per-output duplication interface until it exits. + CaptureError::AlreadyDuplicating => { + info!("screen capture is held by another application; retrying in the background"); + } + other => warn!(%other, "failed to open Windows screen capture"), + } +} diff --git a/crates/hypercolor-core/tests/screen_tests.rs b/crates/hypercolor-core/tests/screen_tests.rs index d33e5d1ab..4517d5a40 100644 --- a/crates/hypercolor-core/tests/screen_tests.rs +++ b/crates/hypercolor-core/tests/screen_tests.rs @@ -901,3 +901,37 @@ fn disabling_letterbox_live_clears_stale_bars() { "full uncropped grid should be reported once letterbox is off" ); } + +// ─── Monitor selection ─────────────────────────────────────────────────────── + +#[test] +fn monitor_source_accepts_prefixed_and_bare_indices() { + for (source, expected) in [ + ("monitor:0", 0), + ("monitor:1", 1), + ("monitor:11", 11), + ("display:2", 2), + ("3", 3), + (" monitor: 2 ", 2), + ] { + assert_eq!( + hypercolor_core::input::screen::monitor_index_from_source(source), + expected, + "source {source:?} should select monitor {expected}" + ); + } +} + +/// Anything unparseable means the primary display. The field is shared with +/// the Wayland backend, where the portal owns source selection and the value +/// stays "auto", so unknown strings must never be an error. +#[test] +fn unknown_monitor_sources_fall_back_to_the_primary_display() { + for source in ["auto", "", "pipewire", "monitor:", "monitor:-1", "HDMI-1"] { + assert_eq!( + hypercolor_core::input::screen::monitor_index_from_source(source), + 0, + "source {source:?} should fall back to the primary display" + ); + } +} diff --git a/crates/hypercolor-daemon/src/startup/services.rs b/crates/hypercolor-daemon/src/startup/services.rs index 26631da24..6c31086f8 100644 --- a/crates/hypercolor-daemon/src/startup/services.rs +++ b/crates/hypercolor-daemon/src/startup/services.rs @@ -28,10 +28,12 @@ use hypercolor_core::input::EvdevHostInput; #[cfg(not(target_os = "linux"))] use hypercolor_core::input::InteractionInput; use hypercolor_core::input::audio::AudioInput; -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "windows"))] use hypercolor_core::input::screen::CaptureConfig as ScreenCaptureConfig; #[cfg(target_os = "linux")] use hypercolor_core::input::screen::WaylandScreenCaptureInput; +#[cfg(target_os = "windows")] +use hypercolor_core::input::screen::WindowsScreenCaptureInput; use hypercolor_core::input::{InputManager, SensorPoller}; use hypercolor_core::scene::SceneManager; use hypercolor_core::spatial::SpatialEngine; @@ -606,6 +608,16 @@ pub(crate) fn build_input_manager( )); } + // Desktop Duplication needs no picker and no consent, so the Windows + // source is registered outright. It stays idle until an effect creates + // capture demand, exactly like the Wayland source. + #[cfg(target_os = "windows")] + if config.capture.enabled { + input_manager.add_source(Box::new(WindowsScreenCaptureInput::new( + screen_capture_config_from(&config.capture), + ))); + } + (input_manager, browser_input) } @@ -661,7 +673,7 @@ pub(crate) fn build_screen_capture_source( Box::new(WaylandScreenCaptureInput::new(capture_config).with_restore_token_sink(sink)) } -#[cfg(target_os = "linux")] +#[cfg(any(target_os = "linux", target_os = "windows"))] pub(crate) fn screen_capture_config_from( capture: &hypercolor_types::config::CaptureConfig, ) -> ScreenCaptureConfig { @@ -680,9 +692,9 @@ pub(crate) fn screen_capture_config_from( } .clamped(), restore_token: capture.restore_token.clone(), + monitor: hypercolor_core::input::screen::monitor_index_from_source(&capture.source), } } - fn audio_source_from_device(device: &str) -> AudioSourceType { let normalized = device.trim(); if normalized.eq_ignore_ascii_case("none") { diff --git a/crates/hypercolor-types/src/config.rs b/crates/hypercolor-types/src/config.rs index 63ac3e650..79f599863 100644 --- a/crates/hypercolor-types/src/config.rs +++ b/crates/hypercolor-types/src/config.rs @@ -88,6 +88,20 @@ mod defaults { pub fn capture_source() -> String { "auto".into() } + /// Whether screen capture is allowed without the user opting in. + /// + /// Windows Desktop Duplication needs no permission grant, shows no + /// source picker, and draws no capture indicator, so there is nothing + /// for a user to consent to and an ambient effect can simply work. The + /// XDG portal on Linux opens a picker the user must answer, and macOS + /// gates capture behind a TCC prompt; forcing either at daemon start + /// would be an ambush, so those stay opt-in. + /// + /// Enabling this only grants permission. Capture opens on demand and + /// stays closed until a screen-reactive effect actually asks for it. + pub fn capture_enabled() -> bool { + cfg!(target_os = "windows") + } pub fn capture_fps() -> u32 { 30 } @@ -645,7 +659,7 @@ impl Default for AudioConfig { /// picker; `restore_token` persists that choice across daemon restarts. #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CaptureConfig { - #[serde(default)] + #[serde(default = "defaults::capture_enabled")] pub enabled: bool, #[serde(default = "defaults::capture_source")] @@ -698,7 +712,7 @@ pub struct CaptureConfig { impl Default for CaptureConfig { fn default() -> Self { Self { - enabled: false, + enabled: defaults::capture_enabled(), source: defaults::capture_source(), capture_fps: defaults::capture_fps(), grid_cols: defaults::capture_grid_cols(), diff --git a/crates/hypercolor-types/tests/config_tests.rs b/crates/hypercolor-types/tests/config_tests.rs index db79a6b86..f7e144cde 100644 --- a/crates/hypercolor-types/tests/config_tests.rs +++ b/crates/hypercolor-types/tests/config_tests.rs @@ -85,10 +85,18 @@ fn audio_defaults_match_spec() { assert!((a.beat_sensitivity - 0.6).abs() < f32::EPSILON); } +/// Screen capture is allowed by default only where turning it on cannot +/// ambush the user. Windows Desktop Duplication has no permission prompt and +/// no picker; the XDG portal and macOS TCC both do, so those stay opt-in. +#[test] +fn capture_is_enabled_by_default_only_where_it_needs_no_consent() { + let c = CaptureConfig::default(); + assert_eq!(c.enabled, cfg!(target_os = "windows")); +} + #[test] fn capture_defaults_match_spec() { let c = CaptureConfig::default(); - assert!(!c.enabled); assert_eq!(c.source, "auto"); assert_eq!(c.capture_fps, 30); assert_eq!(c.grid_cols, 8); @@ -258,7 +266,7 @@ fn full_config_toml_roundtrip() { assert!(restored.web.enabled); assert_eq!(restored.mcp.base_path, "/mcp"); assert_eq!(restored.audio.fft_size, 1024); - assert!(!restored.capture.enabled); + assert_eq!(restored.capture.enabled, cfg!(target_os = "windows")); assert_eq!( restored.effect_engine.compositor_acceleration_mode, RenderAccelerationMode::Auto @@ -290,7 +298,7 @@ fn minimal_toml_fills_defaults() { assert!(config.web.enabled); assert_eq!(config.mcp.base_path, "/mcp"); assert_eq!(config.audio.device, "default"); - assert!(!config.capture.enabled); + assert_eq!(config.capture.enabled, cfg!(target_os = "windows")); assert_eq!( config.effect_engine.compositor_acceleration_mode, RenderAccelerationMode::Auto diff --git a/docs/specs/14-screen-capture.md b/docs/specs/14-screen-capture.md index 7ca5c945c..48a08c906 100644 --- a/docs/specs/14-screen-capture.md +++ b/docs/specs/14-screen-capture.md @@ -1591,54 +1591,70 @@ At every tier, the ambient lighting quality remains perceptually good. The human ## 10. Cross-Platform Strategy -Hypercolor is a Linux-first project, but screen capture should work on Windows (for development) and eventually macOS. +Hypercolor is a Linux-first project. Windows capture shipped as a first-class +backend; macOS is still unimplemented. -### Platform Matrix - -| Capability | Linux (Wayland) | Linux (X11) | Windows | macOS | -| --------------------- | ----------------------------- | --------------------- | -------------------- | --------------------------- | -| **Primary backend** | PipeWire + Portal | XShm | xcap (DXGI/WGC) | xcap (SCKit) | -| **DMA-BUF zero-copy** | Yes | No | No | No | -| **Streaming mode** | Yes (PipeWire) | No | No | No | -| **Permission model** | Portal dialog + restore token | None (open access) | App capability | Screen Recording permission | -| **Multi-monitor** | Portal multi-select | Root window spans all | Per-monitor via xcap | Per-monitor via xcap | -| **GPU downsample** | Yes (wgpu + DMA-BUF) | No (CPU only) | Possible (wgpu) | Possible (wgpu) | -| **Feature gate** | `screen-pipewire` | `screen-x11` | Default (xcap) | Default (xcap) | -| **Performance** | Excellent (<1% CPU) | Good (2-5% CPU) | Moderate (3-7% CPU) | Moderate (3-7% CPU) | - -### What Works on Windows (Development) +The `xcap` fallback this section originally specified was never built. Windows +uses a purpose-built DXGI Desktop Duplication backend instead, in the +`hypercolor-windows-capture` crate: it avoids pulling a cross-platform capture +dependency into a tree that already vendors its own thin Windows FFI crates, +and it lets the readback subsample during the copy rather than after it. -Windows developers can work on the full capture pipeline using the `xcap` backend: - -- `XcapCapture` works out of the box on Windows via DXGI desktop duplication or Windows Graphics Capture. -- `SectorGrid`, `RegionMapper`, `ColorProcessor`, `TemporalSmoother` are all pure Rust, platform-independent. -- `ScreenData` output type is identical across platforms. -- WLED DDP output works over the network (no USB dependencies). - -**What doesn't work on Windows:** +### Platform Matrix -- PipeWire backend (`#[cfg(target_os = "linux")]`). -- XShm backend (`#[cfg(target_os = "linux")]`). -- Portal restore tokens (Linux-specific D-Bus). -- DMA-BUF GPU zero-copy path. +| Capability | Linux (Wayland) | Linux (X11) | Windows | macOS | +| --------------------- | ----------------------------- | --------------------- | ------------------------------ | --------------- | +| **Primary backend** | PipeWire + Portal | XShm (unimplemented) | DXGI Desktop Duplication | Unimplemented | +| **DMA-BUF zero-copy** | Yes | No | No (staging readback) | n/a | +| **Streaming mode** | Yes (PipeWire) | No | Yes (duplication is a stream) | n/a | +| **Permission model** | Portal dialog + restore token | None (open access) | None required | Screen Recording| +| **Multi-monitor** | Portal multi-select | Root window spans all | `capture.source = "monitor:N"` | n/a | +| **Enabled by default**| No (portal picker is consent) | No | Yes (nothing to consent to) | No (TCC prompt) | + +### Windows + +`DesktopDuplicator` owns a D3D11 device, the duplication interface, and a +staging texture, and hands out RGBA frames. Everything downstream — sector +grid, letterbox detection, temporal smoothing — is the same +platform-independent code the Wayland path feeds. + +Three properties drove the design: + +- **No consent surface.** Windows has no portal handshake and no TCC prompt, + and Desktop Duplication draws no border and shows no picker. Nothing exists + for the user to approve, which is why `capture.enabled` defaults to `true` + on Windows alone. Permission to capture is not the same as capturing: the + session opens only when a screen-reactive effect creates demand. +- **Subsample during readback.** Frames are point-sampled by an integer stride + to roughly 1280px wide while being copied out of mapped staging memory, + which matches what PipeWire is asked to deliver on Linux. Sampling after a + full-resolution copy would move 33 MB per frame on a 4K desktop for a result + the sector grid cannot distinguish. +- **Recoverable by construction.** Mode changes, full-screen takeover, and the + UAC secure desktop all raise `DXGI_ERROR_ACCESS_LOST`; the session rebuilds + in place. Only one process may duplicate an output at a time, so the + interface is released whenever capture goes idle and reacquired on a backoff + when another application holds it. + +### macOS + +Unimplemented. ScreenCaptureKit is the intended backend, and unlike Windows it +does have a consent surface (the TCC Screen Recording prompt), so it will need +a demand-driven request flow rather than a default-on config. ### Conditional Compilation ```rust -// In the backend module: +// crates/hypercolor-core/src/input/screen/mod.rs #[cfg(target_os = "linux")] -mod pipewire; - -#[cfg(target_os = "linux")] -mod xshm; - -// Always available: -mod xcap_backend; - -// Re-export the auto-detection function (handles cfg internally) -pub use auto_detect::auto_detect_backend; +pub mod wayland; +#[cfg(target_os = "windows")] +pub mod windows; ``` +Registration lives in the daemon's `build_input_manager`, which adds the +matching source when `capture.enabled` is set. + ### Build Profiles ```toml From e3089bfc8a9dc6435679b9acc7e3ba38c3f08708 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 18:17:04 -0700 Subject: [PATCH 07/16] build(windows): fail fast when a foreign Rust shadows rustup A second Rust on PATH fails late and selectively. Cached crates check fine, so a build looks healthy right up until something needs a fresh link and dies inside a dependency's build script with "cannot execute 'ld'", or hits E0514 rustc-mismatch errors against artifacts the pinned toolchain left in target/. Neither symptom names the real cause, and rustup show happily reports the pinned MSVC toolchain while a different compiler is doing the work. Chocolatey's `rust` package is the usual culprit: it installs a GNU-host toolchain under C:\ProgramData\chocolatey\bin, which lives in the system PATH and therefore beats ~\.cargo\bin in the user PATH. Passing rustup's cargo.exe by absolute path does not help either, since cargo shells out to whichever rustc PATH resolves. Assert the host triple before entering the build, and report the resolved rustc path plus the removal command. Costs one process spawn against a class of failure that reads as a broken repo. Verified both directions: the wrapper builds clean with no PATH workaround, and a GNU-host rustc placed ahead on PATH aborts with the explanatory error instead of reaching cargo. Co-Authored-By: Nova (Claude Opus 5) --- scripts/cargo-cache-build.ps1 | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/scripts/cargo-cache-build.ps1 b/scripts/cargo-cache-build.ps1 index b10812159..6113813e1 100644 --- a/scripts/cargo-cache-build.ps1 +++ b/scripts/cargo-cache-build.ps1 @@ -34,6 +34,35 @@ function Enter-HypercolorVsDevShell { Enter-VsDevShell -VsInstallPath $vs -SkipAutomaticLocation -DevCmdArguments '-arch=x64 -host_arch=x64' | Out-Null } +function Assert-HypercolorRustToolchain { + # A second Rust on PATH is the worst kind of broken: it fails late and + # selectively. Cached crates check fine, so a build looks healthy right up + # until something needs a fresh link and dies inside a dependency's build + # script, or hits E0514 rustc-mismatch errors against artifacts the pinned + # toolchain left in target/. Chocolatey's `rust` package is the usual + # culprit on Windows: it installs a GNU-host toolchain under + # C:\ProgramData\chocolatey\bin, which is in the *system* PATH and + # therefore beats ~\.cargo\bin in the user PATH. Fail here, with the + # actual resolved paths, instead of 200 lines into a mingw linker error. + $rustc = Get-Command rustc -ErrorAction SilentlyContinue + if (-not $rustc) { + throw 'rustc not found on PATH. Install Rust with rustup: https://rustup.rs/' + } + + $version = & rustc -vV + $host_ = ($version | Select-String '^host: (.+)$').Matches.Groups[1].Value + + if ($host_ -ne 'x86_64-pc-windows-msvc') { + throw @" +Wrong Rust host toolchain: $host_ (expected x86_64-pc-windows-msvc) + rustc resolves to: $($rustc.Source) +Hypercolor builds against MSVC. A GNU-host Rust ahead of rustup on PATH is +almost always Chocolatey's `rust` package; remove it so rustup wins: + choco uninstall rust -y +"@ + } +} + function Initialize-HypercolorCargoCache { $cacheRoot = if ($env:HYPERCOLOR_CACHE_DIR) { $env:HYPERCOLOR_CACHE_DIR @@ -116,6 +145,7 @@ function Invoke-HypercolorCargoCommand { } Enter-HypercolorVsDevShell +Assert-HypercolorRustToolchain Initialize-HypercolorNativeTools Initialize-HypercolorCargoCache Invoke-HypercolorCargoCommand From 4c6c90063377fa50f665a596cd6290682a60c1f8 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 18:44:45 -0700 Subject: [PATCH 08/16] fix(input): collapse the evdev wheel arm to satisfy clippy CI has been red on main since 2026-07-21 with a single lint, clippy::collapsible_match, fired by the REL_WHEEL arm in the evdev reader. It failed three jobs because the same code compiles as both lib and lib test. The lint never showed up locally on Windows because evdev.rs is behind cfg(target_os = "linux"), so Windows contributors get a clean clippy run while Linux-only code rots. The inverse of this trap already bit us the other way around. Fold the guard into a match guard as clippy suggests. Behaviour is identical: a REL_WHEEL event on a hi-res device previously entered the arm and did nothing, and now falls through to the catch-all instead. Not verified locally: evdev.rs cannot compile on Windows, and cross- checking against x86_64-unknown-linux-gnu fails in aws-lc-sys, which needs a Linux C toolchain. CI is the verifier here. Co-Authored-By: Nova (Claude Opus 5) --- crates/hypercolor-core/src/input/evdev.rs | 30 +++++++++++------------ 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/crates/hypercolor-core/src/input/evdev.rs b/crates/hypercolor-core/src/input/evdev.rs index d00d40bbb..48ab7d151 100644 --- a/crates/hypercolor-core/src/input/evdev.rs +++ b/crates/hypercolor-core/src/input/evdev.rs @@ -664,23 +664,21 @@ fn fold_event( event_limit, ); } - RelativeAxisCode::REL_WHEEL => { - // Devices with hi-res wheels report both; keep only the - // hi-res stream to avoid double counting. - if !open.caps.hi_res_wheel { - push_event( - state, - TimedInputEvent { - event: InputEvent::MouseWheel { - source_id: open.source_id.clone(), - delta_hi_res: value.saturating_mul(120), - }, - at_ms, - seq: 0, + // Devices with hi-res wheels report both; keep only the + // hi-res stream to avoid double counting. + RelativeAxisCode::REL_WHEEL if !open.caps.hi_res_wheel => { + push_event( + state, + TimedInputEvent { + event: InputEvent::MouseWheel { + source_id: open.source_id.clone(), + delta_hi_res: value.saturating_mul(120), }, - event_limit, - ); - } + at_ms, + seq: 0, + }, + event_limit, + ); } _ => {} } From 1eb3d008ee2cf74f0f113bca501e23b0e5d122df Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 18:44:57 -0700 Subject: [PATCH 09/16] fix(windows): box-filter the capture readback instead of point sampling The readback picked one pixel per stride block. I justified that on the grounds that ambilight averages the result into a coarse sector grid anyway, which was the wrong call: the same buffer is published as canvas_downscale and consumed as an actual image by screen-reactive effects, which then downscale it a second time. Two successive point samplings of a 4K desktop shred thin text into aliased noise, and screen-reactive effects looked correspondingly bad. The Wayland path never hit this because PipeWire delivers an already-filtered frame, so the pipeline only ever downscaled once from something clean. Windows gets the raw desktop and has to do that filtering itself. Average every source pixel in the block instead. Reads scale from one pixel per block to all of them, which is the cost of the frame actually being correct; the loop stays row-major so the reads are sequential through mapped staging memory. Also adds two dump examples, which is how the above was diagnosed rather than guessed at. Averaged zone colors hide wrong row pitches, swapped channels, and off-by-one strides equally well, so writing the frame out is the only way to see what the pipeline really has: dump_frame for the raw backend output, dump_screen_canvas for the published surface that effects consume. Verified by comparing dumps before and after against the same desktop. Co-Authored-By: Nova (Claude Opus 5) --- .../examples/dump_screen_canvas.rs | 97 +++++++++++++++++++ crates/hypercolor-windows-capture/Cargo.toml | 3 + .../examples/dump_frame.rs | 79 +++++++++++++++ .../src/duplication.rs | 50 ++++++++-- 4 files changed, 222 insertions(+), 7 deletions(-) create mode 100644 crates/hypercolor-core/examples/dump_screen_canvas.rs create mode 100644 crates/hypercolor-windows-capture/examples/dump_frame.rs diff --git a/crates/hypercolor-core/examples/dump_screen_canvas.rs b/crates/hypercolor-core/examples/dump_screen_canvas.rs new file mode 100644 index 000000000..18afb1085 --- /dev/null +++ b/crates/hypercolor-core/examples/dump_screen_canvas.rs @@ -0,0 +1,97 @@ +//! Dump the screen-capture canvas that screen-reactive effects actually see. +//! +//! Runs the real pipeline — platform capture backend into `ScreenCaptureInput` +//! — and writes the published `canvas_downscale` surface to PNG. That surface +//! is what `screen_cast` samples and what ambilight's sector grid is computed +//! from, so it is the honest answer to "why does the screen look wrong". +//! +//! ```text +//! cargo run --example dump_screen_canvas -p hypercolor-core -- out.png +//! ``` + +fn main() { + #[cfg(not(target_os = "windows"))] + { + eprintln!("this example currently drives the Windows capture backend only"); + std::process::exit(1); + } + + #[cfg(target_os = "windows")] + { + use std::time::{Duration, Instant}; + + use hypercolor_core::input::screen::{CaptureConfig, ScreenCaptureInput}; + use hypercolor_core::input::{InputData, InputSource}; + use hypercolor_windows_capture::DesktopDuplicator; + + let path = std::env::args() + .nth(1) + .unwrap_or_else(|| "screen-canvas.png".to_owned()); + + let mut duplicator = match DesktopDuplicator::new(0, 1280) { + Ok(duplicator) => duplicator, + Err(error) => { + eprintln!("failed to open capture: {error}"); + std::process::exit(1); + } + }; + let mut analyzer = ScreenCaptureInput::new(CaptureConfig::default()); + if let Err(error) = analyzer.start() { + eprintln!("failed to start analyzer: {error}"); + std::process::exit(1); + } + + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + match duplicator.next_frame(Duration::from_millis(200)) { + Ok(Some(frame)) => { + println!("capture backend produced {}x{}", frame.width, frame.height); + analyzer.push_frame(frame.rgba, frame.width, frame.height); + } + Ok(None) => continue, + Err(error) => { + eprintln!("capture failed: {error}"); + std::process::exit(1); + } + } + + let Ok(InputData::Screen(data)) = analyzer.sample() else { + continue; + }; + let Some(surface) = data.canvas_downscale.as_ref() else { + println!("no canvas_downscale published yet"); + continue; + }; + + let descriptor = surface.descriptor(); + let canvas = hypercolor_core::types::canvas::Canvas::from_published_surface(surface); + let bytes = canvas.as_rgba_bytes().to_vec(); + println!( + "canvas_downscale: {}x{} ({} bytes), source was {}x{}", + descriptor.width, + descriptor.height, + bytes.len(), + data.source_width, + data.source_height + ); + + let Some(buffer) = + image::RgbaImage::from_raw(descriptor.width, descriptor.height, bytes) + else { + eprintln!("canvas bytes did not match the descriptor"); + std::process::exit(1); + }; + match buffer.save(&path) { + Ok(()) => println!("wrote {path}"), + Err(error) => { + eprintln!("failed to write {path}: {error}"); + std::process::exit(1); + } + } + return; + } + + eprintln!("no frame arrived within the budget"); + std::process::exit(1); + } +} diff --git a/crates/hypercolor-windows-capture/Cargo.toml b/crates/hypercolor-windows-capture/Cargo.toml index 17b9e0c5b..35f09a3b8 100644 --- a/crates/hypercolor-windows-capture/Cargo.toml +++ b/crates/hypercolor-windows-capture/Cargo.toml @@ -19,6 +19,9 @@ unwrap_used = "deny" thiserror = { workspace = true } tracing = { workspace = true } +[dev-dependencies] +image = { workspace = true } + [target.'cfg(target_os = "windows")'.dependencies] windows = { version = "0.62", default-features = false, features = [ "Win32_Foundation", diff --git a/crates/hypercolor-windows-capture/examples/dump_frame.rs b/crates/hypercolor-windows-capture/examples/dump_frame.rs new file mode 100644 index 000000000..feedf725e --- /dev/null +++ b/crates/hypercolor-windows-capture/examples/dump_frame.rs @@ -0,0 +1,79 @@ +//! Dump a captured desktop frame to PNG, for eyeballing the readback. +//! +//! Ambient lighting reduces frames to a handful of averaged colors, which +//! hides exactly the bugs that matter most here: a wrong row pitch, swapped +//! channels, or an off-by-one stride all still produce plausible-looking +//! zone colors. Writing the frame out makes those instantly obvious. +//! +//! ```text +//! cargo run --example dump_frame -p hypercolor-windows-capture -- out.png [monitor] [max_width] +//! ``` + +fn main() { + #[cfg(not(target_os = "windows"))] + { + eprintln!("desktop capture is Windows-only"); + std::process::exit(1); + } + + #[cfg(target_os = "windows")] + { + use std::time::{Duration, Instant}; + + use hypercolor_windows_capture::DesktopDuplicator; + + let mut args = std::env::args().skip(1); + let path = args.next().unwrap_or_else(|| "capture.png".to_owned()); + let monitor: usize = args.next().and_then(|v| v.parse().ok()).unwrap_or(0); + let max_width: u32 = args.next().and_then(|v| v.parse().ok()).unwrap_or(1280); + + let mut duplicator = match DesktopDuplicator::new(monitor, max_width) { + Ok(duplicator) => duplicator, + Err(error) => { + eprintln!("failed to open capture: {error}"); + std::process::exit(1); + } + }; + + let (native_width, native_height) = duplicator.native_extent(); + println!("native desktop: {native_width}x{native_height}"); + + // The desktop only produces frames when something changes, so keep + // asking until one lands rather than giving up on the first timeout. + let deadline = Instant::now() + Duration::from_secs(10); + while Instant::now() < deadline { + match duplicator.next_frame(Duration::from_millis(200)) { + Ok(Some(frame)) => { + println!( + "captured {}x{} ({} bytes)", + frame.width, + frame.height, + frame.rgba.len() + ); + let Some(buffer) = + image::RgbaImage::from_raw(frame.width, frame.height, frame.rgba.to_vec()) + else { + eprintln!("frame buffer did not match its declared dimensions"); + std::process::exit(1); + }; + match buffer.save(&path) { + Ok(()) => println!("wrote {path}"), + Err(error) => { + eprintln!("failed to write {path}: {error}"); + std::process::exit(1); + } + } + return; + } + Ok(None) => {} + Err(error) => { + eprintln!("capture failed: {error}"); + std::process::exit(1); + } + } + } + + eprintln!("no frame arrived within the budget; is the desktop completely static?"); + std::process::exit(1); + } +} diff --git a/crates/hypercolor-windows-capture/src/duplication.rs b/crates/hypercolor-windows-capture/src/duplication.rs index 8e2499f02..ef2e62d1c 100644 --- a/crates/hypercolor-windows-capture/src/duplication.rs +++ b/crates/hypercolor-windows-capture/src/duplication.rs @@ -238,7 +238,16 @@ impl DesktopDuplicator { Ok(extent) } - /// Subsample BGRA staging rows into the packed RGBA output buffer. + /// Box-filter BGRA staging rows into the packed RGBA output buffer. + /// + /// Every source pixel in each stride x stride block is averaged rather + /// than one being picked. Point sampling is tempting here — the ambilight + /// sector grid averages the result anyway — but the same buffer is + /// published as `canvas_downscale` and consumed as an actual image by + /// screen-reactive effects, then downscaled a second time. Two successive + /// point samplings of a 4K desktop shred thin text into aliased noise. The + /// Wayland path never had this problem because PipeWire hands over an + /// already-filtered frame. fn copy_mapped_rows( &mut self, mapped: &D3D11_MAPPED_SUBRESOURCE, @@ -260,16 +269,43 @@ impl DesktopDuplicator { // bytes for this subresource, and it stays valid until Unmap. let source = unsafe { std::slice::from_raw_parts(mapped.pData.cast::(), source_len) }; + let stride = stride as usize; + let width = width as usize; + let height = height as usize; + for out_y in 0..out_height as usize { - let src_row_start = (out_y * stride as usize) * row_pitch; let dst_row_start = out_y * out_width as usize * BYTES_PER_PIXEL; + // Blocks on the right and bottom edges are clipped when the + // desktop does not divide evenly by the stride. + let src_y0 = out_y * stride; + let src_y1 = (src_y0 + stride).min(height); + for out_x in 0..out_width as usize { - let src = src_row_start + (out_x * stride as usize) * BYTES_PER_PIXEL; + let src_x0 = out_x * stride; + let src_x1 = (src_x0 + stride).min(width); + + let mut blue = 0_u32; + let mut green = 0_u32; + let mut red = 0_u32; + let mut samples = 0_u32; + + for src_y in src_y0..src_y1 { + let row = src_y * row_pitch; + for src_x in src_x0..src_x1 { + let src = row + src_x * BYTES_PER_PIXEL; + // Desktop Duplication hands back BGRA. + blue += u32::from(source[src]); + green += u32::from(source[src + 1]); + red += u32::from(source[src + 2]); + samples += 1; + } + } + + let samples = samples.max(1); let dst = dst_row_start + out_x * BYTES_PER_PIXEL; - // Desktop Duplication hands back BGRA; the pipeline wants RGBA. - self.rgba[dst] = source[src + 2]; - self.rgba[dst + 1] = source[src + 1]; - self.rgba[dst + 2] = source[src]; + self.rgba[dst] = (red / samples) as u8; + self.rgba[dst + 1] = (green / samples) as u8; + self.rgba[dst + 2] = (blue / samples) as u8; self.rgba[dst + 3] = 0xFF; } } From 409731d276f2190ca63eb4c5a0a38c918a938fc5 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 18:58:18 -0700 Subject: [PATCH 10/16] fix(capture): stop letterbox detection from eating the whole frame Ambilight rendered garbage from a real desktop because the letterbox detector reported bars from all four edges at once. Measured against a live 3840x2160 capture, an 8x6 grid came back as top=6 bottom=6 left=8 right=8: every row and every column classified as a black bar, so cropping removed the entire picture, and because the verdict moves with ordinary content changes it strobed frame to frame. The cause is that pixel_luminance is linear while the threshold reads like an sRGB one. Linear luminance punishes dark content brutally: sRGB 30/255 is 0.013 linear, well under the 0.02 default. A dark-themed desktop is therefore "black" from every edge, which is exactly the content ambient lighting is most often pointed at. Two changes. Detection now discards an axis whose bars would swallow the whole grid, because real letterboxing always leaves something in the middle and a uniformly dark frame is dark rather than letterboxed. And letterbox defaults to off, since ambient lighting almost always mirrors a desktop rather than a letterboxed film; the guard alone still left two of eight columns cropped on the same desktop, quietly discarding a quarter of the screen. An existing test asserted the old degenerate path, where an all-black frame produced bars that consumed everything. That premise is gone, so it now asserts no bars are reported; crop_letterbox's own refusal to leave nothing behind is still covered, driven by explicit bars rather than by detection. Verified on the same live desktop: bars went from [6,6,8,8] to [0,0,0,2], and to none with the default off. Co-Authored-By: Nova (Claude Opus 5) --- .../examples/dump_screen_canvas.rs | 5 ++ .../src/input/screen/sector.rs | 26 +++++- crates/hypercolor-core/tests/screen_tests.rs | 87 +++++++++++++++++-- crates/hypercolor-types/src/config.rs | 9 +- crates/hypercolor-types/tests/config_tests.rs | 4 +- 5 files changed, 118 insertions(+), 13 deletions(-) diff --git a/crates/hypercolor-core/examples/dump_screen_canvas.rs b/crates/hypercolor-core/examples/dump_screen_canvas.rs index 18afb1085..963ebd87b 100644 --- a/crates/hypercolor-core/examples/dump_screen_canvas.rs +++ b/crates/hypercolor-core/examples/dump_screen_canvas.rs @@ -74,6 +74,11 @@ fn main() { data.source_width, data.source_height ); + println!( + "letterbox bars (grid units, t/b/l/r): {:?} of a {}x{} grid", + data.letterbox, data.grid_width, data.grid_height + ); + println!("zone colors published: {}", data.zone_colors.len()); let Some(buffer) = image::RgbaImage::from_raw(descriptor.width, descriptor.height, bytes) diff --git a/crates/hypercolor-core/src/input/screen/sector.rs b/crates/hypercolor-core/src/input/screen/sector.rs index 32ee60511..0856eec0e 100644 --- a/crates/hypercolor-core/src/input/screen/sector.rs +++ b/crates/hypercolor-core/src/input/screen/sector.rs @@ -154,12 +154,30 @@ impl SectorGrid { /// /// Returns `(top_rows, bottom_rows, left_cols, right_cols)` — the number /// of consecutive black rows/columns from each edge. + /// + /// An axis whose bars would swallow the whole grid reports no bars at + /// all. `pixel_luminance` is *linear*, where dark content sits far lower + /// than its sRGB value suggests — sRGB 30/255 is only 0.013 linear — so a + /// dark desktop or a night-time scene can read as black from every edge + /// at once. Cropping on that removes the entire picture, and because the + /// verdict flips with ordinary content changes it strobes frame to frame. + /// Real letterboxing always leaves something in the middle. #[must_use] pub fn detect_letterbox(&self, black_threshold: f32) -> LetterboxBars { - let top = self.count_black_rows_from_top(black_threshold); - let bottom = self.count_black_rows_from_bottom(black_threshold); - let left = self.count_black_cols_from_left(black_threshold); - let right = self.count_black_cols_from_right(black_threshold); + let mut top = self.count_black_rows_from_top(black_threshold); + let mut bottom = self.count_black_rows_from_bottom(black_threshold); + let mut left = self.count_black_cols_from_left(black_threshold); + let mut right = self.count_black_cols_from_right(black_threshold); + + if top.saturating_add(bottom) >= self.rows { + top = 0; + bottom = 0; + } + if left.saturating_add(right) >= self.cols { + left = 0; + right = 0; + } + LetterboxBars { top, bottom, diff --git a/crates/hypercolor-core/tests/screen_tests.rs b/crates/hypercolor-core/tests/screen_tests.rs index 4517d5a40..da1c7f211 100644 --- a/crates/hypercolor-core/tests/screen_tests.rs +++ b/crates/hypercolor-core/tests/screen_tests.rs @@ -299,18 +299,38 @@ fn letterbox_crop_removes_black_bars() { } } +/// An all-black frame is not letterboxed, it is black. Detection used to +/// report bars from every edge here, which cropped the picture out of +/// existence and strobed as content changed; it now reports none. #[test] -fn letterbox_all_black_returns_no_bars_on_crop() { - // Entire frame is black — degenerate case. +fn letterbox_all_black_frame_reports_no_bars() { let frame = solid_frame(80, 60, 0, 0, 0); let grid = SectorGrid::compute(&frame, 80, 60, 4, 3); let bars = grid.detect_letterbox(0.05); - // All rows/cols are black, so bars consume the entire grid. - let cropped = grid.crop_letterbox(&bars); + assert!(!bars.has_bars(), "an all-black frame has no letterbox bars"); assert!( - cropped.is_none(), - "all-black frame should yield no content after crop" + grid.crop_letterbox(&bars).is_some(), + "with no bars the grid survives cropping intact" + ); +} + +/// `crop_letterbox` still has to refuse bars that would leave nothing behind, +/// since callers can hand it any bars they like. +#[test] +fn letterbox_crop_refuses_bars_that_consume_the_grid() { + let frame = solid_frame(80, 60, 10, 10, 10); + let grid = SectorGrid::compute(&frame, 80, 60, 4, 3); + + let devouring = LetterboxBars { + top: 2, + bottom: 1, + left: 0, + right: 0, + }; + assert!( + grid.crop_letterbox(&devouring).is_none(), + "bars covering every row must yield no croppable grid" ); } @@ -935,3 +955,58 @@ fn unknown_monitor_sources_fall_back_to_the_primary_display() { ); } } + +// ─── Letterbox degeneracy ──────────────────────────────────────────────────── + +/// A uniformly dark frame is not letterboxed, it is just dark. Reporting bars +/// from every edge at once made `crop_letterbox` throw the whole picture away, +/// and because the verdict flips with ordinary content changes it strobed. +/// Linear luminance makes this easy to hit: sRGB 30/255 is only 0.013 linear, +/// so a dark-themed desktop reads as black from every edge. +#[test] +fn uniformly_dark_frames_report_no_letterbox_bars() { + let dark = vec![12_u8; 64 * 48 * 4]; + let grid = SectorGrid::compute(&dark, 64, 48, 8, 6); + let bars = grid.detect_letterbox(0.02); + + assert_eq!( + (bars.top, bars.bottom, bars.left, bars.right), + (0, 0, 0, 0), + "an all-dark frame must not be treated as entirely letterbox" + ); + assert!(!bars.has_bars()); + assert!( + grid.crop_letterbox(&bars).is_some(), + "a non-degenerate crop must still be possible" + ); +} + +/// The guard must not disarm real letterboxing, which always leaves content +/// between the bars. +#[test] +fn genuine_letterbox_bars_are_still_detected() { + let width = 64_u32; + let height = 48_u32; + let mut frame = vec![0_u8; (width * height * 4) as usize]; + // Bright band across the middle third, black bars above and below. + for y in 16..32 { + for x in 0..width { + let px = ((y * width + x) * 4) as usize; + frame[px] = 220; + frame[px + 1] = 220; + frame[px + 2] = 220; + frame[px + 3] = 255; + } + } + + let grid = SectorGrid::compute(&frame, width, height, 8, 6); + let bars = grid.detect_letterbox(0.02); + + assert!(bars.top > 0, "top bar should be detected"); + assert!(bars.bottom > 0, "bottom bar should be detected"); + assert!( + bars.top + bars.bottom < 6, + "real bars must leave content behind" + ); + assert_eq!((bars.left, bars.right), (0, 0), "no pillarboxing here"); +} diff --git a/crates/hypercolor-types/src/config.rs b/crates/hypercolor-types/src/config.rs index 79f599863..58b96d856 100644 --- a/crates/hypercolor-types/src/config.rs +++ b/crates/hypercolor-types/src/config.rs @@ -685,7 +685,12 @@ pub struct CaptureConfig { pub scene_cut_threshold: f32, /// Auto-detect and crop black letterbox/pillarbox bars. - #[serde(default = "defaults::bool_true")] + /// + /// Off by default: ambient lighting almost always mirrors a desktop, not + /// a letterboxed film, and dark desktop content trips the detector into + /// cropping real picture away. Turn it on when mirroring video that + /// genuinely has bars. + #[serde(default)] pub letterbox: bool, /// Luminance threshold for letterbox detection (0.0 - 1.0). @@ -719,7 +724,7 @@ impl Default for CaptureConfig { grid_rows: defaults::capture_grid_rows(), smoothing: defaults::capture_smoothing(), scene_cut_threshold: defaults::capture_scene_cut_threshold(), - letterbox: defaults::bool_true(), + letterbox: false, letterbox_threshold: defaults::capture_letterbox_threshold(), saturation: defaults::unit_scale(), brightness: defaults::unit_scale(), diff --git a/crates/hypercolor-types/tests/config_tests.rs b/crates/hypercolor-types/tests/config_tests.rs index f7e144cde..94ee10d52 100644 --- a/crates/hypercolor-types/tests/config_tests.rs +++ b/crates/hypercolor-types/tests/config_tests.rs @@ -103,7 +103,9 @@ fn capture_defaults_match_spec() { assert_eq!(c.grid_rows, 6); assert!((c.smoothing - 0.3).abs() < f32::EPSILON); assert!((c.scene_cut_threshold - 100.0).abs() < f32::EPSILON); - assert!(c.letterbox); + // Off by default: desktops are not letterboxed, and dark desktop content + // trips the detector into cropping real picture away. + assert!(!c.letterbox); assert!((c.letterbox_threshold - 0.02).abs() < f32::EPSILON); assert!((c.saturation - 1.0).abs() < f32::EPSILON); assert!((c.brightness - 1.0).abs() < f32::EPSILON); From 6a75ebddaa9fc5bdae6142de367389df643087ac Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 18:59:13 -0700 Subject: [PATCH 11/16] build: refresh Cargo.lock for the capture crate dev-dependency CI runs cargo with --locked, so the image dev-dependency added to hypercolor-windows-capture for the frame-dump examples failed every Rust job in seconds with "cannot update the lock file". Local builds hid it by updating the lock in place and leaving it uncommitted. Co-Authored-By: Nova (Claude Opus 5) --- Cargo.lock | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.lock b/Cargo.lock index 722dd9c71..94c188369 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5125,6 +5125,7 @@ dependencies = [ name = "hypercolor-windows-capture" version = "0.2.1" dependencies = [ + "image", "thiserror 2.0.18", "tracing", "windows 0.62.2", From 3bacb0f1df0f0373edea24a588e42028ceff8053 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 19:03:00 -0700 Subject: [PATCH 12/16] fix(ui): route bundled presets to the zone instead of a dead endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Selecting a bundled preset never stuck. The dropdown snapped back to whatever was already applied, so Screen Cast in particular was pinned to whichever preset its layer happened to be created with — Center Zoom, whose 40% viewport crop is why the capture looked like a blurry mess. Saved presets and bundled presets took different routes. Saved presets go through apply_preset, which targets the focused zone. Bundled presets went through update_controls, which PATCHes the legacy global /effects/current/controls. There is no legacy "current effect" in a zone scene, so that 404s, and the error branch restores the previous selection — the snap-back was the failure handler doing its job on a request that could never succeed. Send them the same place a manual control edit already goes: the zone's synthetic legacy layer, whose group and layer ids are both the zone id. The legacy endpoint stays as the fallback for when no zone scene is active, which is the only case it was ever right for. Verified: cargo check and clippy -D warnings clean for the wasm target. Not exercised in a browser — the failing request is gone by construction, but the applied-preset round trip still wants a manual pass. Co-Authored-By: Nova (Claude Opus 5) --- .../src/components/preset_panel.rs | 23 ++++++++++++++++++- crates/hypercolor-ui/src/zones.rs | 14 +++++++++++ 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/crates/hypercolor-ui/src/components/preset_panel.rs b/crates/hypercolor-ui/src/components/preset_panel.rs index 2cc156f34..75d442e39 100644 --- a/crates/hypercolor-ui/src/components/preset_panel.rs +++ b/crates/hypercolor-ui/src/components/preset_panel.rs @@ -246,8 +246,29 @@ pub fn PresetToolbar( set_selected_id.set(Some(val)); set_mode.set(ToolbarMode::Idle); let on_applied = on_preset_applied; + // Bundled presets write the same controls a manual edit + // does, so they must take the same route: the zone's + // synthetic legacy layer, keyed by the zone id. The old + // global endpoint 404s whenever no legacy "current + // effect" exists, which is always in a zone scene — and + // the failure branch below then snapped the dropdown + // back, so bundled presets looked like they refused to + // stick. + let scoped = zones_ctx.scene_scoped_target(); leptos::task::spawn_local(async move { - match api::update_controls(&controls_json).await { + let result = match scoped { + Some((scene_id, zone_id)) => api::patch_layer_controls( + &scene_id, + &zone_id, + &zone_id, + &controls_json, + None, + ) + .await + .map(|_| ()), + None => api::update_controls(&controls_json).await, + }; + match result { Ok(()) => on_applied.run(()), Err(error) => { set_selected_id.set(previous_selection); diff --git a/crates/hypercolor-ui/src/zones.rs b/crates/hypercolor-ui/src/zones.rs index 1b68d2428..d9244f831 100644 --- a/crates/hypercolor-ui/src/zones.rs +++ b/crates/hypercolor-ui/src/zones.rs @@ -68,6 +68,20 @@ impl ZonesContext { }) } + /// Scene id plus the zone writes target, for control PATCHes. + /// + /// A zone's controls live on its synthetic legacy layer, whose group and + /// layer ids are both the zone id, so this pair is everything + /// `patch_layer_controls` needs. `None` means no zone scene is active and + /// the caller should fall back to the legacy global endpoint. + pub fn scene_scoped_target(&self) -> Option<(String, String)> { + let scene_id = self + .active_scene + .with_untracked(|scene| scene.as_ref().map(|scene| scene.id.clone()))?; + let zone = self.target_zone()?; + Some((scene_id, zone.id)) + } + /// Untracked focused-zone id, validated against the current scene. /// `None` when unset, stale, or pointing at a Screen. pub fn focused_zone_id_untracked(&self) -> Option { From 394f1914360994c5e5b88103d2b3d316f4c68760 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 19:06:07 -0700 Subject: [PATCH 13/16] fix(capture): keep the screen downscale at the source aspect ratio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The published capture surface targeted the canvas bounds directly, so a 16:9 desktop was resampled into 640x480 — 2x horizontally but 1.5x vertically, a 1.33x vertical stretch. No downstream fit mode can undo that, because the distortion is baked into the pixels before any effect sees them; circles reached screen-mirroring effects as ellipses. Fit within the canvas bounds instead of filling them. A 4K or 1080p desktop now publishes 640x360, a square source 480x480, and portrait or ultrawide sources scale on whichever axis binds first. An existing test asserted a 40x40 frame produced a 640x480 surface, which was the squash itself written down as an expectation; a square source now yields a square surface. Also aligns the core CaptureConfig letterbox default with the one in config::CaptureConfig, which was flipped off in the previous commit. Leaving them disagreeing meant a caller building the core config directly still got the old cropping behaviour. Verified against a live 3840x2160 desktop: canvas_downscale went from 640x480 to 640x360. Co-Authored-By: Nova (Claude Opus 5) --- .../hypercolor-core/src/input/screen/mod.rs | 45 ++++++++++++-- crates/hypercolor-core/tests/screen_tests.rs | 61 ++++++++++++++++++- 2 files changed, 101 insertions(+), 5 deletions(-) diff --git a/crates/hypercolor-core/src/input/screen/mod.rs b/crates/hypercolor-core/src/input/screen/mod.rs index a3f59a2c1..f4b8f6795 100644 --- a/crates/hypercolor-core/src/input/screen/mod.rs +++ b/crates/hypercolor-core/src/input/screen/mod.rs @@ -64,7 +64,10 @@ pub struct CaptureConfig { /// Luminance threshold for letterbox detection (0.0 - 1.0). Default: 0.02. pub letterbox_threshold: f32, - /// Whether letterbox detection is enabled. Default: true. + /// Whether letterbox detection is enabled. Default: false, matching + /// `config::CaptureConfig` — ambient lighting mirrors desktops far more + /// often than letterboxed film, and dark desktop content trips the + /// detector into cropping real picture away. pub letterbox_enabled: bool, /// Color tuning applied to zone colors after smoothing. @@ -88,7 +91,7 @@ impl Default for CaptureConfig { smoothing_alpha: 0.3, scene_cut_threshold: 100.0, letterbox_threshold: 0.02, - letterbox_enabled: true, + letterbox_enabled: false, tuning: ColorTuning::default(), restore_token: None, monitor: 0, @@ -96,6 +99,38 @@ impl Default for CaptureConfig { } } +/// Largest size within `max_width` x `max_height` that keeps `width` x +/// `height`'s aspect ratio. +/// +/// The screen downscale used to target the canvas bounds directly, which +/// squashed a 16:9 desktop into 4:3 — a 1.33x vertical stretch that no +/// downstream fit mode can undo, because the distortion is already baked +/// into the published surface. Circles came out as ellipses on every +/// screen-mirroring effect. +#[must_use] +pub fn fit_within(width: u32, height: u32, max_width: u32, max_height: u32) -> (u32, u32) { + if width == 0 || height == 0 || max_width == 0 || max_height == 0 { + return (max_width.max(1), max_height.max(1)); + } + + // Integer comparison of width/height against max_width/max_height, + // cross-multiplied to avoid floating point entirely. + let source_is_wider = + u64::from(width) * u64::from(max_height) >= u64::from(height) * u64::from(max_width); + + if source_is_wider { + let scaled_height = (u64::from(height) * u64::from(max_width) / u64::from(width)) + .try_into() + .unwrap_or(max_height); + (max_width, u32::max(scaled_height, 1)) + } else { + let scaled_width = (u64::from(width) * u64::from(max_height) / u64::from(height)) + .try_into() + .unwrap_or(max_width); + (u32::max(scaled_width, 1), max_height) + } +} + /// Parse a configured capture source into a zero-based monitor index. /// /// `capture.source` is a free-form string shared across backends. The XDG @@ -200,12 +235,14 @@ impl ScreenCaptureInput { pub fn push_frame(&mut self, frame: &[u8], width: u32, height: u32) { self.frame_width = width; self.frame_height = height; + let (downscale_width, downscale_height) = + fit_within(width, height, DEFAULT_CANVAS_WIDTH, DEFAULT_CANVAS_HEIGHT); self.latest_canvas_downscale = downscale_frame( frame, width, height, - DEFAULT_CANVAS_WIDTH, - DEFAULT_CANVAS_HEIGHT, + downscale_width, + downscale_height, &mut self.downscale_pool, ); diff --git a/crates/hypercolor-core/tests/screen_tests.rs b/crates/hypercolor-core/tests/screen_tests.rs index da1c7f211..c35b8ce77 100644 --- a/crates/hypercolor-core/tests/screen_tests.rs +++ b/crates/hypercolor-core/tests/screen_tests.rs @@ -552,8 +552,11 @@ fn screen_capture_input_produces_screen_data() { .canvas_downscale .as_ref() .expect("screen data should include downscaled canvas"); - assert_eq!(downscale.width(), DEFAULT_CANVAS_WIDTH); + // A square source publishes a square surface: the downscale fits + // within the canvas bounds rather than stretching to fill them. + assert_eq!(downscale.width(), DEFAULT_CANVAS_HEIGHT); assert_eq!(downscale.height(), DEFAULT_CANVAS_HEIGHT); + assert!(downscale.width() <= DEFAULT_CANVAS_WIDTH); assert_eq!( downscale.get_pixel(0, 0), hypercolor_core::types::canvas::Rgba::new(200, 100, 50, 255) @@ -1010,3 +1013,59 @@ fn genuine_letterbox_bars_are_still_detected() { ); assert_eq!((bars.left, bars.right), (0, 0), "no pillarboxing here"); } + +// ─── Aspect preservation ───────────────────────────────────────────────────── + +/// The published surface must carry the source's aspect ratio. Targeting the +/// canvas bounds directly squashed 16:9 into 4:3, and no downstream fit mode +/// can undo distortion already baked into the pixels. +#[test] +fn downscale_target_preserves_source_aspect() { + use hypercolor_core::input::screen::fit_within; + + // 16:9 into a 4:3 box fits by width. + assert_eq!(fit_within(1920, 1080, 640, 480), (640, 360)); + assert_eq!(fit_within(3840, 2160, 640, 480), (640, 360)); + // 4:3 source fills the box exactly. + assert_eq!(fit_within(1600, 1200, 640, 480), (640, 480)); + // A portrait source fits by height instead. + assert_eq!(fit_within(1080, 1920, 640, 480), (270, 480)); + // Ultrawide. + assert_eq!(fit_within(3440, 1440, 640, 480), (640, 267)); +} + +#[test] +fn downscale_target_never_returns_a_zero_dimension() { + use hypercolor_core::input::screen::fit_within; + + assert_eq!(fit_within(0, 0, 640, 480), (640, 480)); + assert_eq!(fit_within(1920, 1080, 0, 0), (1, 1)); + // Extreme ratios still round up to a usable pixel. + let (w, h) = fit_within(100_000, 1, 640, 480); + assert!(w >= 1 && h >= 1, "got {w}x{h}"); +} + +#[test] +fn pushed_frames_publish_an_aspect_correct_surface() { + let mut input = ScreenCaptureInput::new(CaptureConfig::default()); + input.start().expect("start should succeed"); + + // A 16:9 frame must not come back as 4:3. + let frame = solid_frame(320, 180, 90, 40, 200); + input.push_frame(&frame, 320, 180); + + let InputData::Screen(data) = input.sample().expect("sample succeeds") else { + panic!("expected screen data"); + }; + let surface = data + .canvas_downscale + .as_ref() + .expect("a downscaled surface should be published"); + let descriptor = surface.descriptor(); + + assert_eq!( + (descriptor.width, descriptor.height), + (640, 360), + "16:9 input must publish a 16:9 surface" + ); +} From ca1cda518d1d710c7b73333648b0b3060d5dc532 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 19:29:05 -0700 Subject: [PATCH 14/16] feat(capture): enumerate monitors and serve them over the API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Settings "Source" dropdown offered "Auto" and "PipeWire" — backend implementation names, one of them Linux-only, shown to Windows users as the only way to aim screen capture. The real selector, capture.source = "monitor:N", was reachable only by hand-editing config, with nothing telling the user what N meant on their machine. The capture crate now describes every attached output — capture index, OS device name, desktop extent, and whether it anchors the virtual desktop origin — and GET /api/v1/capture/monitors serves the list with a ready-to-store capture.source value per entry. An empty list is the capability signal: portal platforms return nothing, and the UI keeps the portal picker there instead of a dropdown. Index order matches DesktopDuplicator's adapter-then-output enumeration, so what the picker shows is exactly what a duplicator opens. Verified live against real hardware: two outputs reported with correct geometry, including a portrait 1440x2560 secondary, and the listing test asserts the count matches, extents are nonzero, and a primary exists. Co-Authored-By: Nova (Claude Opus 5) --- .../hypercolor-core/src/input/screen/mod.rs | 42 +++++++++++++++++++ crates/hypercolor-daemon/src/api/capture.rs | 38 +++++++++++++++++ crates/hypercolor-daemon/src/api/mod.rs | 4 ++ crates/hypercolor-windows-capture/Cargo.toml | 1 + .../src/duplication.rs | 38 +++++++++++++++++ crates/hypercolor-windows-capture/src/lib.rs | 3 +- .../hypercolor-windows-capture/src/shared.rs | 35 ++++++++++++++++ .../tests/duplication_tests.rs | 32 ++++++++++++++ 8 files changed, 192 insertions(+), 1 deletion(-) diff --git a/crates/hypercolor-core/src/input/screen/mod.rs b/crates/hypercolor-core/src/input/screen/mod.rs index f4b8f6795..90c089c3d 100644 --- a/crates/hypercolor-core/src/input/screen/mod.rs +++ b/crates/hypercolor-core/src/input/screen/mod.rs @@ -131,6 +131,48 @@ pub fn fit_within(width: u32, height: u32, max_width: u32, max_height: u32) -> ( } } +/// One display output the capture backend can address directly. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScreenMonitor { + /// Zero-based capture index — the `N` in `capture.source = "monitor:N"`. + pub index: usize, + /// OS device name, e.g. `\\.\DISPLAY1`. + pub name: String, + /// Desktop width in pixels. + pub width: u32, + /// Desktop height in pixels. + pub height: u32, + /// Whether this output anchors the virtual desktop origin. + pub primary: bool, +} + +/// Display outputs the capture backend can address by index. +/// +/// Empty where the backend owns source selection instead (the XDG portal +/// on Linux) or no backend exists. Emptiness is the capability signal: a +/// UI with monitors shows a picker, a UI without falls back to whatever +/// selection flow the platform provides. +#[must_use] +pub fn available_monitors() -> Vec { + #[cfg(target_os = "windows")] + { + hypercolor_windows_capture::list_monitors() + .into_iter() + .map(|monitor| ScreenMonitor { + index: monitor.index, + name: monitor.name, + width: monitor.width, + height: monitor.height, + primary: monitor.primary, + }) + .collect() + } + #[cfg(not(target_os = "windows"))] + { + Vec::new() + } +} + /// Parse a configured capture source into a zero-based monitor index. /// /// `capture.source` is a free-form string shared across backends. The XDG diff --git a/crates/hypercolor-daemon/src/api/capture.rs b/crates/hypercolor-daemon/src/api/capture.rs index 44f12fe3b..b3cce2f1b 100644 --- a/crates/hypercolor-daemon/src/api/capture.rs +++ b/crates/hypercolor-daemon/src/api/capture.rs @@ -40,3 +40,41 @@ pub async fn pick_capture_source(State(state): State>) -> Response info!("Screen capture source picker requested"); ApiResponse::ok(serde_json::json!({ "picking": true })) } + +/// One display output the capture backend can address, for monitor pickers. +#[derive(Debug, serde::Serialize)] +pub struct CaptureMonitor { + /// Zero-based capture index. + pub index: usize, + /// OS device name, e.g. `\\.\DISPLAY1`. + pub name: String, + /// Desktop width in pixels. + pub width: u32, + /// Desktop height in pixels. + pub height: u32, + /// Whether this output anchors the virtual desktop origin. + pub primary: bool, + /// Ready-to-store `capture.source` value selecting this output. + pub value: String, +} + +/// `GET /api/v1/capture/monitors` — Display outputs capture can address. +/// +/// Empty on platforms where the backend picks its own source (the XDG +/// portal on Linux); the UI uses emptiness to decide between a monitor +/// dropdown and the portal picker button. +pub async fn list_capture_monitors() -> Response { + let monitors: Vec = hypercolor_core::input::screen::available_monitors() + .into_iter() + .map(|monitor| CaptureMonitor { + value: format!("monitor:{}", monitor.index), + index: monitor.index, + name: monitor.name, + width: monitor.width, + height: monitor.height, + primary: monitor.primary, + }) + .collect(); + + ApiResponse::ok(monitors) +} diff --git a/crates/hypercolor-daemon/src/api/mod.rs b/crates/hypercolor-daemon/src/api/mod.rs index f7e53b5b0..71da5162d 100644 --- a/crates/hypercolor-daemon/src/api/mod.rs +++ b/crates/hypercolor-daemon/src/api/mod.rs @@ -1339,6 +1339,10 @@ pub fn build_router(state: Arc, ui_dir: Option<&Path>) -> Router { "/capture/source/pick", axum::routing::post(capture::pick_capture_source), ) + .route( + "/capture/monitors", + axum::routing::get(capture::list_capture_monitors), + ) // ── Config ─────────────────────────────────────────────────── .route("/config", axum::routing::get(config::show_config)) .route("/config/get", axum::routing::get(config::get_config_value)) diff --git a/crates/hypercolor-windows-capture/Cargo.toml b/crates/hypercolor-windows-capture/Cargo.toml index 35f09a3b8..504374b45 100644 --- a/crates/hypercolor-windows-capture/Cargo.toml +++ b/crates/hypercolor-windows-capture/Cargo.toml @@ -29,4 +29,5 @@ windows = { version = "0.62", default-features = false, features = [ "Win32_Graphics_Direct3D11", "Win32_Graphics_Dxgi", "Win32_Graphics_Dxgi_Common", + "Win32_Graphics_Gdi", ] } diff --git a/crates/hypercolor-windows-capture/src/duplication.rs b/crates/hypercolor-windows-capture/src/duplication.rs index ef2e62d1c..cbad71ebe 100644 --- a/crates/hypercolor-windows-capture/src/duplication.rs +++ b/crates/hypercolor-windows-capture/src/duplication.rs @@ -61,6 +61,44 @@ pub(crate) fn output_count() -> CaptureResult { enumerate_outputs().map(|outputs| outputs.len()) } +/// Describe every attached output for monitor pickers. +pub(crate) fn describe_outputs() -> CaptureResult> { + let outputs = enumerate_outputs()?; + let mut monitors = Vec::with_capacity(outputs.len()); + + for (index, (_, output)) in outputs.into_iter().enumerate() { + // SAFETY: GetDesc fills a caller-owned struct from the live output. + let desc = match unsafe { output.GetDesc() } { + Ok(desc) => desc, + Err(source) => { + return Err(CaptureError::windows("describe DXGI output", source)); + } + }; + + let name_len = desc + .DeviceName + .iter() + .position(|&c| c == 0) + .unwrap_or(desc.DeviceName.len()); + let name = String::from_utf16_lossy(&desc.DeviceName[..name_len]); + + let bounds = desc.DesktopCoordinates; + let width = u32::try_from(i64::from(bounds.right) - i64::from(bounds.left)).unwrap_or(0); + let height = u32::try_from(i64::from(bounds.bottom) - i64::from(bounds.top)).unwrap_or(0); + + monitors.push(crate::shared::MonitorInfo { + index, + name, + width, + height, + // The primary output anchors the virtual desktop at the origin. + primary: bounds.left == 0 && bounds.top == 0, + }); + } + + Ok(monitors) +} + /// A live Desktop Duplication session for one display output. pub struct DesktopDuplicator { monitor: usize, diff --git a/crates/hypercolor-windows-capture/src/lib.rs b/crates/hypercolor-windows-capture/src/lib.rs index 0f457f5c7..ac68a1271 100644 --- a/crates/hypercolor-windows-capture/src/lib.rs +++ b/crates/hypercolor-windows-capture/src/lib.rs @@ -22,7 +22,8 @@ pub use duplication::DesktopDuplicator; mod shared; pub use shared::{ - CaptureError, CaptureResult, Frame, monitor_count, subsample_stride, subsampled_extent, + CaptureError, CaptureResult, Frame, MonitorInfo, list_monitors, monitor_count, + subsample_stride, subsampled_extent, }; #[cfg(not(target_os = "windows"))] diff --git a/crates/hypercolor-windows-capture/src/shared.rs b/crates/hypercolor-windows-capture/src/shared.rs index 95020c679..e3b97fcf6 100644 --- a/crates/hypercolor-windows-capture/src/shared.rs +++ b/crates/hypercolor-windows-capture/src/shared.rs @@ -80,6 +80,41 @@ pub fn monitor_count() -> usize { } } +/// One attached display output, in capture index order. +/// +/// `index` is the value a [`crate::DesktopDuplicator`] accepts as its +/// `monitor` argument, so a UI can enumerate here and open what it showed. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MonitorInfo { + /// Zero-based capture index (adapter-then-output enumeration order). + pub index: usize, + /// OS device name, e.g. `\\.\DISPLAY1`. + pub name: String, + /// Desktop width in pixels. + pub width: u32, + /// Desktop height in pixels. + pub height: u32, + /// Whether this output hosts the origin of the virtual desktop. + pub primary: bool, +} + +/// Describe every attached display output. +/// +/// Empty when capture is unavailable (non-Windows, headless, RDP), so +/// callers can use emptiness itself as "this platform has no monitor +/// picker" rather than needing a separate capability probe. +#[must_use] +pub fn list_monitors() -> Vec { + #[cfg(target_os = "windows")] + { + crate::duplication::describe_outputs().unwrap_or_default() + } + #[cfg(not(target_os = "windows"))] + { + Vec::new() + } +} + /// Integer subsample stride that brings `source` at or under `target`. /// /// Ambient lighting reduces the frame to a coarse sector grid, so point diff --git a/crates/hypercolor-windows-capture/tests/duplication_tests.rs b/crates/hypercolor-windows-capture/tests/duplication_tests.rs index cb0da42d4..6ebcd5204 100644 --- a/crates/hypercolor-windows-capture/tests/duplication_tests.rs +++ b/crates/hypercolor-windows-capture/tests/duplication_tests.rs @@ -135,3 +135,35 @@ fn alpha_is_opaque_across_repeated_acquisitions() { eprintln!("desktop was static; only {checked} frame(s) verified"); } } + +#[test] +fn monitor_listing_matches_the_count_and_carries_real_geometry() { + use hypercolor_windows_capture::list_monitors; + + let monitors = list_monitors(); + assert_eq!( + monitors.len(), + monitor_count(), + "listing and count must agree" + ); + + if monitors.is_empty() { + eprintln!("skipping: no display outputs attached"); + return; + } + + for monitor in &monitors { + assert!( + monitor.width > 0 && monitor.height > 0, + "monitor {} ({}) reported a zero extent", + monitor.index, + monitor.name + ); + assert!(!monitor.name.is_empty(), "monitor names must not be empty"); + } + assert!( + monitors.iter().any(|monitor| monitor.primary), + "one output should anchor the virtual desktop origin" + ); + eprintln!("monitors: {monitors:?}"); +} From e66fe9a043c6bbce4012f80c6e7f76d15d0bddf5 Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 19:29:29 -0700 Subject: [PATCH 15/16] feat(ui): reduce capture settings to on, monitor, and a disclosure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Screen Capture section was thirteen rows of raw engine knobs — scene-cut thresholds, letterbox luminance floors, sector grid dimensions — presented flat, with a Source dropdown whose entries were backend implementation names ("PipeWire" on a Windows machine). Users met video pipeline internals before they could answer the only two questions that matter: is capture on, and which screen. The section is now those two decisions. The dropdown lists real monitors from /api/v1/capture/monitors with resolution and a primary tag; on portal platforms the list is empty and the portal picker button renders in its place, so each platform shows only the selector it actually honours. Everything else moves behind a collapsed Advanced disclosure with descriptions written for humans: smoothing becomes "Color response", the letterbox toggle says it is for video and warns that dark desktop scenes can be mistaken for bars, and the sampling grid rows say outright that layouts reference the zones so changing them remaps things. Nothing was removed and no keys changed — everything is still there for the users it exists for, in the place that says how often that is. Verified: cargo check and clippy -D warnings clean for the wasm target; daemon api and openapi tests pass (192 + 3). Co-Authored-By: Nova (Claude Opus 5) --- crates/hypercolor-ui/src/api/config.rs | 20 ++ .../src/components/settings_sections.rs | 282 +++++++++++------- 2 files changed, 187 insertions(+), 115 deletions(-) diff --git a/crates/hypercolor-ui/src/api/config.rs b/crates/hypercolor-ui/src/api/config.rs index 46e355be5..ec9c0697d 100644 --- a/crates/hypercolor-ui/src/api/config.rs +++ b/crates/hypercolor-ui/src/api/config.rs @@ -51,6 +51,26 @@ pub async fn reset_config_key(key: &str) -> Result<(), String> { .map_err(Into::into) } +/// One display output capture can address, from `/api/v1/capture/monitors`. +#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)] +pub struct CaptureMonitor { + pub index: usize, + pub name: String, + pub width: u32, + pub height: u32, + pub primary: bool, + /// Ready-to-store `capture.source` value selecting this output. + pub value: String, +} + +/// Display outputs the capture backend can address. Empty on portal +/// platforms, which is the UI's cue to show the picker button instead. +pub async fn fetch_capture_monitors() -> Result, String> { + client::fetch_json("/api/v1/capture/monitors") + .await + .map_err(Into::into) +} + /// Enumerate available audio devices. pub async fn fetch_audio_devices() -> Result { client::fetch_json("/api/v1/audio/devices") diff --git a/crates/hypercolor-ui/src/components/settings_sections.rs b/crates/hypercolor-ui/src/components/settings_sections.rs index 3e9f12319..b8e6140b5 100644 --- a/crates/hypercolor-ui/src/components/settings_sections.rs +++ b/crates/hypercolor-ui/src/components/settings_sections.rs @@ -5,6 +5,7 @@ use std::net::IpAddr; use hypercolor_types::config::{HypercolorConfig, NetworkAccessMode, NetworkClientScope}; use hypercolor_types::session::{OffOutputBehavior, SleepBehavior}; use leptos::prelude::*; +use leptos_icons::Icon; use crate::components::settings_controls::*; use crate::icons::*; @@ -142,10 +143,36 @@ pub fn CaptureSection( Signal::derive(move || read_config(config, |cfg| f64::from(cfg.capture.brightness))); let gamma = Signal::derive(move || read_config(config, |cfg| f64::from(cfg.capture.gamma))); - let source_options = vec![ - ("auto".to_string(), "Auto".to_string()), - ("pipewire".to_string(), "PipeWire".to_string()), - ]; + // Monitor picker data. Empty means the platform's backend owns source + // selection (the XDG portal on Linux), so the portal button renders + // instead of a dropdown. + let monitors_resource = LocalResource::new(crate::api::fetch_capture_monitors); + let monitors = Signal::derive(move || { + monitors_resource + .get() + .and_then(Result::ok) + .unwrap_or_default() + }); + let has_monitor_picker = Signal::derive(move || !monitors.get().is_empty()); + let monitor_options = Signal::derive(move || { + let mut options = vec![("auto".to_owned(), "Primary monitor".to_owned())]; + for monitor in monitors.get() { + let primary = if monitor.primary { " · primary" } else { "" }; + options.push(( + monitor.value.clone(), + format!( + "Monitor {} · {}\u{d7}{}{}", + monitor.index + 1, + monitor.width, + monitor.height, + primary + ), + )); + } + options + }); + + let (show_advanced, set_show_advanced) = signal(false); let (picking, set_picking) = signal(false); let pick_source = move |_| { @@ -165,123 +192,148 @@ pub fn CaptureSection(
- -
-
-
"Capture source"
-
- "Re-open the system picker to capture a different screen; the choice persists across restarts" + + + + +
+
+
"Capture source"
+
+ "Pick which screen or window to mirror; the choice persists across restarts" +
+
- -
- - - - - - - - - - + + + // Everything below is tuning most users never need: video-era + // features (letterbox crop, scene cuts), sampling resolution + // that layouts depend on, and color shaping. Collapsed so the + // section reads as two decisions — on, and which monitor. + + + + + + + + + + + + +
} From 1d406bcdcb82a3ff74b14a0cdbedb8ca653f1e4a Mon Sep 17 00:00:00 2001 From: Stefanie Jane Date: Sat, 25 Jul 2026 21:28:11 -0700 Subject: [PATCH 16/16] fix(capture): gate the Windows error constructor to Windows CI's Linux shared check failed on dead code: CaptureError::windows() is pub(crate) and its only callers live in the cfg(windows) duplication module, so on Linux the constructor compiles with no callers and the workspace's -D warnings turns that into a build failure. Windows clippy can never surface this, the mirror image of the evdev lint that kept main red for a week. Co-Authored-By: Nova (Claude Opus 5) --- crates/hypercolor-windows-capture/src/shared.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/hypercolor-windows-capture/src/shared.rs b/crates/hypercolor-windows-capture/src/shared.rs index e3b97fcf6..e34287d0d 100644 --- a/crates/hypercolor-windows-capture/src/shared.rs +++ b/crates/hypercolor-windows-capture/src/shared.rs @@ -43,6 +43,10 @@ pub enum CaptureError { }, } +// Only the cfg(windows) duplication module builds this variant, so the +// constructor must be gated with it: on Linux it would have no callers and +// the workspace's -D warnings turns dead code into a build failure. +#[cfg(target_os = "windows")] impl CaptureError { /// Build a [`CaptureError::Windows`] from anything printable. pub(crate) fn windows(context: &'static str, message: impl std::fmt::Display) -> Self {