diff --git a/libs/cua-driver/rust/crates/cua-driver/build.rs b/libs/cua-driver/rust/crates/cua-driver/build.rs index 0c11c9611..017ec12ca 100644 --- a/libs/cua-driver/rust/crates/cua-driver/build.rs +++ b/libs/cua-driver/rust/crates/cua-driver/build.rs @@ -22,14 +22,23 @@ fn main() { } println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift"); - if let Ok(out) = std::process::Command::new("xcode-select").arg("-p").output() { + if let Ok(out) = std::process::Command::new("xcode-select") + .arg("-p") + .output() + { if out.status.success() { let xcode_path = String::from_utf8_lossy(&out.stdout).trim().to_string(); for sub in [ "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx", "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.5/macosx", + "usr/lib/swift/macosx", + "usr/lib/swift-5.5/macosx", ] { - println!("cargo:rustc-link-arg=-Wl,-rpath,{xcode_path}/{sub}"); + let runtime_path = std::path::Path::new(&xcode_path).join(sub); + if runtime_path.is_dir() { + println!("cargo:rustc-link-search=native={}", runtime_path.display()); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", runtime_path.display()); + } } } } diff --git a/libs/cua-driver/rust/crates/platform-macos/build.rs b/libs/cua-driver/rust/crates/platform-macos/build.rs index 720b93b5f..2c09aab25 100644 --- a/libs/cua-driver/rust/crates/platform-macos/build.rs +++ b/libs/cua-driver/rust/crates/platform-macos/build.rs @@ -37,14 +37,23 @@ fn main() { // its own build.rs, but platform-macos' unit-test binary is linked from // this package, so it needs the same runtime search paths here. println!("cargo:rustc-link-arg=-Wl,-rpath,/usr/lib/swift"); - if let Ok(out) = std::process::Command::new("xcode-select").arg("-p").output() { + if let Ok(out) = std::process::Command::new("xcode-select") + .arg("-p") + .output() + { if out.status.success() { let xcode_path = String::from_utf8_lossy(&out.stdout).trim().to_string(); for sub in [ "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/macosx", "Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-5.5/macosx", + "usr/lib/swift/macosx", + "usr/lib/swift-5.5/macosx", ] { - println!("cargo:rustc-link-arg=-Wl,-rpath,{xcode_path}/{sub}"); + let runtime_path = std::path::Path::new(&xcode_path).join(sub); + if runtime_path.is_dir() { + println!("cargo:rustc-link-search=native={}", runtime_path.display()); + println!("cargo:rustc-link-arg=-Wl,-rpath,{}", runtime_path.display()); + } } } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs b/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs index 756295178..51ef3467b 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs @@ -27,6 +27,7 @@ //! selector which `objc2-foundation 0.2.2` does not bind natively — we //! hand-roll the binding in [`apple_event`] via `extern_methods!`. +use std::mem::{size_of, MaybeUninit}; use std::ptr::NonNull; use std::sync::{Arc, OnceLock}; use std::time::Duration; @@ -494,8 +495,10 @@ fn running_application_info(application: &NSRunningApplication) -> RunningApplic let executable_path = unsafe { application.executableURL() } .and_then(|url| unsafe { url.path() }) .map(|path| path.to_string()); - let process_generation = unsafe { application.launchDate() } - .map(|date| unsafe { date.timeIntervalSince1970() }.to_bits()); + let process_generation = process_generation(pid).or_else(|| { + unsafe { application.launchDate() } + .and_then(|date| generation_from_epoch_seconds(unsafe { date.timeIntervalSince1970() })) + }); RunningApplicationInfo { pid, name, @@ -510,6 +513,45 @@ fn running_application_info(application: &NSRunningApplication) -> RunningApplic } } +/// Exact live-process generation derived from the kernel's BSD process row. +/// +/// `NSRunningApplication.launchDate` is legitimately nil for long-lived +/// regular processes including Finder and Docker Desktop. `proc_pidinfo` +/// exposes the process start timestamp for those same PIDs, so use that as +/// the canonical identity and retain launchDate only as a late-notification +/// fallback after the kernel row has disappeared. +pub(crate) fn process_generation(pid: i32) -> Option { + let mut info = MaybeUninit::::zeroed(); + let expected = size_of::(); + let written = unsafe { + libc::proc_pidinfo( + pid, + libc::PROC_PIDTBSDINFO, + 0, + info.as_mut_ptr().cast(), + i32::try_from(expected).ok()?, + ) + }; + if usize::try_from(written).ok()? != expected { + return None; + } + let info = unsafe { info.assume_init() }; + if info.pbi_pid != u32::try_from(pid).ok()? || info.pbi_start_tvsec == 0 { + return None; + } + info.pbi_start_tvsec + .checked_mul(1_000_000)? + .checked_add(info.pbi_start_tvusec) +} + +fn generation_from_epoch_seconds(seconds: f64) -> Option { + if !seconds.is_finite() || seconds <= 0.0 { + return None; + } + let micros = (seconds * 1_000_000.0).round(); + (micros <= u64::MAX as f64).then_some(micros as u64) +} + fn install_workspace_observers(hub: &Arc) { install_workspace_observer( hub, @@ -608,3 +650,16 @@ mod apple_event { } } } + +#[cfg(test)] +mod tests { + use super::process_generation; + + #[test] + fn live_process_generation_is_kernel_backed_and_stable() { + let pid = i32::try_from(std::process::id()).unwrap(); + let first = process_generation(pid).expect("current process must have a BSD start time"); + let second = process_generation(pid).expect("current process must remain readable"); + assert_eq!(first, second); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs index d03165100..2a35fbee4 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs @@ -356,6 +356,53 @@ pub unsafe fn copy_cf_range_attr( } } +/// Copy a `kAXValueCGPointType` attribute. Missing and unsupported attributes +/// are optional; query and type failures remain distinguishable to callers +/// that need an exact read. +pub unsafe fn copy_point_attr( + element: AXUIElementRef, + attr_name: &str, +) -> Result, AXError> { + let attr = CFStr::new(attr_name); + let mut value: CFTypeRef = std::ptr::null(); + let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value); + if err == kAXErrorNoValue || err == kAXErrorAttributeUnsupported { + return Ok(None); + } + if err != kAXErrorSuccess { + return Err(err); + } + if value.is_null() { + return Ok(None); + } + if core_foundation::base::CFGetTypeID(value) != AXValueGetTypeID() { + CFRelease(value); + return Err(kAXErrorFailure); + } + let value = value as AXValueRef; + if AXValueGetType(value) != kAXValueCGPointType { + CFRelease(value as CFTypeRef); + return Err(kAXErrorFailure); + } + #[repr(C)] + struct CGPoint { + x: f64, + y: f64, + } + let mut point = CGPoint { x: 0.0, y: 0.0 }; + let copied = AXValueGetValue( + value, + kAXValueCGPointType, + &mut point as *mut _ as *mut c_void, + ); + CFRelease(value as CFTypeRef); + if copied { + Ok(Some((point.x, point.y))) + } else { + Err(kAXErrorFailure) + } +} + /// Set a `kAXValueCFRangeType` attribute. pub unsafe fn set_cf_range_attr( element: AXUIElementRef, diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/keyboard.rs index 8ad04d777..6c88b03ce 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/keyboard.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/keyboard.rs @@ -1132,7 +1132,10 @@ pub(crate) fn keyboard_capability_cells(os_version: &str) -> Vec WindowStateKind::Visible | WindowStateKind::Occluded if matches!( framework, - Framework::AppKit | Framework::Chromium | Framework::WebKit + Framework::Unknown + | Framework::AppKit + | Framework::Chromium + | Framework::WebKit ) => { RouteDecision::Supported { @@ -1481,6 +1484,21 @@ mod tests { .filter(|cell| { cell.key.action == ActionKind::TypeText && cell.key.framework == Framework::Unknown }) - .all(|cell| matches!(cell.decision, RouteDecision::Unsupported { .. }))); + .all(|cell| { + matches!( + (&cell.key.window_state, &cell.decision), + ( + WindowStateKind::Visible | WindowStateKind::Occluded, + RouteDecision::Supported { + route: Route::Semantic + } + ) | ( + WindowStateKind::Minimized + | WindowStateKind::OffSpace + | WindowStateKind::Unknown, + RouteDecision::Unsupported { .. } + ) + ) + })); } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/mod.rs index b6739a31f..a8e4a9686 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/mod.rs @@ -107,7 +107,7 @@ pub(crate) fn pointer_capability_cells(os_version: &str) -> Vec for state in &states { let proven_click = os_version == "26.5.1" && architecture == "arm64" - && framework == Framework::Chromium + && framework != Framework::Catalyst && *action == ActionKind::Click && *state == WindowStateKind::Visible; let decision = if proven_click { @@ -174,7 +174,7 @@ mod tests { } #[test] - fn targeted_pointer_cells_publish_only_the_proven_chromium_click_posture() { + fn targeted_pointer_cells_publish_the_exact_click_posture_without_framework_guessing() { let cells = pointer_capability_cells("26.5.1"); assert_eq!(cells.len(), 90); let supported: Vec<_> = cells @@ -182,11 +182,10 @@ mod tests { .filter(|cell| matches!(cell.decision, RouteDecision::Supported { .. })) .collect(); if std::env::consts::ARCH == "aarch64" { - assert_eq!(supported.len(), 1); + assert_eq!(supported.len(), 5); assert!(supported.iter().all(|cell| { cell.key.action == ActionKind::Click && cell.key.addressing == AddressingMode::CapturedPoint - && cell.key.framework == Framework::Chromium && cell.key.window_state == WindowStateKind::Visible && matches!( cell.decision, @@ -195,6 +194,15 @@ mod tests { } ) })); + assert!(supported + .iter() + .any(|cell| cell.key.framework == Framework::Unknown)); + assert!(cells.iter().any(|cell| { + cell.key.action == ActionKind::Click + && cell.key.framework == Framework::Catalyst + && cell.key.window_state == WindowStateKind::Visible + && matches!(cell.decision, RouteDecision::Unsupported { .. }) + })); assert!(cells.iter().any(|cell| { cell.key.action == ActionKind::Click && cell.key.framework == Framework::Chromium diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/pointer.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/pointer.rs index d545c2d35..9f3ac7bcc 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/pointer.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/pointer.rs @@ -1,20 +1,12 @@ //! Exact window-stamped targeted pointer sequences for the v2 driver. use std::{ - sync::{ - atomic::{AtomicI64, Ordering}, - Arc, - }, + sync::Arc, thread, time::{Duration, Instant}, }; use async_trait::async_trait; -use core_graphics::{ - event::{CGEvent, CGEventFlags, CGEventType, CGMouseButton, EventField, ScrollEventUnit}, - event_source::{CGEventSource, CGEventSourceStateID}, - geometry::CGPoint, -}; use cua_driver_core::api::{ contracts::{Modifier, MouseButton, Point, VerificationLevel}, errors::{ErrorCode, ErrorPhase, NativeError}, @@ -24,7 +16,6 @@ use cua_driver_core::api::{ settlement::SettlementSignal, Route, }; -use foreign_types::ForeignType; use serde_json::Value; use crate::{ @@ -33,18 +24,11 @@ use crate::{ target::MacTargetState, windows::{MacRelatedWindowFacts, MacWindowFacts, MacWindowRegistry}, }, - input::skylight, + focus_steal, + input::synthesized_event::{self, MouseEventKind as NativeMouseEventKind, MouseEventSpec}, }; -const CLICK_PRIMER_MOVE_MS: u64 = 15; -const CLICK_PRIMER_UP_MS: u64 = 16; -const CLICK_TARGET_START_MS: u64 = 116; -const CLICK_UP_DELAY_MS: u64 = 1; -const CLICK_PAIR_GAP_MS: u64 = 80; -const MAX_DRAG_STEPS: usize = 60; -const MIN_DRAG_STEPS: usize = 2; -const DRAG_POINTS_PER_STEP: f64 = 16.0; -const FIXED_POINT_SCALE: f64 = 65_536.0; +const CLICK_UP_DELAY_MS: u64 = 100; const POINT_EPSILON: f64 = 0.001; #[derive(Clone)] @@ -147,57 +131,28 @@ impl MacPointerActions { "targeted pointer prepare requires a live targeted-pointer scope", )); } - if !skylight::is_targeted_pointer_route_available() { - return Err(NativeError::unsupported( - "recipe_unproven: complete SkyLight targeted-pointer post/window-stamp symbols are unavailable", - ) - .with_detail("recipe_status", "recipe_unproven")); - } - let (sequence, final_cursor, route_name) = match action { ResolvedAction::PointClick { point, spec } => { + if spec.button != MouseButton::Left + || spec.click_count != 1 + || !spec.modifiers.is_empty() + { + return Err(NativeError::unsupported( + "recipe_unproven: the canonical macOS pointer route currently proves only an unmodified single left click", + ) + .with_detail("recipe_status", "recipe_unproven")); + } let point = self .prepare_point(target.window.clone(), scope.owner.clone(), point) .await?; - let sequence = build_gesture( - point.clone(), - None, - spec.button, - spec.click_count, - &spec.modifiers, - 0, - )?; + let sequence = build_click(point.clone()); (sequence, point.logical, "macos_targeted_pointer_click") } - ResolvedAction::Drag(drag) => { - let start = self - .prepare_point(target.window.clone(), scope.owner.clone(), &drag.start) - .await?; - let end = self - .prepare_point(target.window.clone(), scope.owner.clone(), &drag.end) - .await?; - ensure_same_surface(&start, &end)?; - if same_point(start.screen, end.screen) { - return Err(NativeError::invalid( - "drag endpoints must differ so the native sequence contains real motion", - )); - } - let sequence = build_gesture( - start, - Some(end.clone()), - drag.button, - 1, - &drag.modifiers, - drag.duration_ms, - )?; - (sequence, end.logical, "macos_targeted_pointer_drag") - } - ResolvedAction::DeltaScroll(scroll) => { - let point = self - .prepare_point(target.window.clone(), scope.owner.clone(), &scroll.point) - .await?; - let sequence = build_scroll(point.clone(), scroll.delta_x, scroll.delta_y)?; - (sequence, point.logical, "macos_targeted_pointer_scroll") + ResolvedAction::Drag(_) | ResolvedAction::DeltaScroll(_) => { + return Err(NativeError::unsupported( + "recipe_unproven: drag and scroll are not published on the canonical macOS pointer route", + ) + .with_detail("recipe_status", "recipe_unproven")); } _ => { return Err(NativeError::new( @@ -272,14 +227,15 @@ impl MacPointerActions { ); return Err(error); } + focus_steal::record_synthesized_action(action.sequence.target.pid); scope.logical_cursor.update(action.final_cursor); target .signals .record(SettlementSignal::PointerSequenceComplete); - // Both SkyLight and public CGEvent process posting are void APIs. A - // completed call proves only that the complete targeted sequence was - // attempted; it is not a native delivery acknowledgement. + // The helper's CGEventPostToPid transport is a void API. A completed + // call proves only that the complete targeted sequence was attempted; + // it is not a native delivery acknowledgement. let verification = VerificationLevel::DispatchUnverified; let mut native = NativeEvidence::default(); native @@ -291,7 +247,7 @@ impl MacPointerActions { ); native.fields.insert( "pointer_transport".to_owned(), - "skylight_then_core_graphics_post_to_pid".into(), + "appkit_event_core_graphics_post_to_pid_once".into(), ); native.fields.insert( "surface_id".to_owned(), @@ -333,11 +289,6 @@ impl MacPointerActions { "pointer target/window geometry changed after native action preparation", )); } - if !skylight::is_targeted_pointer_route_available() { - return Err(NativeError::unsupported( - "recipe_unproven: targeted pointer symbols disappeared before dispatch", - )); - } let (pid, cg_window_id, bounds, stamp) = if sequence.target.owner == target_window { let facts = self.windows.facts_for_stamp(&target_window).await?; (facts.pid, facts.cg_window_id, facts.bounds, facts.stamp) @@ -365,12 +316,6 @@ impl MacPointerActions { )); } for event in &sequence.events { - if matches!( - event.kind, - PointerEventKind::PrimerDown | PointerEventKind::PrimerUp - ) { - continue; - } let expected_screen = Point { x: bounds.x + event.point.window_local.x, y: bounds.y + event.point.window_local.y, @@ -446,13 +391,8 @@ struct NativePoint { #[derive(Debug, Clone, Copy, PartialEq, Eq)] enum PointerEventKind { - Moved, - PrimerDown, - PrimerUp, Down, - Dragged, Up, - Scroll, } #[derive(Debug, Clone)] @@ -463,177 +403,31 @@ struct PreparedPointerEvent { click_state: u8, modifiers: Vec, scheduled_after: Duration, - native_scroll_x: f64, - native_scroll_y: f64, } -fn build_gesture( - start: NativePoint, - drag_to: Option, - button: MouseButton, - click_count: u8, - modifiers: &[Modifier], - duration_ms: u32, -) -> Result { - if !(1..=3).contains(&click_count) { - return Err(NativeError::invalid( - "targeted pointer click_count must be between 1 and 3", - )); - } - if drag_to.is_some() && click_count != 1 { - return Err(NativeError::invalid( - "a targeted drag is one gesture and cannot carry multiple click pairs", - )); - } - let primer = NativePoint { - screen: Point { x: -1.0, y: -1.0 }, - window_local: Point { x: -1.0, y: -1.0 }, - logical: start.logical, - ..start.clone() - }; - let mut events = vec![ +fn build_click(point: NativePoint) -> PreparedSequence { + let events = vec![ pointer_event( - PointerEventKind::Moved, - start.clone(), - button, - 0, - modifiers, - 0, - ), - pointer_event( - PointerEventKind::PrimerDown, - primer.clone(), + PointerEventKind::Down, + point.clone(), MouseButton::Left, 1, &[], - CLICK_PRIMER_MOVE_MS, + 0, ), pointer_event( - PointerEventKind::PrimerUp, - primer, + PointerEventKind::Up, + point.clone(), MouseButton::Left, 1, &[], - CLICK_PRIMER_UP_MS, + CLICK_UP_DELAY_MS, ), ]; - - if let Some(end) = drag_to { - events.push(pointer_event( - PointerEventKind::Down, - start.clone(), - button, - 1, - modifiers, - CLICK_TARGET_START_MS, - )); - let distance = ((end.screen.x - start.screen.x).powi(2) - + (end.screen.y - start.screen.y).powi(2)) - .sqrt(); - let steps = ((distance / DRAG_POINTS_PER_STEP).ceil() as usize) - .clamp(MIN_DRAG_STEPS, MAX_DRAG_STEPS); - for index in 1..=steps { - let fraction = index as f64 / (steps + 1) as f64; - let point = interpolate_point(&start, &end, fraction); - let delay = u64::from(duration_ms) * index as u64 / (steps + 1) as u64; - events.push(pointer_event( - PointerEventKind::Dragged, - point, - button, - 1, - modifiers, - CLICK_TARGET_START_MS + delay, - )); - } - events.push(pointer_event( - PointerEventKind::Up, - end, - button, - 1, - modifiers, - CLICK_TARGET_START_MS + u64::from(duration_ms), - )); - return Ok(PreparedSequence { - target: start, - events, - }); - } - - for pair_index in 0..click_count { - let down_at = - CLICK_TARGET_START_MS + u64::from(pair_index) * (CLICK_UP_DELAY_MS + CLICK_PAIR_GAP_MS); - let click_state = pair_index + 1; - events.push(pointer_event( - PointerEventKind::Down, - start.clone(), - button, - click_state, - modifiers, - down_at, - )); - events.push(pointer_event( - PointerEventKind::Up, - start.clone(), - button, - click_state, - modifiers, - down_at + CLICK_UP_DELAY_MS, - )); - } - Ok(PreparedSequence { - target: start, - events, - }) -} - -fn build_scroll( - point: NativePoint, - public_delta_x: f64, - public_delta_y: f64, -) -> Result { - if !public_delta_x.is_finite() || !public_delta_y.is_finite() { - return Err(NativeError::invalid( - "targeted pointer scroll deltas must be finite", - )); - } - if public_delta_x == 0.0 && public_delta_y == 0.0 { - return Err(NativeError::invalid( - "targeted pointer scroll deltas cannot both be zero", - )); - } - // Core is positive-right/positive-down. Native pixel wheel deltas are the - // opposite: positive reveals left/up content. - let native_scroll_x = exact_fixed_delta(-public_delta_x)?; - let native_scroll_y = exact_fixed_delta(-public_delta_y)?; - let event = PreparedPointerEvent { - kind: PointerEventKind::Scroll, - point: point.clone(), - button: MouseButton::Left, - click_state: 0, - modifiers: Vec::new(), - scheduled_after: Duration::ZERO, - native_scroll_x, - native_scroll_y, - }; - Ok(PreparedSequence { + PreparedSequence { target: point, - events: vec![event], - }) -} - -fn exact_fixed_delta(value: f64) -> Result { - let scaled = value * FIXED_POINT_SCALE; - if !scaled.is_finite() - || scaled < f64::from(i32::MIN) - || scaled > f64::from(i32::MAX) - || (scaled.round() - scaled).abs() > f64::EPSILON * scaled.abs().max(1.0) * 4.0 - { - return Err(NativeError::unsupported( - "exact macOS pixel scroll route requires a signed 16.16-representable logical delta", - ) - .with_detail("requested_native_delta", value)); + events, } - Ok(scaled.round() / FIXED_POINT_SCALE) } fn pointer_event( @@ -651,27 +445,6 @@ fn pointer_event( click_state, modifiers: modifiers.to_vec(), scheduled_after: Duration::from_millis(scheduled_after_ms), - native_scroll_x: 0.0, - native_scroll_y: 0.0, - } -} - -fn interpolate_point(start: &NativePoint, end: &NativePoint, fraction: f64) -> NativePoint { - let interpolate = |left: f64, right: f64| left + (right - left) * fraction; - NativePoint { - screen: Point { - x: interpolate(start.screen.x, end.screen.x), - y: interpolate(start.screen.y, end.screen.y), - }, - window_local: Point { - x: interpolate(start.window_local.x, end.window_local.x), - y: interpolate(start.window_local.y, end.window_local.y), - }, - logical: Point { - x: interpolate(start.logical.x, end.logical.x), - y: interpolate(start.logical.y, end.logical.y), - }, - ..start.clone() } } @@ -756,23 +529,6 @@ fn related_point_facts( Ok((facts.pid, facts.cg_window_id, facts.bounds)) } -fn ensure_same_surface(start: &NativePoint, end: &NativePoint) -> Result<(), NativeError> { - if start.pid != end.pid - || start.cg_window_id != end.cg_window_id - || start.owner != end.owner - || start.surface_id != end.surface_id - || start.capture_revision != end.capture_revision - || start.geometry_revision != end.geometry_revision - || start.observation_epoch != end.observation_epoch - { - return Err(NativeError::stale( - ErrorCode::SurfaceStale, - "drag endpoints do not share one current captured surface and native owner", - )); - } - Ok(()) -} - fn surface_owner_stamp(owner: &SurfaceOwner) -> &ResolvedWindowStamp { match owner { SurfaceOwner::Target(owner) | SurfaceOwner::RelatedTransient { owner, .. } => owner, @@ -794,7 +550,6 @@ trait TargetedPointerSink: Send + Sync { fn post( &self, event: &PreparedPointerEvent, - gesture_id: i64, boundary: &mut NativeSideEffectBoundary<'_>, epoch: Option<&PointerEpochWitness>, ) -> Result<(), NativeError>; @@ -829,33 +584,24 @@ impl TargetedPointerSink for SystemTargetedPointerSink { fn post( &self, event: &PreparedPointerEvent, - gesture_id: i64, boundary: &mut NativeSideEffectBoundary<'_>, epoch: Option<&PointerEpochWitness>, ) -> Result<(), NativeError> { - let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { - NativeError::new( - ErrorCode::DispatchFailed, - ErrorPhase::Dispatch, - false, - "CGEventSource creation failed for targeted pointer dispatch", - ) - })?; - let native = match event.kind { - PointerEventKind::Scroll => build_native_scroll(source, event)?, - _ => build_native_mouse(source, event)?, - }; - stamp_targeted_event(&native, event, gesture_id)?; - let pointer = native.as_ptr() as *mut std::ffi::c_void; let mut post = || { boundary.begin()?; - post_both_targeted_transports( - || skylight::post_to_pid(event.point.pid, pointer, false), - || { - native.post_to_pid(event.point.pid); - Ok(()) + synthesized_event::post_mouse_event(&MouseEventSpec { + pid: event.point.pid, + cg_window_id: event.point.cg_window_id, + screen: event.point.screen, + window_local: event.point.window_local, + button: event.button, + click_count: event.click_state, + modifiers: &event.modifiers, + kind: match event.kind { + PointerEventKind::Down => NativeMouseEventKind::Down, + PointerEventKind::Up => NativeMouseEventKind::Up, }, - ) + }) }; match epoch { Some(epoch) => epoch.commit_first_post(post), @@ -864,230 +610,6 @@ impl TargetedPointerSink for SystemTargetedPointerSink { } } -fn post_both_targeted_transports( - skylight_post: impl FnOnce() -> bool, - core_graphics_post: impl FnOnce() -> Result<(), NativeError>, -) -> Result<(), NativeError> { - let skylight_succeeded = skylight_post(); - let core_graphics_result = core_graphics_post(); - let core_graphics_succeeded = core_graphics_result.is_ok(); - - let skylight_error = (!skylight_succeeded).then(|| { - NativeError::new( - ErrorCode::DispatchFailed, - ErrorPhase::Dispatch, - false, - "SLEventPostToPid became unavailable after pointer preflight", - ) - }); - let mut failures = Vec::new(); - if let Some(error) = skylight_error { - failures.push(error); - } - if let Err(error) = core_graphics_result { - failures.push(error); - } - let Some(error) = NativeError::primary(failures) else { - return Ok(()); - }; - Err(error - .with_detail("skylight_attempted", true) - .with_detail("skylight_succeeded", skylight_succeeded) - .with_detail("core_graphics_attempted", true) - .with_detail("core_graphics_call_completed", core_graphics_succeeded)) -} - -fn build_native_mouse( - source: CGEventSource, - event: &PreparedPointerEvent, -) -> Result { - let (event_type, button) = native_mouse_shape(event.kind, event.button)?; - let native = CGEvent::new_mouse_event( - source, - event_type, - CGPoint::new(event.point.screen.x, event.point.screen.y), - button, - ) - .map_err(|_| { - NativeError::new( - ErrorCode::DispatchFailed, - ErrorPhase::Dispatch, - false, - "CGEvent mouse event construction failed", - ) - })?; - let flags = modifier_flags(&event.modifiers); - if flags != CGEventFlags::CGEventFlagNull { - native.set_flags(flags); - } - Ok(native) -} - -fn build_native_scroll( - source: CGEventSource, - event: &PreparedPointerEvent, -) -> Result { - let native_y = event.native_scroll_y.round(); - let native_x = event.native_scroll_x.round(); - if native_y < f64::from(i32::MIN) - || native_y > f64::from(i32::MAX) - || native_x < f64::from(i32::MIN) - || native_x > f64::from(i32::MAX) - { - return Err(NativeError::unsupported( - "macOS pixel scroll delta exceeds the native event field range", - )); - } - let native = CGEvent::new_scroll_event( - source, - ScrollEventUnit::PIXEL, - 2, - native_y as i32, - native_x as i32, - 0, - ) - .map_err(|_| { - NativeError::new( - ErrorCode::DispatchFailed, - ErrorPhase::Dispatch, - false, - "CGEvent pixel scroll construction failed", - ) - })?; - native.set_double_value_field( - EventField::SCROLL_WHEEL_EVENT_FIXED_POINT_DELTA_AXIS_1, - event.native_scroll_y, - ); - native.set_double_value_field( - EventField::SCROLL_WHEEL_EVENT_FIXED_POINT_DELTA_AXIS_2, - event.native_scroll_x, - ); - native.set_integer_value_field(EventField::SCROLL_WHEEL_EVENT_IS_CONTINUOUS, 1); - unsafe { - CGEventSetLocation( - native.as_ptr() as *mut std::ffi::c_void, - event.point.screen.x, - event.point.screen.y, - ); - } - Ok(native) -} - -fn stamp_targeted_event( - event: &CGEvent, - spec: &PreparedPointerEvent, - gesture_id: i64, -) -> Result<(), NativeError> { - let pointer = event.as_ptr() as *mut std::ffi::c_void; - if !skylight::set_window_location( - pointer, - spec.point.window_local.x, - spec.point.window_local.y, - ) { - return Err(missing_stamp("CGEventSetWindowLocation")); - } - for (field, value) in [ - (0, phase(spec.kind)), - (1, i64::from(spec.click_state)), - (3, button_number(spec.button)), - (7, 3), - (40, i64::from(spec.point.pid)), - (51, i64::from(spec.point.cg_window_id)), - (58, gesture_id), - (91, i64::from(spec.point.cg_window_id)), - (92, i64::from(spec.point.cg_window_id)), - ] { - if !skylight::set_integer_field(pointer, field, value) { - return Err(missing_stamp("SLEventSetIntegerValueField")); - } - } - Ok(()) -} - -fn native_mouse_shape( - kind: PointerEventKind, - button: MouseButton, -) -> Result<(CGEventType, CGMouseButton), NativeError> { - let native_button = match button { - MouseButton::Left => CGMouseButton::Left, - MouseButton::Right => CGMouseButton::Right, - MouseButton::Middle => CGMouseButton::Center, - }; - let event_type = match (kind, button) { - (PointerEventKind::Moved, _) => CGEventType::MouseMoved, - (PointerEventKind::PrimerDown, _) | (PointerEventKind::Down, MouseButton::Left) => { - CGEventType::LeftMouseDown - } - (PointerEventKind::PrimerUp, _) | (PointerEventKind::Up, MouseButton::Left) => { - CGEventType::LeftMouseUp - } - (PointerEventKind::Down, MouseButton::Right) => CGEventType::RightMouseDown, - (PointerEventKind::Up, MouseButton::Right) => CGEventType::RightMouseUp, - (PointerEventKind::Down, MouseButton::Middle) => CGEventType::OtherMouseDown, - (PointerEventKind::Up, MouseButton::Middle) => CGEventType::OtherMouseUp, - (PointerEventKind::Dragged, MouseButton::Left) => CGEventType::LeftMouseDragged, - (PointerEventKind::Dragged, MouseButton::Right) => CGEventType::RightMouseDragged, - (PointerEventKind::Dragged, MouseButton::Middle) => CGEventType::OtherMouseDragged, - (PointerEventKind::Scroll, _) => { - return Err(NativeError::new( - ErrorCode::Internal, - ErrorPhase::Dispatch, - false, - "scroll event reached mouse event construction", - )) - } - }; - Ok((event_type, native_button)) -} - -fn phase(kind: PointerEventKind) -> i64 { - match kind { - PointerEventKind::Moved | PointerEventKind::PrimerUp => 2, - PointerEventKind::PrimerDown => 1, - PointerEventKind::Down - | PointerEventKind::Dragged - | PointerEventKind::Up - | PointerEventKind::Scroll => 3, - } -} - -fn button_number(button: MouseButton) -> i64 { - match button { - MouseButton::Left => 0, - MouseButton::Right => 1, - MouseButton::Middle => 2, - } -} - -fn modifier_flags(modifiers: &[Modifier]) -> CGEventFlags { - modifiers - .iter() - .fold(CGEventFlags::CGEventFlagNull, |flags, modifier| { - flags - | match modifier { - Modifier::Shift => CGEventFlags::CGEventFlagShift, - Modifier::Control => CGEventFlags::CGEventFlagControl, - Modifier::Alt => CGEventFlags::CGEventFlagAlternate, - Modifier::Meta => CGEventFlags::CGEventFlagCommand, - } - }) -} - -fn missing_stamp(symbol: &'static str) -> NativeError { - NativeError::new( - ErrorCode::DispatchFailed, - ErrorPhase::Dispatch, - false, - "required targeted pointer event stamp became unavailable after preflight", - ) - .with_detail("symbol", symbol) -} - -#[link(name = "CoreGraphics", kind = "framework")] -extern "C" { - fn CGEventSetLocation(event: *mut std::ffi::c_void, x: f64, y: f64); -} - fn dispatch_sequence( sink: &dyn TargetedPointerSink, sequence: &PreparedSequence, @@ -1098,14 +620,13 @@ fn dispatch_sequence( epoch: &PointerEpochWitness, ) -> Result<(), NativeError> { let started = Instant::now(); - let gesture_id = gesture_id(sequence); let mut pressed: Option<(MouseButton, NativePoint)> = None; for event in &sequence.events { if let Err(error) = wait_until(started + event.scheduled_after, deadline) { let dispatch_started = boundary.started() || evidence.completed_events > 0; let cleanup = if dispatch_started { - cleanup_pressed(sink, pressed.take(), gesture_id, boundary) + cleanup_pressed(sink, pressed.take(), boundary) } else { CleanupResult::not_needed() }; @@ -1119,19 +640,16 @@ fn dispatch_sequence( evidence.completed_events, )); } - if matches!( - event.kind, - PointerEventKind::PrimerDown | PointerEventKind::Down - ) { + if event.kind == PointerEventKind::Down { // Posting APIs are void. A returned error may still follow partial // native delivery, so cleanup must conservatively assume down. pressed = Some((event.button, event.point.clone())); } let first_post_epoch = (evidence.completed_events == 0).then_some(epoch); - if let Err(error) = sink.post(event, gesture_id, boundary, first_post_epoch) { + if let Err(error) = sink.post(event, boundary, first_post_epoch) { let dispatch_started = boundary.started() || evidence.completed_events > 0; let cleanup = if dispatch_started { - cleanup_pressed(sink, pressed.take(), gesture_id, boundary) + cleanup_pressed(sink, pressed.take(), boundary) } else { CleanupResult::not_needed() }; @@ -1148,12 +666,11 @@ fn dispatch_sequence( evidence.completed_events += 1; evidence.may_have_partially_landed = true; match event.kind { - PointerEventKind::PrimerDown | PointerEventKind::Down => {} - PointerEventKind::PrimerUp | PointerEventKind::Up => pressed = None, - PointerEventKind::Moved | PointerEventKind::Dragged => { + PointerEventKind::Down => {} + PointerEventKind::Up => { + pressed = None; logical_cursor.update(event.point.logical); } - PointerEventKind::Scroll => logical_cursor.update(event.point.logical), } } evidence.may_have_partially_landed = false; @@ -1196,7 +713,6 @@ impl CleanupResult { fn cleanup_pressed( sink: &dyn TargetedPointerSink, pressed: Option<(MouseButton, NativePoint)>, - gesture_id: i64, boundary: &mut NativeSideEffectBoundary<'_>, ) -> CleanupResult { let Some((button, point)) = pressed else { @@ -1205,7 +721,7 @@ fn cleanup_pressed( let cleanup = pointer_event(PointerEventKind::Up, point, button, 1, &[], 0); CleanupResult { attempted: true, - succeeded: sink.post(&cleanup, gesture_id, boundary, None).is_ok(), + succeeded: sink.post(&cleanup, boundary, None).is_ok(), } } @@ -1229,14 +745,6 @@ fn error_with_cleanup( error } -fn gesture_id(sequence: &PreparedSequence) -> i64 { - static NEXT_GESTURE_ID: AtomicI64 = AtomicI64::new(1); - let sequence_id = NEXT_GESTURE_ID.fetch_add(1, Ordering::Relaxed); - sequence_id - ^ i64::from(sequence.target.cg_window_id) - ^ i64::from(sequence.target.pid).rotate_left(17) -} - struct PointerDispatchEvidence { completed_events: usize, cleanup_attempted: bool, @@ -1340,111 +848,21 @@ mod tests { } #[test] - fn one_builder_preserves_button_count_modifiers_and_drag_motion() { - for (button, count) in [ - (MouseButton::Left, 1), - (MouseButton::Right, 2), - (MouseButton::Middle, 3), - ] { - let sequence = build_gesture( - point(10.0, 20.0), - None, - button, - count, - &[Modifier::Shift, Modifier::Meta], - 0, - ) - .unwrap(); - let target: Vec<_> = sequence - .events - .iter() - .filter(|event| matches!(event.kind, PointerEventKind::Down | PointerEventKind::Up)) - .collect(); - assert_eq!(target.len(), usize::from(count) * 2); - assert!(target.iter().all(|event| { - event.button == button && event.modifiers == [Modifier::Shift, Modifier::Meta] - })); - assert_eq!( - target - .chunks_exact(2) - .map(|pair| pair[0].click_state) - .collect::>(), - (1..=count).collect::>() - ); - } - - let drag = build_gesture( - point(0.0, 0.0), - Some(point(64.0, 32.0)), - MouseButton::Right, - 1, - &[Modifier::Alt], - 400, - ) - .unwrap(); - assert!(matches!(drag.events[0].kind, PointerEventKind::Moved)); - let dragged: Vec<_> = drag - .events - .iter() - .filter(|event| event.kind == PointerEventKind::Dragged) - .collect(); - assert!(dragged.len() >= 2); - assert!(dragged.windows(2).all(|pair| { - pair[0].scheduled_after <= pair[1].scheduled_after - && pair[0].point.screen.x < pair[1].point.screen.x + fn canonical_click_is_one_down_up_pair_with_the_swift_interval() { + let sequence = build_click(point(10.0, 20.0)); + assert_eq!(sequence.events.len(), 2); + assert_eq!(sequence.events[0].kind, PointerEventKind::Down); + assert_eq!(sequence.events[1].kind, PointerEventKind::Up); + assert_eq!(sequence.events[0].scheduled_after, Duration::ZERO); + assert_eq!( + sequence.events[1].scheduled_after, + Duration::from_millis(100) + ); + assert!(sequence.events.iter().all(|event| { + event.button == MouseButton::Left + && event.click_state == 1 + && event.modifiers.is_empty() })); - assert!(matches!( - drag.events.last().unwrap().kind, - PointerEventKind::Up - )); - } - - #[test] - fn scroll_converts_public_right_down_only_at_native_boundary() { - let sequence = build_scroll(point(5.0, 6.0), 12.5, 7.25).unwrap(); - let event = &sequence.events[0]; - assert_eq!(event.native_scroll_x, -12.5); - assert_eq!(event.native_scroll_y, -7.25); - assert!(build_scroll(point(0.0, 0.0), 0.1, 0.0).is_err()); - } - - #[test] - fn proved_pointer_transport_posts_skylight_then_core_graphics() { - let calls = Mutex::new(Vec::new()); - post_both_targeted_transports( - || { - calls.lock().unwrap().push("skylight"); - true - }, - || { - calls.lock().unwrap().push("core_graphics"); - Ok(()) - }, - ) - .unwrap(); - assert_eq!(*calls.lock().unwrap(), ["skylight", "core_graphics"]); - } - - #[test] - fn failed_skylight_post_still_attempts_core_graphics_and_reports_both() { - let calls = Mutex::new(Vec::new()); - let error = post_both_targeted_transports( - || { - calls.lock().unwrap().push("skylight"); - false - }, - || { - calls.lock().unwrap().push("core_graphics"); - Ok(()) - }, - ) - .unwrap_err(); - - assert_eq!(*calls.lock().unwrap(), ["skylight", "core_graphics"]); - assert_eq!(error.code, ErrorCode::DispatchFailed); - assert_eq!(error.details["skylight_succeeded"], false); - assert_eq!(error.details["core_graphics_attempted"], true); - assert_eq!(error.details["core_graphics_call_completed"], true); } struct FakeSink { @@ -1457,7 +875,6 @@ mod tests { fn post( &self, event: &PreparedPointerEvent, - _gesture_id: i64, _boundary: &mut NativeSideEffectBoundary<'_>, epoch: Option<&PointerEpochWitness>, ) -> Result<(), NativeError> { @@ -1491,24 +908,11 @@ mod tests { } #[test] - fn partial_drag_posts_targeted_button_cleanup_without_cursor_warp() { - let sequence = build_gesture( - point(0.0, 0.0), - Some(point(64.0, 0.0)), - MouseButton::Left, - 1, - &[Modifier::Shift], - 0, - ) - .unwrap(); - let down_index = sequence - .events - .iter() - .position(|event| event.kind == PointerEventKind::Down) - .unwrap(); + fn failed_up_posts_targeted_button_cleanup_without_cursor_warp() { + let sequence = build_click(point(0.0, 0.0)); let sink = FakeSink { events: Mutex::new(Vec::new()), - fail_at: down_index + 1, + fail_at: 1, signal_after_first: None, }; let mut evidence = PointerDispatchEvidence { @@ -1558,7 +962,7 @@ mod tests { #[test] fn advanced_observation_epoch_refuses_before_the_first_pointer_post() { - let sequence = build_scroll(point(5.0, 6.0), 1.0, 0.0).unwrap(); + let sequence = build_click(point(5.0, 6.0)); let sink = FakeSink { events: Mutex::new(Vec::new()), fail_at: usize::MAX, @@ -1604,8 +1008,7 @@ mod tests { #[test] fn signal_from_first_post_does_not_abort_the_owned_pointer_sequence() { - let sequence = - build_gesture(point(10.0, 20.0), None, MouseButton::Left, 1, &[], 0).unwrap(); + let sequence = build_click(point(10.0, 20.0)); let journal = MacSignalJournal::default(); let sink = FakeSink { events: Mutex::new(Vec::new()), diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/focus.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/focus.rs index 6cfb14ee7..f59951da2 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/focus.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/focus.rs @@ -1,26 +1,32 @@ -//! Pure macOS interaction recipes and target-only SLPS make-key leases. +//! Exact macOS interaction recipes and durable synthetic app-focus belief. -use std::sync::{Arc, Mutex}; +use std::{ + sync::{Arc, Mutex}, + thread, + time::{Duration, Instant}, +}; +use core_foundation::base::{CFRelease, CFTypeRef}; use cua_driver_core::api::{ capabilities::Framework, - contracts::{ActionId, MouseButton, Route}, + contracts::{MouseButton, Point, Rect, Route}, errors::{ErrorCode, ErrorPhase, NativeError}, - interaction::ScopeRequirements, + interaction::{NativeEvidence, ScopeRequirements}, platform::ResolvedAction, }; -use crate::input::slps_make_key::{self, SlpsMakeKeyState}; +use crate::{ + apps, + ax::bindings::{copy_bool_attr, AXUIElementCreateApplication}, + input::synthesized_event, +}; -use super::target::{MacActiveBelief, MacFocusState}; +use super::{target::MacFocusState, windows::MacWindowFacts}; const PROVEN_OS_VERSION: &str = "26.5.1"; const PROVEN_ARCHITECTURE: &str = "arm64"; -// The live F1b recipe posted KeyFocusReturned followed by NewFront. Route B's -// recovered SLPS record has no subtype field, so those two grants are the same -// make-key byte shape but must still be posted twice, in order. -const PROVEN_CHROMIUM_GRANT_RECORDS: [SlpsMakeKeyState; 2] = - [SlpsMakeKeyState::MakeKey, SlpsMakeKeyState::MakeKey]; +const BELIEF_ACK_TIMEOUT: Duration = Duration::from_secs(2); +const BELIEF_ACK_POLL_INTERVAL: Duration = Duration::from_millis(10); #[derive(Debug, Clone, PartialEq, Eq)] pub struct HostRecipeContext { @@ -65,7 +71,7 @@ pub enum MenuSuppressionRecipe { #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TargetBeliefRecipe { NotApplicable, - ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64, + SwiftCoordinateClick, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -79,8 +85,8 @@ impl MacScopeRecipe { pub fn evidence_name(self) -> &'static str { match self.target_belief { TargetBeliefRecipe::NotApplicable => "semantic_no_target_belief", - TargetBeliefRecipe::ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64 => { - "chromium_point_click_two_slps_make_key_records_macos_26_5_1_arm64" + TargetBeliefRecipe::SwiftCoordinateClick => { + "swift_coordinate_click_cps_key_focus_returned_macos_26_5_1_arm64" } } } @@ -102,6 +108,15 @@ pub fn select_scope_recipe( "exact per-pid and per-menu event-tap predicate is not proved", )); } + if route == Route::TargetedPointer && *framework == Framework::Catalyst { + return Err(recipe_unproven( + host, + framework, + route, + action, + "the helper's Catalyst preparation and focus reassertion branch is not ported", + )); + } let accessibility = if requirements.accessibility && *framework == Framework::Chromium { AccessibilityRecipe::ChromiumPriorStatePreserving @@ -115,10 +130,9 @@ pub fn select_scope_recipe( if requirements.target_belief && host.os_version == PROVEN_OS_VERSION && host.architecture == PROVEN_ARCHITECTURE - && *framework == Framework::Chromium && is_exact_proven_point_click(action) => { - TargetBeliefRecipe::ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64 + TargetBeliefRecipe::SwiftCoordinateClick } _ => { return Err(recipe_unproven( @@ -126,7 +140,7 @@ pub fn select_scope_recipe( framework, route, action, - "no exact target-belief recipe is proved for this host/framework/action cell", + "no exact target-belief recipe is proved for this host/action cell", )); } }; @@ -179,161 +193,213 @@ fn recipe_unproven( .with_detail("action", action_shape(action)) } -pub(crate) trait TargetBeliefPoster: Send + Sync { - fn post(&self, pid: i32, window_id: u32, state: SlpsMakeKeyState) -> Result<(), NativeError>; +pub(crate) trait TargetFocusPoster: Send + Sync { + fn post_key_focus_returned(&self, pid: i32) -> Result<(), NativeError>; + fn post_app_activated( + &self, + pid: i32, + cg_window_id: u32, + window_bounds: Rect, + activation_point: Option, + ) -> Result; +} + +pub(crate) trait TargetFocusReader: Send + Sync { + fn application_is_active(&self, pid: i32) -> bool; + fn application_believes_frontmost(&self, pid: i32) -> Option; } #[derive(Default)] -pub(crate) struct SystemTargetBeliefPoster; - -impl TargetBeliefPoster for SystemTargetBeliefPoster { - fn post(&self, pid: i32, window_id: u32, state: SlpsMakeKeyState) -> Result<(), NativeError> { - slps_make_key::post_target_only(pid, window_id, &[state]).map_err(|error| { - NativeError::new( - ErrorCode::UnsupportedInBackground, - ErrorPhase::Preflight, - false, - format!("target-only SLPS make-key record failed: {error}"), - ) - .with_detail("pid", pid) - .with_detail("cg_window_id", window_id) - .with_detail("slps_record_state", state.evidence_name()) +pub(crate) struct SystemTargetFocusPoster; + +impl TargetFocusPoster for SystemTargetFocusPoster { + fn post_key_focus_returned(&self, pid: i32) -> Result<(), NativeError> { + synthesized_event::post_key_focus_returned(pid).map_err(|mut error| { + error.phase = ErrorPhase::Preflight; + error + .with_detail("pid", pid) + .with_detail("cps_notification", "key_focus_returned") }) } -} -pub(crate) struct TargetBeliefLease { - action_id: ActionId, - pid: i32, - window_id: u32, - state: Arc>, - poster: Arc, - released: bool, -} - -impl TargetBeliefLease { - pub fn acquire( - action_id: ActionId, + fn post_app_activated( + &self, pid: i32, - window_id: u32, - state: Arc>, - ) -> Result { - Self::acquire_with( - action_id, - pid, - window_id, - state, - Arc::new(SystemTargetBeliefPoster), - ) + cg_window_id: u32, + window_bounds: Rect, + activation_point: Option, + ) -> Result { + synthesized_event::post_app_activated(pid, cg_window_id, window_bounds, activation_point) + .map_err(|mut error| { + error.phase = ErrorPhase::Preflight; + error + .with_detail("pid", pid) + .with_detail("appkit_notification", "app_activated") + }) } +} - fn acquire_with( - action_id: ActionId, - pid: i32, - window_id: u32, - state: Arc>, - poster: Arc, - ) -> Result { - { - let mut state_guard = state.lock().expect("macOS focus coordinator poisoned"); - if state_guard.shutdown { - return Err(NativeError::new( - ErrorCode::WindowIdentityChanged, - ErrorPhase::Preflight, - false, - "target focus coordinator is shut down", - )); - } - if let Some(active) = &state_guard.active_belief { - return Err(NativeError::new( - ErrorCode::TargetBusy, - ErrorPhase::Preflight, - true, - "target already owns an active belief lease", - ) - .with_detail("active_action_id", active.action_id.to_string())); - } - state_guard.active_belief = Some(MacActiveBelief { - action_id: action_id.clone(), - pid, - cg_window_id: window_id, - }); - } +#[derive(Default)] +pub(crate) struct SystemTargetFocusReader; - let mut grants_posted = 0usize; - for grant in PROVEN_CHROMIUM_GRANT_RECORDS { - if let Err(error) = poster.post(pid, window_id, grant) { - let mut error = error - .with_detail("grant_records_posted", grants_posted) - .with_detail( - "grant_records_expected", - PROVEN_CHROMIUM_GRANT_RECORDS.len(), - ); - let mut rollback_complete = true; - if grants_posted > 0 { - if let Err(cleanup) = poster.post(pid, window_id, SlpsMakeKeyState::RemoveKey) { - rollback_complete = false; - error = error.with_related(&cleanup); - } - } - if rollback_complete { - state - .lock() - .expect("macOS focus coordinator poisoned") - .active_belief = None; - } - return Err(error); +impl TargetFocusReader for SystemTargetFocusReader { + fn application_is_active(&self, pid: i32) -> bool { + apps::frontmost_pid() == Some(pid) + } + + fn application_believes_frontmost(&self, pid: i32) -> Option { + unsafe { + let application = AXUIElementCreateApplication(pid); + if application.is_null() { + return None; } - grants_posted += 1; + let frontmost = copy_bool_attr(application, "AXFrontmost"); + CFRelease(application as CFTypeRef); + frontmost } + } +} - Ok(Self { - action_id, - pid, - window_id, +pub(crate) fn prepare_target_focus( + recipe: TargetBeliefRecipe, + facts: &MacWindowFacts, + state: Arc>, + deadline: Instant, +) -> Result, NativeError> { + match recipe { + TargetBeliefRecipe::NotApplicable => Ok(None), + TargetBeliefRecipe::SwiftCoordinateClick => prepare_target_focus_with( + facts, state, - poster, - released: false, - }) + deadline, + &SystemTargetFocusPoster, + &SystemTargetFocusReader, + ) + .map(Some), } +} - pub fn release(&mut self) -> Result<(), NativeError> { - if self.released { - return Ok(()); +fn prepare_target_focus_with( + facts: &MacWindowFacts, + state: Arc>, + deadline: Instant, + poster: &dyn TargetFocusPoster, + reader: &dyn TargetFocusReader, +) -> Result { + let pid = facts.pid; + let cg_window_id = facts.cg_window_id; + let application_is_active = reader.application_is_active(pid); + let (should_post_key_focus_returned, should_post_app_activated) = { + let mut state = state.lock().expect("macOS focus coordinator poisoned"); + if state.shutdown { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "target focus coordinator is shut down", + )); + } + if state.pid != pid || state.cg_window_id != cg_window_id { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "target focus coordinator identity no longer matches the interaction target", + )); + } + state.application_is_active = application_is_active; + if application_is_active { + state.application_believes_it_is_active = true; + state.application_believes_it_has_focus = true; } - self.poster - .post(self.pid, self.window_id, SlpsMakeKeyState::RemoveKey) - .map_err(|error| { - NativeError::new( - error.code, - ErrorPhase::Verify, - error.retryable, - error.message, - ) - .with_detail("action_id", self.action_id.to_string()) - .with_detail("pid", self.pid) - .with_detail("cg_window_id", self.window_id) - })?; - self.state - .lock() - .expect("macOS focus coordinator poisoned") - .active_belief = None; - self.released = true; - Ok(()) + ( + !state.application_is_active && !state.application_believes_it_has_focus, + !state.application_is_active && !state.application_believes_it_is_active, + ) + }; + + if should_post_key_focus_returned { + poster.post_key_focus_returned(pid)?; + let mut state = state.lock().expect("macOS focus coordinator poisoned"); + state.application_believes_it_has_focus = true; } -} + let app_activation_event_count = if should_post_app_activated { + let event_count = + poster.post_app_activated(pid, cg_window_id, facts.bounds, facts.activation_point)?; + let mut state = state.lock().expect("macOS focus coordinator poisoned"); + state.application_believes_it_is_active = true; + event_count + } else { + 0 + }; -impl Drop for TargetBeliefLease { - fn drop(&mut self) { - if let Err(error) = self.release() { - tracing::error!(error = %error, "failed to post target-only SLPS remove-key record"); + let ack_deadline = deadline.min(Instant::now() + BELIEF_ACK_TIMEOUT); + let mut ax_frontmost_acknowledged = application_is_active; + while !ax_frontmost_acknowledged && Instant::now() <= ack_deadline { + ax_frontmost_acknowledged = reader.application_believes_frontmost(pid) == Some(true); + if !ax_frontmost_acknowledged { + thread::sleep( + BELIEF_ACK_POLL_INTERVAL + .min(ack_deadline.saturating_duration_since(Instant::now())), + ); } } + if !ax_frontmost_acknowledged { + return Err(NativeError::new( + ErrorCode::PostureUnverifiable, + ErrorPhase::Preflight, + true, + "target did not acknowledge the synthetic focus belief through AXFrontmost", + ) + .with_detail("pid", pid) + .with_detail("cg_window_id", cg_window_id) + .with_detail( + "cps_key_focus_returned_posted", + should_post_key_focus_returned, + ) + .with_detail("app_activated_posted", should_post_app_activated) + .with_detail("app_activation_event_count", app_activation_event_count) + .with_detail( + "belief_ack_timeout_ms", + BELIEF_ACK_TIMEOUT.as_millis() as u64, + )); + } + + let mut evidence = NativeEvidence::default(); + evidence.fields.insert( + "target_application_was_active".to_owned(), + application_is_active.into(), + ); + evidence.fields.insert( + "target_focus_notification_posted".to_owned(), + should_post_key_focus_returned.into(), + ); + evidence.fields.insert( + "target_focus_notification".to_owned(), + "cps_key_focus_returned".into(), + ); + evidence.fields.insert( + "target_app_activation_posted".to_owned(), + should_post_app_activated.into(), + ); + evidence.fields.insert( + "target_app_activation_event_count".to_owned(), + app_activation_event_count.into(), + ); + evidence.fields.insert( + "target_ax_frontmost_acknowledged".to_owned(), + ax_frontmost_acknowledged.into(), + ); + evidence.fields.insert( + "target_focus_lifetime".to_owned(), + "target_controller".into(), + ); + evidence.fields.insert( + "target_focus_event_taps".to_owned(), + "process_notification,target_mouse,view_bridge_keyboard_when_present".into(), + ); + Ok(evidence) } #[cfg(test)] mod tests { - use std::sync::Mutex; + use std::{collections::VecDeque, sync::Mutex}; use cua_driver_core::api::{ contracts::{CaptureRevision, GeometryRevision, Point, Rect, SurfaceId, WindowGeneration}, @@ -347,34 +413,50 @@ mod tests { use super::*; struct FakePoster { - signals: Mutex>, - fail_at: Vec, + posts: Mutex>, } - impl TargetBeliefPoster for FakePoster { - fn post( + impl TargetFocusPoster for FakePoster { + fn post_key_focus_returned(&self, pid: i32) -> Result<(), NativeError> { + self.posts.lock().unwrap().push(("key_focus_returned", pid)); + Ok(()) + } + + fn post_app_activated( &self, - _pid: i32, - _window_id: u32, - signal: SlpsMakeKeyState, - ) -> Result<(), NativeError> { - let mut signals = self.signals.lock().unwrap(); - let index = signals.len(); - signals.push(signal); - if self.fail_at.contains(&index) { - Err(NativeError::new( - ErrorCode::Internal, - ErrorPhase::Preflight, - false, - "injected SLPS failure", - )) - } else { - Ok(()) - } + pid: i32, + cg_window_id: u32, + window_bounds: Rect, + activation_point: Option, + ) -> Result { + assert_eq!(cg_window_id, 99); + assert_eq!(window_bounds.width, 10.0); + assert_eq!(activation_point, Some(Point { x: 1.0, y: 1.0 })); + self.posts.lock().unwrap().push(("app_activated", pid)); + Ok(3) } } - fn point_click(button: MouseButton, count: u8) -> ResolvedAction { + struct FakeReader { + active: bool, + frontmost: Mutex>>, + } + + impl TargetFocusReader for FakeReader { + fn application_is_active(&self, _pid: i32) -> bool { + self.active + } + + fn application_believes_frontmost(&self, _pid: i32) -> Option { + self.frontmost + .lock() + .unwrap() + .pop_front() + .unwrap_or(Some(true)) + } + } + + fn point_click(framework: Framework, button: MouseButton, count: u8) -> ResolvedAction { let app = cua_driver_core::api::contracts::AppRef { id: cua_driver_core::api::contracts::AppId::parse("app").unwrap(), name: None, @@ -390,7 +472,7 @@ mod tests { public, native: NativeWindowHandle::new("native").unwrap(), process: NativeProcessHandle::new("process").unwrap(), - framework: Framework::Chromium, + framework, geometry: WindowGeometry { bounds: Rect { x: 0.0, @@ -426,177 +508,106 @@ mod tests { } } + fn focus_facts() -> MacWindowFacts { + let ResolvedAction::PointClick { point, .. } = + point_click(Framework::Unknown, MouseButton::Left, 1) + else { + unreachable!("fixture is a point click"); + }; + MacWindowFacts { + stamp: point.window.stamp(), + pid: 44, + process_generation: 1, + cg_window_id: 99, + owner_name: "Fixture".to_owned(), + layer: 0, + bounds: point.window.geometry.bounds, + activation_point: Some(Point { x: 1.0, y: 1.0 }), + scale_factor: Some(1.0), + state: cua_driver_core::api::capabilities::WindowStateKind::Visible, + is_on_screen: true, + on_current_space: Some(true), + space_ids: Some(vec![1]), + minimized: Some(false), + } + } + #[test] - fn recipe_selector_accepts_only_the_proven_pointer_cell() { + fn recipe_selector_uses_exact_host_and_action_facts_not_framework_guessing() { let host = HostRecipeContext { os_version: PROVEN_OS_VERSION.to_owned(), architecture: PROVEN_ARCHITECTURE.to_owned(), }; let requirements = ScopeRequirements::for_route(Route::TargetedPointer); - let recipe = select_scope_recipe( - &host, - &Framework::Chromium, - Route::TargetedPointer, - &point_click(MouseButton::Left, 1), - &requirements, - ) - .unwrap(); - assert_eq!( - recipe.target_belief, - TargetBeliefRecipe::ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64 - ); - - for (host, framework, action) in [ - ( - HostRecipeContext { - os_version: "26.5.2".to_owned(), - architecture: "arm64".to_owned(), - }, - Framework::Chromium, - point_click(MouseButton::Left, 1), - ), - ( - HostRecipeContext { - os_version: PROVEN_OS_VERSION.to_owned(), - architecture: "x86_64".to_owned(), - }, - Framework::Chromium, - point_click(MouseButton::Left, 1), - ), - ( - host.clone(), - Framework::Electron, - point_click(MouseButton::Left, 1), - ), - ( - host.clone(), - Framework::Chromium, - point_click(MouseButton::Right, 1), - ), - ] { - let error = select_scope_recipe( + for framework in [Framework::Unknown, Framework::Chromium, Framework::Electron] { + let recipe = select_scope_recipe( &host, &framework, Route::TargetedPointer, - &action, + &point_click(framework.clone(), MouseButton::Left, 1), &requirements, ) - .unwrap_err(); - assert_eq!(error.code, ErrorCode::UnsupportedInBackground); - assert_eq!(error.details["recipe_status"], "recipe_unproven"); + .unwrap(); + assert_eq!( + recipe.target_belief, + TargetBeliefRecipe::SwiftCoordinateClick + ); } - } - - #[test] - fn target_belief_grant_and_revoke_are_target_only_and_stateful() { - let state = Arc::new(Mutex::new(MacFocusState::default())); - let poster = Arc::new(FakePoster { - signals: Mutex::new(Vec::new()), - fail_at: Vec::new(), - }); - let mut lease = TargetBeliefLease::acquire_with( - ActionId::parse("action").unwrap(), - 44, - 99, - Arc::clone(&state), - poster.clone(), + let catalyst_error = select_scope_recipe( + &host, + &Framework::Catalyst, + Route::TargetedPointer, + &point_click(Framework::Catalyst, MouseButton::Left, 1), + &requirements, ) - .unwrap(); - assert_eq!( - poster.signals.lock().unwrap().as_slice(), - [SlpsMakeKeyState::MakeKey, SlpsMakeKeyState::MakeKey] - ); - assert!(state.lock().unwrap().active_belief.is_some()); - lease.release().unwrap(); - assert_eq!( - poster.signals.lock().unwrap().as_slice(), - [ - SlpsMakeKeyState::MakeKey, - SlpsMakeKeyState::MakeKey, - SlpsMakeKeyState::RemoveKey - ] - ); - assert!(state.lock().unwrap().active_belief.is_none()); + .unwrap_err(); + assert_eq!(catalyst_error.code, ErrorCode::UnsupportedInBackground); + let error = select_scope_recipe( + &host, + &Framework::Unknown, + Route::TargetedPointer, + &point_click(Framework::Unknown, MouseButton::Right, 1), + &requirements, + ) + .unwrap_err(); + assert_eq!(error.code, ErrorCode::UnsupportedInBackground); } #[test] - fn failed_make_key_record_clears_controller_ownership_without_claiming_a_grant() { - let state = Arc::new(Mutex::new(MacFocusState::default())); - let poster = Arc::new(FakePoster { - signals: Mutex::new(Vec::new()), - fail_at: vec![0], - }); - let error = TargetBeliefLease::acquire_with( - ActionId::parse("action").unwrap(), - 44, - 99, + fn focus_belief_posts_once_and_remains_owned_by_the_target_controller() { + let facts = focus_facts(); + let state = Arc::new(Mutex::new(MacFocusState::new(44, 99))); + let poster = FakePoster { + posts: Mutex::new(Vec::new()), + }; + let reader = FakeReader { + active: false, + frontmost: Mutex::new(VecDeque::from([Some(false), Some(true)])), + }; + prepare_target_focus_with( + &facts, Arc::clone(&state), - poster.clone(), + Instant::now() + Duration::from_secs(1), + &poster, + &reader, ) - .err() .unwrap(); - assert_eq!(error.phase, ErrorPhase::Preflight); - assert_eq!( - poster.signals.lock().unwrap().as_slice(), - [SlpsMakeKeyState::MakeKey] - ); - assert!(state.lock().unwrap().active_belief.is_none()); - } - - #[test] - fn second_grant_failure_posts_paired_revoke_and_clears_ownership() { - let state = Arc::new(Mutex::new(MacFocusState::default())); - let poster = Arc::new(FakePoster { - signals: Mutex::new(Vec::new()), - fail_at: vec![1], - }); - let error = TargetBeliefLease::acquire_with( - ActionId::parse("action").unwrap(), - 44, - 99, + prepare_target_focus_with( + &facts, Arc::clone(&state), - poster.clone(), + Instant::now() + Duration::from_secs(1), + &poster, + &reader, ) - .err() - .expect("second grant failure must refuse the lease"); - assert_eq!(error.phase, ErrorPhase::Preflight); - assert_eq!(error.details["grant_records_posted"], 1); - assert_eq!( - poster.signals.lock().unwrap().as_slice(), - [ - SlpsMakeKeyState::MakeKey, - SlpsMakeKeyState::MakeKey, - SlpsMakeKeyState::RemoveKey - ] - ); - assert!(state.lock().unwrap().active_belief.is_none()); - } + .unwrap(); - #[test] - fn failed_partial_grant_rollback_keeps_ownership_for_poison_and_shutdown_retry() { - let state = Arc::new(Mutex::new(MacFocusState::default())); - let poster = Arc::new(FakePoster { - signals: Mutex::new(Vec::new()), - fail_at: vec![1, 2], - }); - let error = TargetBeliefLease::acquire_with( - ActionId::parse("action").unwrap(), - 44, - 99, - Arc::clone(&state), - poster.clone(), - ) - .err() - .expect("grant and rollback failure must refuse the lease"); - assert_eq!(error.related_failures.len(), 1); assert_eq!( - poster.signals.lock().unwrap().as_slice(), - [ - SlpsMakeKeyState::MakeKey, - SlpsMakeKeyState::MakeKey, - SlpsMakeKeyState::RemoveKey - ] + *poster.posts.lock().unwrap(), + [("key_focus_returned", 44), ("app_activated", 44)] ); - assert!(state.lock().unwrap().active_belief.is_some()); + let state = state.lock().unwrap(); + assert!(state.application_believes_it_is_active); + assert!(state.application_believes_it_has_focus); + assert!(!state.application_is_active); } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/interaction.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/interaction.rs index eb21801dd..53d22dd5f 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/interaction.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/interaction.rs @@ -25,13 +25,13 @@ use crate::{ apps::nsworkspace::WorkspaceEventHub, ax::enablement::AxEnablementLease, focus_steal::{SuppressionLease, SuppressionOutcome}, - input::{keyboard::normalize_chord, slps_make_key}, + input::keyboard::normalize_chord, }; use super::{ focus::{ - select_scope_recipe, AccessibilityRecipe, HostRecipeContext, MacScopeRecipe, - MenuSuppressionRecipe, TargetBeliefLease, TargetBeliefRecipe, + prepare_target_focus, select_scope_recipe, AccessibilityRecipe, HostRecipeContext, + MacScopeRecipe, MenuSuppressionRecipe, TargetBeliefRecipe, }, menu::{self, MenuSuppressionPlan}, posture::MacInteractionPostureWitness, @@ -39,9 +39,6 @@ use super::{ windows::{MacWindowFacts, MacWindowRegistry}, }; -#[cfg(test)] -use super::target::MacActiveBelief; - const CONTAINMENT_BARRIER_TIMEOUT: Duration = Duration::from_millis(250); #[derive(Debug)] @@ -95,14 +92,13 @@ trait MacInteractionHooks: Send + Sync { plan: &MenuSuppressionPlan, ) -> Result>, NativeError>; - fn acquire_target_belief( + fn prepare_target_belief( &self, recipe: TargetBeliefRecipe, - action_id: &ActionId, - pid: i32, - cg_window_id: u32, + facts: &MacWindowFacts, focus_state: Arc>, - ) -> Result>, NativeError>; + deadline: Instant, + ) -> Result, NativeError>; } #[derive(Default)] @@ -163,22 +159,14 @@ impl MacInteractionHooks for SystemInteractionHooks { }) } - fn acquire_target_belief( + fn prepare_target_belief( &self, recipe: TargetBeliefRecipe, - action_id: &ActionId, - pid: i32, - cg_window_id: u32, + facts: &MacWindowFacts, focus_state: Arc>, - ) -> Result>, NativeError> { - match recipe { - TargetBeliefRecipe::NotApplicable => Ok(None), - TargetBeliefRecipe::ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64 => { - Ok(Some(Box::new(BeliefLeaseResource( - TargetBeliefLease::acquire(action_id.clone(), pid, cg_window_id, focus_state)?, - )))) - } - } + deadline: Instant, + ) -> Result, NativeError> { + prepare_target_focus(recipe, facts, focus_state, deadline) } } @@ -307,22 +295,6 @@ impl LeaseResource for MenuResourceAdapter { } } -struct BeliefLeaseResource(TargetBeliefLease); - -impl LeaseResource for BeliefLeaseResource { - fn release(&mut self, _deadline: Instant) -> LeaseRelease { - let mut evidence = NativeEvidence::default(); - let failure = self.0.release().err(); - evidence - .fields - .insert("target_belief_release_attempted".to_owned(), true.into()); - evidence - .fields - .insert("target_belief_revoked".to_owned(), failure.is_none().into()); - LeaseRelease { evidence, failure } - } -} - fn remaining(deadline: Instant) -> Duration { deadline .saturating_duration_since(Instant::now()) @@ -371,15 +343,6 @@ impl InteractionProvider for MacInter } let recipe = select_scope_recipe(&self.host, &window.framework, route, action, &requirements)?; - if recipe.target_belief != TargetBeliefRecipe::NotApplicable && !slps_make_key::available() - { - return Err(NativeError::unsupported( - "recipe_unproven: target-only SLPS make-key symbols are unavailable on this host", - ) - .with_detail("recipe_status", "recipe_unproven") - .with_detail("os_version", self.host.os_version.clone()) - .with_detail("architecture", self.host.architecture.clone())); - } let menu = if requirements.menu_dismissal { return Err(NativeError::unsupported( "recipe_unproven: required menu suppression has no exact predicate", @@ -477,6 +440,7 @@ fn acquire_resources( ) -> Result { let posture = hooks.acquire_posture()?; let mut cleanup = MacScopeCleanup::new(poison, posture.resource, plan.deadline.teardown); + let mut target_belief_evidence = None; let result = (|| { cleanup.accessibility = @@ -488,36 +452,21 @@ fn acquire_resources( plan.deadline.teardown, )?; cleanup.menu = hooks.acquire_menu(plan.native.recipe.menu, &plan.native.menu)?; - cleanup.target_belief = hooks.acquire_target_belief( + target_belief_evidence = hooks.prepare_target_belief( plan.native.recipe.target_belief, - &plan.action_id, - plan.native.facts.pid, - plan.native.facts.cg_window_id, + &plan.native.facts, Arc::clone(&focus_state), + plan.deadline.work, )?; Ok::<(), NativeError>(()) })(); if let Err(primary) = result { let outcome = cleanup.cleanup(plan.deadline.teardown); - let belief_still_active = focus_state - .lock() - .expect("macOS focus coordinator poisoned") - .active_belief - .is_some(); - if belief_still_active { - cleanup.poison.store(true, Ordering::Release); - } let mut failures = Vec::with_capacity(1 + outcome.failures.len()); failures.push(primary); failures.extend(outcome.failures); - let mut combined = NativeError::primary(failures).expect("acquisition failure is nonempty"); - if belief_still_active { - combined = combined - .with_detail("target_poisoned", true) - .with_target_invalidated(); - } - return Err(combined); + return Err(NativeError::primary(failures).expect("acquisition failure is nonempty")); } let acquisition = ScopeLeaseAcquisition { @@ -525,7 +474,11 @@ fn acquire_resources( accessibility: decision(&cleanup.accessibility), containment: decision(&cleanup.containment), menu_dismissal: decision(&cleanup.menu), - target_belief: decision(&cleanup.target_belief), + target_belief: if target_belief_evidence.is_some() { + LeaseDecision::Acquired + } else { + LeaseDecision::NotApplicable + }, }; let mut evidence = NativeEvidence::default(); evidence.fields.insert( @@ -547,6 +500,9 @@ fn acquire_resources( "cg_window_id".to_owned(), plan.native.facts.cg_window_id.into(), ); + if let Some(target_belief_evidence) = target_belief_evidence { + evidence.merge(target_belief_evidence); + } Ok(AcquiredResources { acquisition, evidence, @@ -569,7 +525,6 @@ struct MacScopeCleanup { accessibility: Option>, containment: Option>, menu: Option>, - target_belief: Option>, outcome: Option, } @@ -586,7 +541,6 @@ impl MacScopeCleanup { accessibility: None, containment: None, menu: None, - target_belief: None, outcome: None, } } @@ -600,12 +554,6 @@ impl ScopeCleanup for MacScopeCleanup { let mut evidence = NativeEvidence::default(); let mut failures = Vec::new(); - let target_belief = release_lease( - &mut self.target_belief, - deadline, - &mut evidence, - &mut failures, - ); let menu_dismissal = release_lease(&mut self.menu, deadline, &mut evidence, &mut failures); let containment = release_lease( &mut self.containment, @@ -658,7 +606,7 @@ impl ScopeCleanup for MacScopeCleanup { accessibility, containment, menu_dismissal, - target_belief, + target_belief: LeaseTeardownStatus::NotApplicable, }, failures, }; @@ -792,7 +740,6 @@ mod tests { log: Arc>>, containment_deadlines: Arc>>, fail_at: Option<&'static str>, - poison_belief: bool, } impl LoggingHooks { @@ -854,31 +801,23 @@ mod tests { self.acquire("menu+", "menu-") } - fn acquire_target_belief( + fn prepare_target_belief( &self, _recipe: TargetBeliefRecipe, - _action_id: &ActionId, - _pid: i32, - _cg_window_id: u32, - focus_state: Arc>, - ) -> Result>, NativeError> { - if self.poison_belief { - focus_state - .lock() - .expect("macOS focus coordinator poisoned") - .active_belief = Some(MacActiveBelief { - action_id: ActionId::parse("partial-belief").unwrap(), - pid: 44, - cg_window_id: 99, - }); + _facts: &MacWindowFacts, + _focus_state: Arc>, + _deadline: Instant, + ) -> Result, NativeError> { + self.log.lock().unwrap().push("belief+"); + if self.fail_at == Some("belief+") { return Err(NativeError::new( ErrorCode::Internal, ErrorPhase::Preflight, false, - "injected partial belief rollback failure", + "injected belief preparation failure", )); } - self.acquire("belief+", "belief-") + Ok(Some(NativeEvidence::default())) } } @@ -920,6 +859,7 @@ mod tests { owner_name: "Fixture".to_owned(), layer: 0, bounds: window.geometry.bounds, + activation_point: Some(cua_driver_core::api::contracts::Point { x: 10.0, y: 16.0 }), scale_factor: Some(2.0), state: WindowStateKind::Visible, is_on_screen: true, @@ -942,7 +882,7 @@ mod tests { recipe: MacScopeRecipe { accessibility: AccessibilityRecipe::ChromiumPriorStatePreserving, menu: MenuSuppressionRecipe::NotApplicable, - target_belief: TargetBeliefRecipe::ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64, + target_belief: TargetBeliefRecipe::SwiftCoordinateClick, }, menu: MenuSuppressionPlan::NotApplicable, }, @@ -961,7 +901,6 @@ mod tests { log: Arc::clone(&log), containment_deadlines: Arc::new(Mutex::new(Vec::new())), fail_at: None, - poison_belief: false, }; let poison = Arc::new(AtomicBool::new(false)); let plan = scope_plan(); @@ -969,7 +908,7 @@ mod tests { &hooks, &plan, poison, - Arc::new(Mutex::new(MacFocusState::default())), + Arc::new(Mutex::new(MacFocusState::new(44, 99))), ) .unwrap(); assert_eq!( @@ -986,7 +925,6 @@ mod tests { "containment+", "menu+", "belief+", - "belief-", "menu-", "containment-", "ax-", @@ -1035,13 +973,12 @@ mod tests { log: Arc::clone(&log), containment_deadlines: Arc::new(Mutex::new(Vec::new())), fail_at: Some(fail_at), - poison_belief: false, }; let error = acquire_resources( &hooks, &scope_plan(), Arc::new(AtomicBool::new(false)), - Arc::new(Mutex::new(MacFocusState::default())), + Arc::new(Mutex::new(MacFocusState::new(44, 99))), ) .err() .unwrap(); @@ -1050,36 +987,6 @@ mod tests { } } - #[test] - fn partial_target_belief_rollback_emits_typed_core_invalidation_signal() { - let log = Arc::new(Mutex::new(Vec::new())); - let hooks = LoggingHooks { - log, - containment_deadlines: Arc::new(Mutex::new(Vec::new())), - fail_at: None, - poison_belief: true, - }; - let poison = Arc::new(AtomicBool::new(false)); - let focus_state = Arc::new(Mutex::new(MacFocusState::default())); - let error = acquire_resources( - &hooks, - &scope_plan(), - Arc::clone(&poison), - Arc::clone(&focus_state), - ) - .err() - .expect("partial belief rollback must refuse acquisition"); - - assert!(error.target_invalidated()); - assert_eq!(error.details["target_poisoned"], true); - assert!(poison.load(Ordering::Acquire)); - assert!(focus_state - .lock() - .expect("macOS focus coordinator poisoned") - .active_belief - .is_some()); - } - #[test] fn cleanup_is_reverse_order_and_restored_violation_poison_is_sticky() { let log = Arc::new(Mutex::new(Vec::new())); @@ -1101,12 +1008,11 @@ mod tests { cleanup.accessibility = Some(logged(&log, "ax-")); cleanup.containment = Some(logged(&log, "containment-")); cleanup.menu = Some(logged(&log, "menu-")); - cleanup.target_belief = Some(logged(&log, "belief-")); let outcome = cleanup.cleanup(test_deadline().teardown); assert_eq!( *log.lock().unwrap(), - vec!["belief-", "menu-", "containment-", "ax-", "posture-"] + vec!["menu-", "containment-", "ax-", "posture-"] ); assert!(outcome.posture.restored_after_violation); assert_eq!(outcome.failures[0].code, ErrorCode::PostureViolated); @@ -1125,9 +1031,9 @@ mod tests { }), test_deadline().teardown, ); - cleanup.target_belief = Some(Box::new(LoggedLease { + cleanup.menu = Some(Box::new(LoggedLease { log: Arc::clone(&log), - release: "belief-", + release: "menu-", fail: true, })); cleanup.containment = Some(logged(&log, "containment-")); @@ -1136,11 +1042,15 @@ mod tests { let outcome = cleanup.cleanup(test_deadline().teardown); assert_eq!( *log.lock().unwrap(), - vec!["belief-", "containment-", "ax-", "posture-"] + vec!["menu-", "containment-", "ax-", "posture-"] + ); + assert_eq!(outcome.leases.menu_dismissal, LeaseTeardownStatus::Failed); + assert_eq!( + outcome.leases.target_belief, + LeaseTeardownStatus::NotApplicable ); - assert_eq!(outcome.leases.target_belief, LeaseTeardownStatus::Failed); assert_eq!(outcome.leases.containment, LeaseTeardownStatus::Released); - assert_eq!(outcome.native_evidence.fields["belief-_attempted"], true); + assert_eq!(outcome.native_evidence.fields["menu-_attempted"], true); assert!(poison.load(Ordering::Acquire)); } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/lifecycle.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/lifecycle.rs index c843ecbcc..e867842c3 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/lifecycle.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/lifecycle.rs @@ -246,46 +246,53 @@ impl LifecycleProvider for MacLifecycle { ], Vec::new(), )); - match nsworkspace::running_application(pid) { - None => Err(launch_error( - "NSWorkspace returned an application that is not in the running-app registry", - ) - .with_detail("pid", pid)), - Some(application) => match application.process_generation { + match nsworkspace::process_generation(pid) { None => Err(launch_error( - "NSWorkspace returned an application without an exact process generation", + "NSWorkspace returned a pid without an exact live process generation", ) .with_detail("pid", pid)), - Some(generation) => { - let app = app_ref_for_running(&application); - posture_scope.record_partial_result(app.clone(), Vec::new()); - wait_for_stable_windows( - &self.windows, - LaunchWait { - app: &app, - pid, - expected_generation: generation, - baseline_windows: &baseline_windows, - dispatched: true, - settlement_action_id: &settlement_action_id, - profile: "macos_background_launch", - started, - deadline, - }, - &mut events, - &mut witness, - posture_scope, - ) - .await - .map(|stable| LaunchAttempt { - reused_running_app: preexisting - .as_ref() - .is_some_and(|prior| same_process_identity(prior, &application)), - app, - stable, - }) - } - }, + Some(generation) => match wait_for_registered_application( + pid, + generation, + &resolved, + deadline, + &mut events, + nsworkspace::running_application, + ) + .await + { + Err(error) => Err(error), + Ok(application) => { + witness.drain_events(); + let app = app_ref_for_running(&application); + posture_scope.record_partial_result(app.clone(), Vec::new()); + wait_for_stable_windows( + &self.windows, + LaunchWait { + app: &app, + pid, + expected_generation: generation, + baseline_windows: &baseline_windows, + dispatched: true, + settlement_action_id: &settlement_action_id, + profile: "macos_background_launch", + started, + deadline, + }, + &mut events, + &mut witness, + posture_scope, + ) + .await + .map(|stable| LaunchAttempt { + reused_running_app: preexisting.as_ref().is_some_and(|prior| { + same_process_identity(prior, &application) + }), + app, + stable, + }) + } + }, } } }, @@ -355,6 +362,82 @@ struct LaunchWait<'a> { deadline: Instant, } +async fn wait_for_registered_application( + pid: i32, + expected_generation: u64, + resolved: &ResolvedLaunch, + deadline: Instant, + events: &mut tokio::sync::broadcast::Receiver, + mut lookup: F, +) -> Result +where + F: FnMut(i32) -> Option, +{ + let mut poll = tokio::time::interval(Duration::from_millis(25)); + poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + loop { + if let Some(application) = lookup(pid) { + if let Some(false) = resolved.compares_to(&application) { + return Err(launch_settle_error( + "NSWorkspace launch pid resolved to a different application", + ) + .with_detail("pid", pid) + .with_detail( + "actual_bundle_id", + application.bundle_id.clone().unwrap_or_default(), + )); + } + match application.process_generation { + Some(generation) if generation != expected_generation => { + return Err(NativeError::new( + ErrorCode::WindowIdentityChanged, + ErrorPhase::Settle, + true, + "launched process identity changed before registration completed", + ) + .with_detail("pid", pid) + .with_detail("expected_process_generation", expected_generation) + .with_detail("current_process_generation", generation)); + } + Some(_) + if application.finished_launching + && resolved.compares_to(&application) == Some(true) => + { + return Ok(application) + } + Some(_) | None => {} + } + } + if Instant::now() >= deadline { + return Err(launch_settle_error( + "launched app did not register and finish launching before its deadline", + ) + .with_detail("pid", pid)); + } + tokio::select! { + _ = poll.tick() => {} + event = events.recv() => match event { + Ok(event) => { + if let Some(error) = launch_event_failure(pid, &event) { + return Err(error); + } + } + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {} + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + return Err(launch_settle_error( + "workspace lifecycle event stream closed before app registration", + ) + .with_detail("pid", pid)); + } + }, + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(deadline)) => { + return Err(launch_settle_error("app launch deadline elapsed before registration") + .with_detail("pid", pid)); + } + } + } +} + async fn wait_for_stable_windows( registry: &MacWindowRegistry, wait: LaunchWait<'_>, @@ -367,7 +450,7 @@ async fn wait_for_stable_windows( let mut stability = LaunchWindowStability::new(Instant::now()); loop { if Instant::now() >= wait.deadline { - return Err(launch_error( + return Err(launch_settle_error( "app launch did not reach a stable window set before its deadline", ) .with_detail("pid", wait.pid)); @@ -386,22 +469,22 @@ async fn wait_for_stable_windows( // the conservative resynchronization boundary. Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {} Err(tokio::sync::broadcast::error::RecvError::Closed) => { - return Err(launch_error("workspace lifecycle event stream closed during launch")); + return Err(launch_settle_error("workspace lifecycle event stream closed during launch")); } } } _ = tokio::time::sleep_until(tokio::time::Instant::from_std(wait.deadline)) => { - return Err(launch_error("app launch deadline elapsed") + return Err(launch_settle_error("app launch deadline elapsed") .with_detail("pid", wait.pid)); } } witness.drain_events(); let application = nsworkspace::running_application(wait.pid).ok_or_else(|| { - launch_error("app disappeared before launch settlement completed") + launch_settle_error("app disappeared before launch settlement completed") .with_detail("pid", wait.pid) })?; if application.process_generation != Some(wait.expected_generation) { - return Err(launch_error( + return Err(launch_settle_error( "process identity changed before launch settlement completed", ) .with_detail("pid", wait.pid) @@ -554,7 +637,8 @@ impl LaunchWindowStability { fn launch_event_failure(pid: i32, event: &nsworkspace::WorkspaceEvent) -> Option { (event.pid == pid && event.kind == WorkspaceEventKind::Terminated).then(|| { - launch_error("app terminated before launch settlement completed").with_detail("pid", pid) + launch_settle_error("app terminated before launch settlement completed") + .with_detail("pid", pid) }) } @@ -568,23 +652,32 @@ struct ResolvedLaunch { } impl ResolvedLaunch { - fn matches(&self, application: &RunningApplicationInfo) -> bool { + /// Compare only identity fields that NSRunningApplication has published. + /// `None` means registry publication is still incomplete, not a mismatch. + fn compares_to(&self, application: &RunningApplicationInfo) -> Option { if let Some(bundle_id) = &self.bundle_id { - return application.bundle_id.as_ref() == Some(bundle_id); + return application + .bundle_id + .as_ref() + .map(|candidate| candidate == bundle_id); } if let Some(executable) = &self.executable_path { return application .executable_path .as_ref() - .is_some_and(|path| Path::new(path) == executable); + .map(|path| Path::new(path) == executable); } - self.name.as_ref().is_some_and(|name| { + self.name.as_ref().and_then(|name| { application .name .as_ref() - .is_some_and(|candidate| candidate.eq_ignore_ascii_case(name)) + .map(|candidate| candidate.eq_ignore_ascii_case(name)) }) } + + fn matches(&self, application: &RunningApplicationInfo) -> bool { + self.compares_to(application) == Some(true) + } } fn reusable_existing_launch( @@ -618,9 +711,69 @@ impl LaunchCatalog for SystemLaunchCatalog { } fn resolve_launch(selector: AppSelector) -> Result { + if let AppSelector::Name { name } = &selector { + if name.trim().is_empty() { + return Err(NativeError::invalid("app name cannot be empty")); + } + if let Some(resolved) = resolve_running_name(name, &nsworkspace::running_applications())? { + return Ok(resolved); + } + } resolve_launch_with_catalog(selector, &SystemLaunchCatalog) } +fn resolve_running_name( + name: &str, + applications: &[RunningApplicationInfo], +) -> Result, NativeError> { + let matches: Vec<_> = applications + .iter() + .filter(|application| { + application + .name + .as_ref() + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(name)) + }) + .collect(); + if matches.len() > 1 { + return Err(NativeError::invalid( + "app name matched multiple running macOS processes; use a bundle id or executable", + ) + .with_detail("name", name.to_owned()) + .with_detail( + "pids", + matches + .iter() + .map(|application| application.pid) + .collect::>(), + )); + } + let Some(application) = matches.first() else { + return Ok(None); + }; + let app_ref = application + .bundle_id + .clone() + .or_else(|| application.executable_path.clone()) + .ok_or_else(|| { + NativeError::new( + ErrorCode::AppNotFound, + ErrorPhase::Preflight, + false, + "running macOS app has neither a bundle id nor executable path", + ) + .with_detail("name", name.to_owned()) + .with_detail("pid", application.pid) + })?; + Ok(Some(ResolvedLaunch { + app_ref, + bundle_id: application.bundle_id.clone(), + executable_path: application.executable_path.as_deref().map(PathBuf::from), + name: Some(name.to_owned()), + arguments: Vec::new(), + })) +} + fn resolve_launch_with_catalog( selector: AppSelector, catalog: &dyn LaunchCatalog, @@ -841,6 +994,15 @@ fn launch_error(message: impl Into) -> NativeError { ) } +fn launch_settle_error(message: impl Into) -> NativeError { + NativeError::new( + ErrorCode::AppLaunchFailed, + ErrorPhase::Settle, + true, + message, + ) +} + fn join_error(error: tokio::task::JoinError) -> NativeError { NativeError::new( ErrorCode::Internal, @@ -1024,6 +1186,64 @@ mod tests { assert!(!same_process_identity(&prior, &same)); } + #[test] + fn running_name_resolution_handles_core_services_and_rejects_ambiguity() { + let mut finder = running_fixture(); + finder.name = Some("Finder".to_owned()); + finder.bundle_id = Some("com.apple.finder".to_owned()); + finder.executable_path = + Some("/System/Library/CoreServices/Finder.app/Contents/MacOS/Finder".to_owned()); + let resolved = resolve_running_name("finder", &[finder.clone()]) + .unwrap() + .unwrap(); + assert_eq!(resolved.app_ref, "com.apple.finder"); + assert_eq!(resolved.bundle_id.as_deref(), Some("com.apple.finder")); + + let mut duplicate = finder.clone(); + duplicate.pid += 1; + duplicate.process_generation = Some(8); + let error = resolve_running_name("Finder", &[finder, duplicate]).unwrap_err(); + assert_eq!(error.code, ErrorCode::InvalidRequest); + assert_eq!(error.phase, ErrorPhase::Validate); + } + + #[tokio::test] + async fn cold_launch_waits_for_registry_publication_and_finished_state() { + let resolved = ResolvedLaunch { + app_ref: "com.example.fixture".to_owned(), + bundle_id: Some("com.example.fixture".to_owned()), + executable_path: None, + name: None, + arguments: Vec::new(), + }; + let (_sender, mut events) = tokio::sync::broadcast::channel(4); + let mut lookups = 0; + let application = wait_for_registered_application( + 120, + 7, + &resolved, + Instant::now() + Duration::from_millis(250), + &mut events, + move |_| { + lookups += 1; + match lookups { + 1 => None, + 2 => { + let mut application = running_fixture(); + application.bundle_id = None; + application.finished_launching = false; + Some(application) + } + _ => Some(running_fixture()), + } + }, + ) + .await + .unwrap(); + assert!(application.finished_launching); + assert_eq!(application.process_generation, Some(7)); + } + #[test] fn settlement_reports_only_signals_that_were_observed() { let started = Instant::now(); @@ -1089,7 +1309,7 @@ mod tests { }; let error = launch_event_failure(120, &target).unwrap(); assert_eq!(error.code, ErrorCode::AppLaunchFailed); - assert_eq!(error.phase, ErrorPhase::Dispatch); + assert_eq!(error.phase, ErrorPhase::Settle); } #[test] diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/mod.rs index 2c86ee2c6..387183a9b 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/mod.rs @@ -99,7 +99,7 @@ impl PlatformDriver for MacDriver { facts.cg_window_id, self.invalidations.clone(), )?, - MacTargetFocusCoordinator::default(), + MacTargetFocusCoordinator::new(facts.pid, facts.cg_window_id)?, )) } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/observation.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/observation.rs index 719388227..5b3c4f646 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/observation.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/observation.rs @@ -48,6 +48,7 @@ use super::{ const MAX_AX_ELEMENTS: usize = 2_000; const MAX_AX_DEPTH: usize = 25; const OBSERVATION_TIMEOUT: Duration = Duration::from_secs(5); +const SCK_CAPTURE_TEARDOWN_RESERVE: Duration = Duration::from_millis(100); static SCK_CAPTURE_SLOTS: Semaphore = Semaphore::const_new(2); #[derive(Clone)] @@ -74,42 +75,58 @@ impl MacObservationProvider { target: &mut MacTargetState, deadline: Instant, ) -> Result { - let epoch = target.signals.epoch(); - let facts_a = self.windows.facts_for_identity(&target.window).await?; - validate_observable(&facts_a)?; - let pending = capture(&facts_a, SurfaceKind::Window, deadline).await?; - let facts_b = self.windows.facts_for_identity(&facts_a.stamp).await?; - if facts_a != facts_b || !same_stable_identity(&target.window, &facts_b.stamp) { - return Err(NativeError::stale( - ErrorCode::ObservationRaced, - "target changed while producing FreshFrame evidence", - )); - } - let journal = target.signals.clone(); - let freshness = journal - .commit_if_epoch(epoch, || { + let mut race_count = 0_u64; + loop { + if Instant::now() >= deadline { + return Err(NativeError::stale( + ErrorCode::ObservationRaced, + "target did not hold a coherent FreshFrame bracket before settlement deadline", + ) + .with_detail("race_count", race_count)); + } + let epoch = target.signals.epoch(); + let facts_a = self.windows.facts_for_identity(&target.window).await?; + validate_observable(&facts_a)?; + let pending = capture(&facts_a, SurfaceKind::Window, deadline).await?; + let facts_b = self.windows.facts_for_identity(&facts_a.stamp).await?; + if !same_stable_identity(&target.window, &facts_b.stamp) { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "target identity changed while producing FreshFrame evidence", + )); + } + if facts_a != facts_b { + race_count = race_count.saturating_add(1); + continue; + } + let journal = target.signals.clone(); + let Some(freshness) = journal.commit_if_epoch(epoch, || { target.frames.classify_and_commit( pending.cg_window_id, &pending.sample, pending.scale_factor, ) })? - .ok_or_else(|| { - NativeError::stale( - ErrorCode::ObservationRaced, - "target signaled a mutation while producing FreshFrame evidence", - ) - })?; - if !freshness.action_safe() { - return Err(NativeError::stale( - ErrorCode::SurfaceStale, - "ScreenCaptureKit did not produce action-safe FreshFrame evidence", - )); + else { + // Target notifications during post-action capture are expected + // evidence that the UI is still moving. Discard the raced + // sample and immediately recapture inside the same owned + // deadline; do not turn normal settlement progress into a + // fatal observation race. + race_count = race_count.saturating_add(1); + continue; + }; + if !freshness.action_safe() { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "ScreenCaptureKit did not produce action-safe FreshFrame evidence", + )); + } + target + .signals + .record(cua_driver_core::api::settlement::SettlementSignal::FreshFrame); + return Ok(freshness); } - target - .signals - .record(cua_driver_core::api::settlement::SettlementSignal::FreshFrame); - Ok(freshness) } async fn attempt( @@ -540,9 +557,15 @@ async fn capture_sample_until( .await .map_err(|_| capture_deadline_error(window_id))? .map_err(|_| capture_deadline_error(window_id))?; + let capture_timeout = deadline + .saturating_duration_since(tokio::time::Instant::now()) + .saturating_sub(SCK_CAPTURE_TEARDOWN_RESERVE); + if capture_timeout.is_zero() { + return Err(capture_deadline_error(window_id)); + } let worker = tokio::task::spawn_blocking(move || { let _permit = permit; - capture_window_sample(window_id, scale) + capture_window_sample(window_id, scale, capture_timeout) }); tokio::time::timeout_at(deadline, worker) .await @@ -607,9 +630,15 @@ fn validated_surface_transform( expected_scale: f64, ) -> Result { let metadata = &sample.metadata; - let scale = metadata.scale_factor.ok_or_else(missing_frame_metadata)?; - let content_scale = metadata.content_scale.ok_or_else(missing_frame_metadata)?; - let content = metadata.content_rect.ok_or_else(missing_frame_metadata)?; + let scale = metadata + .scale_factor + .ok_or_else(|| missing_frame_metadata(metadata))?; + let content_scale = metadata + .content_scale + .ok_or_else(|| missing_frame_metadata(metadata))?; + let content = metadata + .content_rect + .ok_or_else(|| missing_frame_metadata(metadata))?; let sampled = sample.source_frame; let surface_width = f64::from(sample.pixel_width) / scale; let surface_height = f64::from(sample.pixel_height) / scale; @@ -630,6 +659,27 @@ fn validated_surface_transform( return Err(NativeError::stale( ErrorCode::SurfaceStale, "ScreenCaptureKit source frame, content rectangle and raster geometry disagree", + ) + .with_detail("expected_source_bounds", rect_detail(source)) + .with_detail("sample_source_frame", frame_rect_detail(sampled)) + .with_detail("target_bounds", rect_detail(target)) + .with_detail( + "pixel_size", + serde_json::json!({ + "width": sample.pixel_width, + "height": sample.pixel_height, + }), + ) + .with_detail("expected_scale_factor", expected_scale) + .with_detail("sample_scale_factor", scale) + .with_detail("content_scale", content_scale) + .with_detail("content_rect", frame_rect_detail(content)) + .with_detail( + "surface_points_from_raster", + serde_json::json!({ + "width": surface_width, + "height": surface_height, + }), )); } Ok(SurfaceToWindowTransform { @@ -644,6 +694,24 @@ fn approximately(left: f64, right: f64) -> bool { left.is_finite() && right.is_finite() && (left - right).abs() <= 0.75 } +fn rect_detail(rect: Rect) -> serde_json::Value { + serde_json::json!({ + "x": rect.x, + "y": rect.y, + "width": rect.width, + "height": rect.height, + }) +} + +fn frame_rect_detail(rect: crate::video_sckit::FrameRect) -> serde_json::Value { + serde_json::json!({ + "x": rect.x, + "y": rect.y, + "width": rect.width, + "height": rect.height, + }) +} + #[derive(Default)] pub struct MacFrameHistory { previous: HashMap, @@ -663,11 +731,21 @@ impl MacFrameHistory { expected_scale: f64, ) -> Result { let metadata = &sample.metadata; - let status = metadata.frame_status.ok_or_else(missing_frame_metadata)?; - let display_time = metadata.display_time.ok_or_else(missing_frame_metadata)?; - let scale = metadata.scale_factor.ok_or_else(missing_frame_metadata)?; - let content_scale = metadata.content_scale.ok_or_else(missing_frame_metadata)?; - let content_rect = metadata.content_rect.ok_or_else(missing_frame_metadata)?; + let status = metadata + .frame_status + .ok_or_else(|| missing_frame_metadata(metadata))?; + let display_time = metadata + .display_time + .ok_or_else(|| missing_frame_metadata(metadata))?; + let scale = metadata + .scale_factor + .ok_or_else(|| missing_frame_metadata(metadata))?; + let content_scale = metadata + .content_scale + .ok_or_else(|| missing_frame_metadata(metadata))?; + let content_rect = metadata + .content_rect + .ok_or_else(|| missing_frame_metadata(metadata))?; if metadata.completion_unix_ms == 0 || !scale.is_finite() || (scale - expected_scale).abs() > 0.05 @@ -678,7 +756,8 @@ impl MacFrameHistory { || content_rect.width <= 0.0 || content_rect.height <= 0.0 { - return Err(missing_frame_metadata()); + return Err(missing_frame_metadata(metadata) + .with_detail("expected_scale_factor", expected_scale)); } let freshness = if status.has_content() { @@ -712,11 +791,61 @@ impl MacFrameHistory { } } -fn missing_frame_metadata() -> NativeError { - NativeError::stale( +fn missing_frame_metadata(metadata: &crate::video_sckit::WindowFrameMetadata) -> NativeError { + let mut missing_fields = Vec::new(); + if metadata.completion_unix_ms == 0 { + missing_fields.push("completion_unix_ms"); + } + if metadata.display_time.is_none() { + missing_fields.push("display_time"); + } + if metadata.frame_status.is_none() { + missing_fields.push("frame_status"); + } + if metadata.scale_factor.is_none() { + missing_fields.push("scale_factor"); + } + if metadata.content_scale.is_none() { + missing_fields.push("content_scale"); + } + if metadata.content_rect.is_none() { + missing_fields.push("content_rect"); + } + + let mut invalid_fields = Vec::new(); + if metadata + .scale_factor + .is_some_and(|value| !value.is_finite() || value <= 0.0) + { + invalid_fields.push("scale_factor"); + } + if metadata + .content_scale + .is_some_and(|value| !value.is_finite() || value <= 0.0) + { + invalid_fields.push("content_scale"); + } + if metadata.content_rect.is_some_and(|rect| { + !rect.x.is_finite() + || !rect.y.is_finite() + || !rect.width.is_finite() + || !rect.height.is_finite() + || rect.width <= 0.0 + || rect.height <= 0.0 + }) { + invalid_fields.push("content_rect"); + } + + let mut error = NativeError::stale( ErrorCode::SurfaceStale, "same-sample ScreenCaptureKit freshness metadata is missing or incoherent", ) + .with_detail("missing_fields", serde_json::json!(missing_fields)) + .with_detail("invalid_fields", serde_json::json!(invalid_fields)); + if let Some(status) = metadata.frame_status { + error = error.with_detail("frame_status", status.to_string()); + } + error } pub struct RetainedAxElement(usize); @@ -917,16 +1046,9 @@ fn capture_ax_snapshot( "target AX tree is not materialized; Plan003 observation is read-only and will not durably enable application accessibility", )); } - let focused = match focused_element_of_pid(pid) { - Some(element) if bindings::ax_get_window_id(element) == Some(target_window_id) => { - Some(RetainedAxElement(element as usize)) - } - Some(element) => { - CFRelease(element as CFTypeRef); - None - } - None => None, - }; + let focused = focused_element_of_pid(pid) + .map(|element| RetainedAxElement(element as usize)) + .filter(|focused| nodes.iter().any(|node| node.element.same_identity(focused))); let selected_text = focused .as_ref() .and_then(|element| copy_string_attr(element.as_ptr(), "AXSelectedText")); @@ -1015,8 +1137,14 @@ unsafe fn walk_ax( height: frame[3], }); let native_window_id = bindings::ax_get_window_id(element); - let owner_window_id = native_window_id.unwrap_or(inherited_owner_window_id); - if let (Some(window_id), Some(kind)) = (native_window_id, transient_kind(&role)) { + let related_kind = transient_kind(&role); + let owner_window_id = structural_owner_window_id( + native_window_id, + related_kind, + target_window_id, + inherited_owner_window_id, + ); + if let (Some(window_id), Some(kind)) = (native_window_id, related_kind) { if window_id != target_window_id { related_windows.push(RawRelatedWindow { window_id, @@ -1081,6 +1209,18 @@ fn transient_kind(role: &str) -> Option { } } +fn structural_owner_window_id( + native_window_id: Option, + transient_kind: Option, + target_window_id: u32, + inherited_owner_window_id: u32, +) -> u32 { + match (native_window_id, transient_kind) { + (Some(window_id), Some(_)) if window_id != target_window_id => window_id, + _ => inherited_owner_window_id, + } +} + struct RegisteredElement { id: ElementId, native: NativeElementHandle, @@ -1668,6 +1808,10 @@ mod tests { width: 100.0, height: 50.0, }, + activation_point: Some(cua_driver_core::api::contracts::Point { + x: x + 10.0, + y: 16.0, + }), scale_factor: Some(2.0), state: WindowStateKind::Visible, is_on_screen: true, @@ -1709,12 +1853,26 @@ mod tests { let mut history = MacFrameHistory::default(); let mut incomplete = sample(SCFrameStatus::Complete, 10, b"pixels"); incomplete.metadata.display_time = None; + let error = history + .classify_and_commit(1, &incomplete, 2.0) + .unwrap_err(); + assert_eq!(error.code, ErrorCode::SurfaceStale); + assert_eq!( + error.details.get("missing_fields"), + Some(&serde_json::json!(["display_time"])) + ); + assert_eq!( + error.details.get("invalid_fields"), + Some(&serde_json::json!([])) + ); + } + + #[test] + fn embedded_remote_ax_children_inherit_target_ownership_but_transients_do_not() { + assert_eq!(structural_owner_window_id(Some(99), None, 1, 1), 1); assert_eq!( - history - .classify_and_commit(1, &incomplete, 2.0) - .unwrap_err() - .code, - ErrorCode::SurfaceStale + structural_owner_window_id(Some(99), Some(SurfaceKind::Popover), 1, 1), + 99 ); } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/target.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/target.rs index 0ac0516a3..1d617df64 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/target.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/target.rs @@ -204,25 +204,46 @@ impl Drop for MacTargetState { } } -#[derive(Debug, Default)] -pub(crate) struct MacActiveBelief { - pub action_id: cua_driver_core::api::contracts::ActionId, +#[derive(Debug)] +pub(crate) struct MacFocusState { + pub shutdown: bool, pub pid: i32, pub cg_window_id: u32, + pub application_is_active: bool, + pub application_believes_it_is_active: bool, + pub application_believes_it_has_focus: bool, } -#[derive(Debug, Default)] -pub(crate) struct MacFocusState { - pub shutdown: bool, - pub active_belief: Option, +impl MacFocusState { + pub(crate) fn new(pid: i32, cg_window_id: u32) -> Self { + let application_is_active = crate::apps::frontmost_pid() == Some(pid); + Self { + shutdown: false, + pid, + cg_window_id, + application_is_active, + application_believes_it_is_active: application_is_active, + application_believes_it_has_focus: application_is_active, + } + } } -#[derive(Default)] pub struct MacTargetFocusCoordinator { state: Arc>, + focus_taps: Option, } impl MacTargetFocusCoordinator { + pub(crate) fn new(pid: i32, cg_window_id: u32) -> Result { + let state = Arc::new(Mutex::new(MacFocusState::new(pid, cg_window_id))); + let focus_taps = + crate::focus_steal::TargetFocusTapRegistration::start(pid, Arc::downgrade(&state))?; + Ok(Self { + state, + focus_taps: Some(focus_taps), + }) + } + pub(crate) fn state_handle(&self) -> Arc> { Arc::clone(&self.state) } @@ -238,53 +259,28 @@ impl MacTargetFocusCoordinator { #[async_trait] impl TargetFocusCoordinator for MacTargetFocusCoordinator { async fn shutdown(&mut self) -> Result<(), NativeError> { + if let Some(mut focus_taps) = self.focus_taps.take() { + focus_taps.close(); + } let mut state = self.state.lock().expect("macOS focus coordinator poisoned"); state.shutdown = true; - if let Some(active) = &state.active_belief { - crate::input::slps_make_key::post_target_only( - active.pid, - active.cg_window_id, - &[crate::input::slps_make_key::SlpsMakeKeyState::RemoveKey], - ) - .map_err(|error| { - NativeError::new( - cua_driver_core::api::errors::ErrorCode::Internal, - cua_driver_core::api::errors::ErrorPhase::Verify, - false, - format!("SLPS remove-key retry failed during target teardown: {error}"), - ) - .with_detail("action_id", active.action_id.to_string()) - .with_detail("pid", active.pid) - .with_detail("cg_window_id", active.cg_window_id) - })?; - state.active_belief = None; - } + state.application_believes_it_is_active = false; + state.application_believes_it_has_focus = false; Ok(()) } } impl Drop for MacTargetFocusCoordinator { fn drop(&mut self) { + if let Some(mut focus_taps) = self.focus_taps.take() { + focus_taps.close(); + } let Ok(mut state) = self.state.lock() else { tracing::error!("macOS focus coordinator lock was poisoned during final drop"); return; }; - let Some(active) = &state.active_belief else { - return; - }; - match crate::input::slps_make_key::post_target_only( - active.pid, - active.cg_window_id, - &[crate::input::slps_make_key::SlpsMakeKeyState::RemoveKey], - ) { - Ok(()) => state.active_belief = None, - Err(error) => tracing::error!( - error = %error, - action_id = %active.action_id, - pid = active.pid, - cg_window_id = active.cg_window_id, - "final target-only SLPS remove-key record failed during coordinator drop" - ), - } + state.shutdown = true; + state.application_believes_it_is_active = false; + state.application_believes_it_has_focus = false; } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/windows.rs index ba3f13378..fc549e4e5 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/windows.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/windows.rs @@ -7,7 +7,9 @@ use async_trait::async_trait; use core_foundation::base::{CFEqual, CFRelease, CFRetain, CFTypeRef}; use cua_driver_core::api::{ capabilities::{Framework, WindowStateKind}, - contracts::{AppId, AppRef, GeometryRevision, Rect, WindowGeneration, WindowId, WindowRef}, + contracts::{ + AppId, AppRef, GeometryRevision, Point, Rect, WindowGeneration, WindowId, WindowRef, + }, errors::{ErrorCode, ErrorPhase, NativeError}, observation::{ NativeProcessHandle, NativeWindowHandle, ResolvedWindow, ResolvedWindowStamp, @@ -111,6 +113,7 @@ pub struct MacWindowFacts { pub owner_name: String, pub layer: i32, pub bounds: Rect, + pub activation_point: Option, pub scale_factor: Option, pub state: WindowStateKind, pub is_on_screen: bool, @@ -137,14 +140,14 @@ pub struct MacRelatedWindowFacts { } trait WindowSnapshotSource: Send + Sync { - fn snapshot(&self) -> Result, NativeError>; + fn snapshot(&self, pid_scope: Option) -> Result, NativeError>; } #[derive(Default)] struct SystemWindowSnapshotSource; impl WindowSnapshotSource for SystemWindowSnapshotSource { - fn snapshot(&self) -> Result, NativeError> { + fn snapshot(&self, pid_scope: Option) -> Result, NativeError> { if !permissions::status::accessibility_granted() { return Err(NativeError::new( ErrorCode::PermissionDenied, @@ -154,9 +157,9 @@ impl WindowSnapshotSource for SystemWindowSnapshotSource { )); } - match self.snapshot_once() { + match self.snapshot_once(pid_scope) { Ok(snapshots) => Ok(snapshots), - Err(_) => self.snapshot_once().map_err(|racing_pids| { + Err(_) => self.snapshot_once(pid_scope).map_err(|racing_pids| { NativeError::new( ErrorCode::WindowIdentityChanged, ErrorPhase::Preflight, @@ -173,13 +176,16 @@ impl WindowSnapshotSource for SystemWindowSnapshotSource { } impl SystemWindowSnapshotSource { - fn snapshot_once(&self) -> Result, Vec> { + fn snapshot_once(&self, pid_scope: Option) -> Result, Vec> { let applications_before: HashMap = nsworkspace::running_applications() .into_iter() .map(|application| (application.pid, application)) .collect(); - let server_windows = windows::all_windows(); + let server_windows: Vec<_> = windows::all_windows() + .into_iter() + .filter(|window| pid_scope.is_none_or(|pid| window.pid == pid)) + .collect(); let mut ax_by_pid: HashMap = HashMap::new(); let mut snapshots = Vec::new(); @@ -408,8 +414,8 @@ impl MacWindowRegistry { } } - fn refresh(&self) -> Result<(), NativeError> { - let snapshots = self.source.snapshot()?; + fn refresh(&self, pid_scope: Option) -> Result<(), NativeError> { + let snapshots = self.source.snapshot(pid_scope)?; let mut state = self .state .lock() @@ -422,7 +428,9 @@ impl MacWindowRegistry { let missing: Vec<_> = state .by_native .keys() - .filter(|key| !current_keys.contains(*key)) + .filter(|key| { + pid_scope.is_none_or(|pid| key.pid == pid) && !current_keys.contains(*key) + }) .cloned() .collect(); for key in missing { @@ -512,7 +520,22 @@ impl MacWindowRegistry { } fn list(&self, app: Option<&AppRef>) -> Result, NativeError> { - self.refresh()?; + let pid_scope = match app { + None => None, + Some(app) if !app.running => return Ok(Vec::new()), + Some(app) => Some(app.pid.and_then(|pid| i32::try_from(pid).ok()).ok_or_else( + || { + NativeError::new( + ErrorCode::WindowIdentityChanged, + ErrorPhase::Preflight, + false, + "running app reference has no valid process id", + ) + .with_detail("app_id", app.id.to_string()) + }, + )?), + }; + self.refresh(pid_scope)?; let state = self .state .lock() @@ -533,31 +556,24 @@ impl MacWindowRegistry { } fn entry(&self, id: &WindowId, app: Option<&AppRef>) -> Result { - self.refresh()?; + let pid_scope = { + let state = self + .state + .lock() + .expect("macOS window registry lock poisoned"); + match state.by_id.get(id) { + Some(entry) => entry.key.pid, + None => return Err(unknown_window_error(&state, id)), + } + }; + self.refresh(Some(pid_scope))?; let state = self .state .lock() .expect("macOS window registry lock poisoned"); let entry = match state.by_id.get(id) { Some(entry) => entry, - None => { - let (code, message) = match state.tombstones.get(id) { - Some(WindowTombstone::IdentityChanged) => ( - ErrorCode::WindowIdentityChanged, - "native window identity changed", - ), - Some(WindowTombstone::Missing) => ( - ErrorCode::WindowNotFound, - "window closed or disappeared from WindowServer", - ), - None => ( - ErrorCode::WindowNotFound, - "window id is not known to the macOS registry", - ), - }; - return Err(NativeError::new(code, ErrorPhase::Preflight, true, message) - .with_detail("window_id", id.to_string())); - } + None => return Err(unknown_window_error(&state, id)), }; if let Some(app) = app { if entry.public.app.id != app.id { @@ -811,6 +827,25 @@ impl MacWindowRegistry { } } +fn unknown_window_error(state: &RegistryState, id: &WindowId) -> NativeError { + let (code, message) = match state.tombstones.get(id) { + Some(WindowTombstone::IdentityChanged) => ( + ErrorCode::WindowIdentityChanged, + "native window identity changed", + ), + Some(WindowTombstone::Missing) => ( + ErrorCode::WindowNotFound, + "window closed or disappeared from WindowServer", + ), + None => ( + ErrorCode::WindowNotFound, + "window id is not known to the macOS registry", + ), + }; + NativeError::new(code, ErrorPhase::Preflight, true, message) + .with_detail("window_id", id.to_string()) +} + struct RelatedNativeSnapshot { key: NativeWindowKey, title: Option, @@ -1073,6 +1108,12 @@ fn facts_for_entry(entry: &RegistryEntry) -> MacWindowFacts { owner_name: entry.snapshot.owner_name.clone(), layer: entry.snapshot.layer, bounds: entry.snapshot.bounds, + activation_point: unsafe { + bindings::copy_point_attr(entry.snapshot.ax_identity.as_ptr(), "AXActivationPoint") + .ok() + .flatten() + .map(|(x, y)| Point { x, y }) + }, scale_factor: entry.snapshot.scale_factor, state: window_state(&entry.snapshot), is_on_screen: entry.snapshot.is_on_screen, @@ -1129,12 +1170,18 @@ mod tests { } impl WindowSnapshotSource for FakeSnapshotSource { - fn snapshot(&self) -> Result, NativeError> { + fn snapshot( + &self, + pid_scope: Option, + ) -> Result, NativeError> { Ok(self .snapshots .lock() .expect("fake source lock poisoned") - .clone()) + .iter() + .filter(|snapshot| pid_scope.is_none_or(|pid| snapshot.key.pid == pid)) + .cloned() + .collect()) } } @@ -1151,15 +1198,33 @@ mod tests { cg_window_id: u32, title: Option<&str>, ax_label: &str, + ) -> NativeWindowSnapshot { + snapshot_for_pid( + 101, + bundle_id, + process_generation, + cg_window_id, + title, + ax_label, + ) + } + + fn snapshot_for_pid( + pid: i32, + bundle_id: &str, + process_generation: u64, + cg_window_id: u32, + title: Option<&str>, + ax_label: &str, ) -> NativeWindowSnapshot { NativeWindowSnapshot { key: NativeWindowKey { - pid: 101, + pid, process_generation, cg_window_id, }, process: ProcessSnapshot { - pid: 101, + pid, generation: process_generation, name: Some("Fixture".to_owned()), bundle_id: Some(bundle_id.to_owned()), @@ -1334,6 +1399,49 @@ mod tests { assert_ne!(windows[0].app.id, windows[1].app.id); } + #[tokio::test] + async fn scoped_refresh_never_tombstones_an_unrelated_process() { + let (registry, source, _) = registry(); + let first = snapshot_for_pid(101, "com.example.first", 10, 44, None, "ax-1"); + let second = snapshot_for_pid(202, "com.example.second", 20, 45, None, "ax-2"); + source.replace(vec![first.clone(), second.clone()]); + let initial = registry.list_windows(None).await.unwrap(); + let first_window = initial + .iter() + .find(|window| window.app.pid == Some(101)) + .unwrap() + .clone(); + let second_window = initial + .iter() + .find(|window| window.app.pid == Some(202)) + .unwrap() + .clone(); + + source.replace(vec![first]); + assert_eq!( + registry + .list_windows(Some(&first_window.app)) + .await + .unwrap(), + vec![first_window] + ); + assert!(registry + .state + .lock() + .expect("macOS window registry lock poisoned") + .by_id + .contains_key(&second_window.id)); + + source.replace(vec![second]); + assert_eq!( + registry + .rehydrate(&second_window.id, Some(&second_window.app)) + .await + .unwrap(), + second_window + ); + } + #[tokio::test] async fn cross_app_rehydration_is_an_identity_error() { let (registry, source, _) = registry(); diff --git a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs index a4d6ab1bf..9cc63fa3d 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs @@ -12,7 +12,10 @@ //! all do exactly that — so a "background" launch flashes the target on //! top of the user's work for a few frames. //! -//! The preventer subscribes to +//! The v2 target-focus registration mirrors the helper's controller-lifetime +//! process-notification, ViewBridge keyboard, and per-target mouse event-tap +//! inputs, and reconciles the target's three focus-state booleans from the +//! process-wide workspace observer. The final restoration guard subscribes to //! `NSWorkspace.didActivateApplicationNotification` and, when an activation //! matches a registered suppression entry, immediately re-activates the //! prior frontmost app on a background thread. AppKit's @@ -56,12 +59,19 @@ //! queue's own thread regardless of run-loop state. use std::collections::HashMap; +use std::ffi::c_void; use std::sync::{ atomic::{AtomicBool, AtomicUsize, Ordering}, - Arc, Mutex, OnceLock, + mpsc, Arc, Mutex, OnceLock, Weak, }; +use std::thread; use std::time::{Duration, Instant}; +use core_foundation::{ + base::{CFRelease, CFTypeRef}, + runloop::{kCFRunLoopDefaultMode, CFRunLoop}, +}; +use cua_driver_core::api::errors::{ErrorCode, ErrorPhase, NativeError}; use objc2::rc::Retained; use objc2_app_kit::{ NSApplicationActivationOptions, NSRunningApplication, NSWorkspace, NSWorkspaceApplicationKey, @@ -70,6 +80,8 @@ use objc2_app_kit::{ use objc2_foundation::NSOperationQueue; use uuid::Uuid; +use crate::apps::nsworkspace::{WorkspaceEventHub, WorkspaceEventKind}; + /// Per-entry deadline. After this much wall-clock time the dispatcher's /// observer (and the janitor) treats the entry as leaked and prunes it /// without firing. Mirrors Swift PR #1521. @@ -78,6 +90,400 @@ const ENTRY_DEADLINE: Duration = Duration::from_secs(5); /// Janitor tick interval. The task wakes up this often while the /// dispatcher is non-empty and prunes expired entries. const JANITOR_TICK: Duration = Duration::from_secs(1); +const FOCUS_TAP_HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(1); +const FOCUS_TAP_POLL_INTERVAL: Duration = Duration::from_millis(10); +const SYNTHESIZED_ACTION_WINDOW: Duration = Duration::from_secs(1); + +fn synthesized_action_marker() -> &'static Mutex> { + static LAST_SYNTHESIZED_ACTION: OnceLock>> = OnceLock::new(); + LAST_SYNTHESIZED_ACTION.get_or_init(|| Mutex::new(None)) +} + +/// Record the same process-scoped synthesized-action marker used by the +/// signed helper's event-tap containment. +pub(crate) fn record_synthesized_action(pid: i32) { + *synthesized_action_marker() + .lock() + .expect("synthesized-action marker poisoned") = Some((pid, Instant::now())); +} + +fn synthesized_action_is_recent(pid: i32) -> bool { + synthesized_action_marker() + .lock() + .expect("synthesized-action marker poisoned") + .is_some_and(|(marked_pid, marked_at)| { + marked_pid == pid && marked_at.elapsed() <= SYNTHESIZED_ACTION_WINDOW + }) +} + +/// Controller-lifetime event-tap inputs corresponding to the helper's +/// process-notification, keyboard, and per-target mouse observation. +pub(crate) struct TargetFocusTapRegistration { + close: mpsc::Sender<()>, + run_loop: Arc, + thread: Option>, +} + +impl TargetFocusTapRegistration { + pub(crate) fn start( + pid: i32, + state: Weak>, + ) -> Result { + let (started_tx, started_rx) = mpsc::sync_channel(1); + let (close_tx, close_rx) = mpsc::channel(); + let run_loop = Arc::new(AtomicUsize::new(0)); + let thread_run_loop = Arc::clone(&run_loop); + let workspace_events = WorkspaceEventHub::shared().subscribe(); + let thread = thread::Builder::new() + .name(format!("cua-focus-taps-{pid}")) + .spawn(move || { + run_target_focus_taps( + pid, + state, + workspace_events, + &thread_run_loop, + &started_tx, + &close_rx, + ) + }) + .map_err(|error| { + focus_tap_error(format!("failed to spawn focus-tap thread: {error}")) + })?; + match started_rx.recv_timeout(FOCUS_TAP_HANDSHAKE_TIMEOUT) { + Ok(Ok(())) => Ok(Self { + close: close_tx, + run_loop, + thread: Some(thread), + }), + Ok(Err(error)) => { + let _ = thread.join(); + Err(error) + } + Err(_) => { + let _ = close_tx.send(()); + let run_loop = run_loop.load(Ordering::Acquire); + if run_loop != 0 { + unsafe { CFRunLoopStop(run_loop as *mut c_void) }; + } + let _ = thread.join(); + Err(focus_tap_error( + "focus event-tap registration exceeded its bounded handshake", + )) + } + } + } + + pub(crate) fn close(&mut self) { + if self.thread.is_none() { + return; + } + let _ = self.close.send(()); + let run_loop = self.run_loop.load(Ordering::Acquire); + if run_loop != 0 { + unsafe { CFRunLoopStop(run_loop as *mut c_void) }; + } + if let Some(thread) = self.thread.take() { + if thread.join().is_err() { + tracing::error!("macOS target focus-tap thread panicked during teardown"); + } + } + } +} + +impl Drop for TargetFocusTapRegistration { + fn drop(&mut self) { + self.close(); + } +} + +struct FocusTapContext { + pid: i32, + state: Weak>, +} + +unsafe extern "C" fn focus_tap_callback( + _proxy: *const c_void, + event_type: u32, + event: *mut c_void, + user_info: *mut c_void, +) -> *mut c_void { + let context = &*(user_info as *const FocusTapContext); + if let Some(state) = context.state.upgrade() { + let real_active = crate::apps::frontmost_pid() == Some(context.pid); + reconcile_real_active(&state, real_active); + if synthesized_action_is_recent(context.pid) + && matches!(event_type, 1 | 2 | 3 | 4 | 10 | 11 | 12 | 21 | 25 | 26) + { + let mut state = state.lock().expect("macOS focus coordinator poisoned"); + state.application_believes_it_is_active = true; + state.application_believes_it_has_focus = true; + } + } + event +} + +fn run_target_focus_taps( + pid: i32, + state: Weak>, + mut workspace_events: tokio::sync::broadcast::Receiver< + crate::apps::nsworkspace::WorkspaceEvent, + >, + run_loop_slot: &AtomicUsize, + started: &mpsc::SyncSender>, + close: &mpsc::Receiver<()>, +) { + let context = Box::new(FocusTapContext { + pid, + state: state.clone(), + }); + let context_ptr = Box::into_raw(context); + let process_tap = + unsafe { CGEventTapCreate(2, 1, 1, 1_u64 << 21, focus_tap_callback, context_ptr.cast()) }; + let mouse_tap = unsafe { + CGEventTapCreateForPid( + pid, + 1, + 1, + (1_u64 << 1) + | (1_u64 << 2) + | (1_u64 << 3) + | (1_u64 << 4) + | (1_u64 << 25) + | (1_u64 << 26), + focus_tap_callback, + context_ptr.cast(), + ) + }; + let view_bridge_tap = view_bridge_pid().map(|view_bridge_pid| unsafe { + CGEventTapCreateForPid( + view_bridge_pid, + 1, + 1, + (1_u64 << 10) | (1_u64 << 11) | (1_u64 << 12), + focus_tap_callback, + context_ptr.cast(), + ) + }); + if process_tap.is_null() + || mouse_tap.is_null() + || view_bridge_tap.is_some_and(|tap| tap.is_null()) + { + let taps = [Some(process_tap), view_bridge_tap, Some(mouse_tap)]; + unsafe { + release_taps(&taps); + drop(Box::from_raw(context_ptr)); + } + let _ = started.send(Err(focus_tap_error( + "required process-notification, keyboard, or target mouse event tap is unavailable", + ))); + return; + } + let taps = [Some(process_tap), view_bridge_tap, Some(mouse_tap)]; + + let sources = taps.map(|tap| { + tap.map(|tap| unsafe { CFMachPortCreateRunLoopSource(std::ptr::null(), tap, 0) }) + }); + if sources.iter().flatten().any(|source| source.is_null()) { + unsafe { + release_sources(&sources); + release_taps(&taps); + drop(Box::from_raw(context_ptr)); + } + let _ = started.send(Err(focus_tap_error( + "required focus event tap has no run-loop source", + ))); + return; + } + let run_loop = unsafe { CFRunLoopGetCurrent() }; + run_loop_slot.store(run_loop as usize, Ordering::Release); + for source in sources.iter().flatten().copied() { + unsafe { + CFRunLoopAddSource(run_loop, source, kCFRunLoopDefaultMode.cast::()); + } + } + for tap in taps.iter().flatten().copied() { + unsafe { CGEventTapEnable(tap, true) }; + } + if started.send(Ok(())).is_err() { + for source in sources.iter().flatten().copied() { + unsafe { + CFRunLoopRemoveSource(run_loop, source, kCFRunLoopDefaultMode.cast::()); + } + } + run_loop_slot.store(0, Ordering::Release); + unsafe { + release_sources(&sources); + release_taps(&taps); + drop(Box::from_raw(context_ptr)); + } + return; + } + + loop { + drain_workspace_focus_events(pid, &state, &mut workspace_events); + match close.try_recv() { + Ok(()) | Err(mpsc::TryRecvError::Disconnected) => break, + Err(mpsc::TryRecvError::Empty) => { + CFRunLoop::run_in_mode( + unsafe { kCFRunLoopDefaultMode }, + FOCUS_TAP_POLL_INTERVAL, + true, + ); + } + } + } + for source in sources.iter().flatten().copied() { + unsafe { + CFRunLoopRemoveSource(run_loop, source, kCFRunLoopDefaultMode.cast::()); + } + } + run_loop_slot.store(0, Ordering::Release); + unsafe { + release_sources(&sources); + release_taps(&taps); + drop(Box::from_raw(context_ptr)); + } +} + +fn drain_workspace_focus_events( + pid: i32, + state: &Weak>, + events: &mut tokio::sync::broadcast::Receiver, +) { + loop { + match events.try_recv() { + Ok(event) + if matches!( + event.kind, + WorkspaceEventKind::Activated | WorkspaceEventKind::Deactivated + ) => + { + let Some(state) = state.upgrade() else { + return; + }; + reconcile_real_active(&state, crate::apps::frontmost_pid() == Some(pid)); + } + Ok(_) => {} + Err(tokio::sync::broadcast::error::TryRecvError::Lagged(_)) => { + let Some(state) = state.upgrade() else { + return; + }; + reconcile_real_active(&state, crate::apps::frontmost_pid() == Some(pid)); + } + Err(tokio::sync::broadcast::error::TryRecvError::Empty) + | Err(tokio::sync::broadcast::error::TryRecvError::Closed) => return, + } + } +} + +fn reconcile_real_active( + state: &Arc>, + application_is_active: bool, +) { + let mut state = state.lock().expect("macOS focus coordinator poisoned"); + let changed = state.application_is_active != application_is_active; + state.application_is_active = application_is_active; + if changed { + state.application_believes_it_is_active = application_is_active; + state.application_believes_it_has_focus = application_is_active; + } +} + +fn view_bridge_pid() -> Option { + let capacity = unsafe { libc::proc_listallpids(std::ptr::null_mut(), 0) }; + if capacity <= 0 { + return None; + } + let mut pids = vec![0_i32; capacity as usize]; + let byte_len = pids + .len() + .checked_mul(std::mem::size_of::()) + .and_then(|value| i32::try_from(value).ok())?; + let count = unsafe { libc::proc_listallpids(pids.as_mut_ptr().cast(), byte_len) }; + for pid in pids.into_iter().take(count.max(0) as usize) { + if pid <= 0 { + continue; + } + let mut name = [0_u8; 256]; + let length = unsafe { + libc::proc_name( + pid, + name.as_mut_ptr().cast(), + u32::try_from(name.len()).expect("process name buffer length fits u32"), + ) + }; + if length > 0 + && std::str::from_utf8(&name[..length as usize]).ok() == Some("ViewBridgeAuxiliary") + { + return Some(pid); + } + } + None +} + +unsafe fn release_sources(sources: &[Option<*mut c_void>; 3]) { + for source in sources + .iter() + .flatten() + .copied() + .filter(|source| !source.is_null()) + { + CFRelease(source as CFTypeRef); + } +} + +unsafe fn release_taps(taps: &[Option<*mut c_void>; 3]) { + for tap in taps.iter().flatten().copied().filter(|tap| !tap.is_null()) { + CFRelease(tap as CFTypeRef); + } +} + +fn focus_tap_error(message: impl Into) -> NativeError { + NativeError::new( + ErrorCode::UnsupportedInBackground, + ErrorPhase::Preflight, + false, + message, + ) + .with_detail("recipe_status", "recipe_unproven") +} + +type EventTapCallback = + unsafe extern "C" fn(*const c_void, u32, *mut c_void, *mut c_void) -> *mut c_void; + +#[link(name = "CoreGraphics", kind = "framework")] +extern "C" { + fn CGEventTapCreate( + location: u32, + placement: u32, + options: u32, + event_mask: u64, + callback: EventTapCallback, + user_info: *mut c_void, + ) -> *mut c_void; + fn CGEventTapCreateForPid( + pid: i32, + placement: u32, + options: u32, + event_mask: u64, + callback: EventTapCallback, + user_info: *mut c_void, + ) -> *mut c_void; + fn CGEventTapEnable(tap: *mut c_void, enable: bool); +} + +#[link(name = "CoreFoundation", kind = "framework")] +extern "C" { + fn CFMachPortCreateRunLoopSource( + allocator: *const c_void, + port: *mut c_void, + order: isize, + ) -> *mut c_void; + fn CFRunLoopGetCurrent() -> *mut c_void; + fn CFRunLoopAddSource(run_loop: *mut c_void, source: *mut c_void, mode: *const c_void); + fn CFRunLoopRemoveSource(run_loop: *mut c_void, source: *mut c_void, mode: *const c_void); + fn CFRunLoopStop(run_loop: *mut c_void); +} /// Identifier for a suppression. `with_suppression` and `begin_suppression` /// hand one of these back; `end_suppression` consumes it. @@ -763,6 +1169,32 @@ mod tests { use super::*; use std::sync::Arc; + #[test] + fn real_active_transitions_reconcile_belief_without_revoking_synthetic_state() { + let state = Arc::new(Mutex::new(crate::driver::target::MacFocusState { + shutdown: false, + pid: 42, + cg_window_id: 7, + application_is_active: false, + application_believes_it_is_active: true, + application_believes_it_has_focus: true, + })); + + reconcile_real_active(&state, false); + { + let state = state.lock().unwrap(); + assert!(state.application_believes_it_is_active); + assert!(state.application_believes_it_has_focus); + } + + reconcile_real_active(&state, true); + reconcile_real_active(&state, false); + let state = state.lock().unwrap(); + assert!(!state.application_is_active); + assert!(!state.application_believes_it_is_active); + assert!(!state.application_believes_it_has_focus); + } + /// Dispatcher::add returns a handle, the entry is reachable by /// match, and remove() drops it. #[test] diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mod.rs index cc3e0921d..4ec779cb2 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/input/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mod.rs @@ -13,6 +13,7 @@ pub mod keyboard; pub mod mouse; pub mod skylight; pub mod slps_make_key; +pub(crate) mod synthesized_event; pub use ax_actions::perform_ax_action; pub use keyboard::{hotkey, press_key, type_text}; diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/synthesized_event.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/synthesized_event.rs new file mode 100644 index 000000000..612dd4142 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/input/synthesized_event.rs @@ -0,0 +1,305 @@ +//! Canonical AppKit-to-CGEvent synthesis used by the signed macOS helper. + +use std::{ + ffi::c_void, + sync::atomic::{AtomicIsize, Ordering}, +}; + +use cua_driver_core::api::{ + contracts::{Modifier, MouseButton, Point}, + errors::{ErrorCode, ErrorPhase, NativeError}, +}; +use objc2::msg_send; +use objc2_app_kit::{NSEvent, NSEventModifierFlags, NSEventType}; +use objc2_foundation::NSPoint; + +pub(crate) const CPS_NOTIFY_KEY_FOCUS_RETURNED: i16 = i16::MIN; + +const APPKIT_DEFINED_EVENT_TYPE: usize = 13; +const APP_ACTIVATED_SUBTYPE: i16 = 1; +const APP_ACTIVATED_MODIFIER_FLAGS: usize = 0xC0000; +const PROCESS_NOTIFICATION_EVENT_TYPE: usize = 21; + +// `-[NSEvent CGEvent]` returns a `CGEventRef` whose Objective-C type encoding +// is `^{__CGEvent=}`. objc2 checks that encoding at runtime, so a generic +// `*mut c_void` (`^v`) is not a valid return type for the selector. +#[repr(C)] +struct CGEvent { + _opaque: [u8; 0], +} + +unsafe impl objc2::RefEncode for CGEvent { + const ENCODING_REF: objc2::Encoding = + objc2::Encoding::Pointer(&objc2::Encoding::Struct("__CGEvent", &[])); +} + +type CGEventRef = *mut CGEvent; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum MouseEventKind { + Down, + Up, +} + +pub(crate) struct MouseEventSpec<'a> { + pub pid: i32, + pub cg_window_id: u32, + pub screen: Point, + pub window_local: Point, + pub button: MouseButton, + pub click_count: u8, + pub modifiers: &'a [Modifier], + pub kind: MouseEventKind, +} + +pub(crate) fn post_mouse_event(spec: &MouseEventSpec<'_>) -> Result<(), NativeError> { + post_mouse_event_with_number(spec, next_event_number()) +} + +fn post_mouse_event_with_number( + spec: &MouseEventSpec<'_>, + event_number: isize, +) -> Result<(), NativeError> { + let event_type = match (spec.kind, spec.button) { + (MouseEventKind::Down, MouseButton::Left) => NSEventType::LeftMouseDown, + (MouseEventKind::Up, MouseButton::Left) => NSEventType::LeftMouseUp, + (MouseEventKind::Down, MouseButton::Right) => NSEventType::RightMouseDown, + (MouseEventKind::Up, MouseButton::Right) => NSEventType::RightMouseUp, + (MouseEventKind::Down, MouseButton::Middle) => NSEventType::OtherMouseDown, + (MouseEventKind::Up, MouseButton::Middle) => NSEventType::OtherMouseUp, + }; + let event = unsafe { + NSEvent::mouseEventWithType_location_modifierFlags_timestamp_windowNumber_context_eventNumber_clickCount_pressure( + event_type, + NSPoint::new(spec.screen.x, spec.screen.y), + ns_modifier_flags(spec.modifiers), + 0.0, + isize::try_from(spec.cg_window_id).unwrap_or(isize::MAX), + None, + event_number, + isize::from(spec.click_count), + 1.0, + ) + } + .ok_or_else(|| construction_failed("NSEvent.mouseEvent"))?; + + let cg_event = unsafe { cg_event(&event) }?; + unsafe { + CGEventSetFlags(cg_event, modifier_bits(spec.modifiers)); + CGEventSetLocation(cg_event.cast(), spec.screen.x, spec.screen.y); + CGEventSetIntegerValueField(cg_event, 3, button_number(spec.button)); + CGEventSetIntegerValueField(cg_event, 7, 3); + CGEventSetIntegerValueField(cg_event, 91, i64::from(spec.cg_window_id)); + CGEventSetIntegerValueField(cg_event, 92, i64::from(spec.cg_window_id)); + CGEventSetWindowLocation(cg_event, spec.window_local.x, spec.window_local.y); + timestamp_and_post(spec.pid, cg_event)?; + } + Ok(()) +} + +pub(crate) fn post_app_activated( + pid: i32, + cg_window_id: u32, + window_bounds: cua_driver_core::api::contracts::Rect, + activation_point: Option, +) -> Result { + let event = unsafe { + NSEvent::otherEventWithType_location_modifierFlags_timestamp_windowNumber_context_subtype_data1_data2( + NSEventType(APPKIT_DEFINED_EVENT_TYPE), + NSPoint::new(0.0, 0.0), + NSEventModifierFlags(if cg_window_id == 0 { + 0 + } else { + APP_ACTIVATED_MODIFIER_FLAGS + }), + 0.0, + isize::try_from(cg_window_id).unwrap_or(isize::MAX), + None, + APP_ACTIVATED_SUBTYPE, + 0, + 0, + ) + } + .ok_or_else(|| construction_failed("NSEvent.appActivated"))?; + let cg_event = unsafe { cg_event(&event) }?; + unsafe { + timestamp_and_post(pid, cg_event)?; + } + + let Some(screen) = activation_point else { + return Ok(1); + }; + let window_local = Point { + x: screen.x - window_bounds.x, + y: screen.y - window_bounds.y, + }; + for (event_number, kind) in [(1, MouseEventKind::Down), (2, MouseEventKind::Up)] { + post_mouse_event_with_number( + &MouseEventSpec { + pid, + cg_window_id, + screen, + window_local, + button: MouseButton::Left, + click_count: 1, + modifiers: &[], + kind, + }, + event_number, + )?; + } + Ok(3) +} + +pub(crate) fn post_key_focus_returned(pid: i32) -> Result<(), NativeError> { + post_other_event( + pid, + NSEventType(PROCESS_NOTIFICATION_EVENT_TYPE), + CPS_NOTIFY_KEY_FOCUS_RETURNED, + "NSEvent.processNotification", + ) +} + +fn post_other_event( + pid: i32, + event_type: NSEventType, + subtype: i16, + event_name: &'static str, +) -> Result<(), NativeError> { + let event = unsafe { + NSEvent::otherEventWithType_location_modifierFlags_timestamp_windowNumber_context_subtype_data1_data2( + event_type, + NSPoint::new(0.0, 0.0), + NSEventModifierFlags(0), + 0.0, + 0, + None, + subtype, + 0, + 0, + ) + } + .ok_or_else(|| construction_failed(event_name))?; + let cg_event = unsafe { cg_event(&event) }?; + unsafe { + timestamp_and_post(pid, cg_event)?; + } + Ok(()) +} + +unsafe fn cg_event(event: &NSEvent) -> Result { + let cg_event: CGEventRef = msg_send![event, CGEvent]; + if cg_event.is_null() { + Err(construction_failed("NSEvent.CGEvent")) + } else { + Ok(cg_event) + } +} + +unsafe fn timestamp_and_post(pid: i32, event: CGEventRef) -> Result<(), NativeError> { + let mut time = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + if libc::clock_gettime(libc::CLOCK_UPTIME_RAW, &mut time) != 0 { + return Err(NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "failed to read the macOS uptime clock before targeted event posting", + )); + } + let seconds = u64::try_from(time.tv_sec).unwrap_or_default(); + let nanoseconds = u64::try_from(time.tv_nsec).unwrap_or_default(); + let timestamp = seconds + .checked_mul(1_000_000_000) + .and_then(|value| value.checked_add(nanoseconds)) + .ok_or_else(|| { + NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "macOS event timestamp overflowed", + ) + })?; + CGEventSetTimestamp(event, timestamp); + CGEventPostToPid(pid, event); + Ok(()) +} + +fn next_event_number() -> isize { + static NEXT_EVENT_NUMBER: AtomicIsize = AtomicIsize::new(1); + NEXT_EVENT_NUMBER.fetch_add(1, Ordering::Relaxed) +} + +fn ns_modifier_flags(modifiers: &[Modifier]) -> NSEventModifierFlags { + NSEventModifierFlags(modifier_bits(modifiers) as usize) +} + +fn modifier_bits(modifiers: &[Modifier]) -> u64 { + modifiers.iter().fold(0, |flags, modifier| { + flags + | match modifier { + Modifier::Shift => 1 << 17, + Modifier::Control => 1 << 18, + Modifier::Alt => 1 << 19, + Modifier::Meta => 1 << 20, + } + }) +} + +fn button_number(button: MouseButton) -> i64 { + match button { + MouseButton::Left => 0, + MouseButton::Right => 1, + MouseButton::Middle => 2, + } +} + +fn construction_failed(native_constructor: &'static str) -> NativeError { + NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "AppKit could not construct the synthesized native event", + ) + .with_detail("native_constructor", native_constructor) +} + +#[link(name = "CoreGraphics", kind = "framework")] +extern "C" { + fn CGEventSetFlags(event: CGEventRef, flags: u64); + fn CGEventSetLocation(event: *mut c_void, x: f64, y: f64); + fn CGEventSetIntegerValueField(event: CGEventRef, field: u32, value: i64); + fn CGEventSetWindowLocation(event: CGEventRef, x: f64, y: f64); + fn CGEventSetTimestamp(event: CGEventRef, timestamp: u64); + fn CGEventPostToPid(pid: i32, event: CGEventRef); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn appkit_cg_event_selector_uses_the_real_cg_event_ref_abi() { + let event = unsafe { + NSEvent::otherEventWithType_location_modifierFlags_timestamp_windowNumber_context_subtype_data1_data2( + NSEventType(PROCESS_NOTIFICATION_EVENT_TYPE), + NSPoint::new(0.0, 0.0), + NSEventModifierFlags(0), + 0.0, + 0, + None, + CPS_NOTIFY_KEY_FOCUS_RETURNED, + 0, + 0, + ) + } + .expect("AppKit should construct the process-notification event"); + + let cg_event = + unsafe { cg_event(&event) }.expect("NSEvent.CGEvent should return a CGEventRef"); + + assert!(!cg_event.is_null()); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/video_sckit.rs b/libs/cua-driver/rust/crates/platform-macos/src/video_sckit.rs index 438086275..c61995790 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/video_sckit.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/video_sckit.rs @@ -22,22 +22,24 @@ //! 3. `stop()` calls `stop_capture()` (which finalises the mp4 moov //! atom) and returns the elapsed-time metadata. -use std::path::Path; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; +use std::{ + ffi::c_void, + path::Path, + sync::mpsc::{sync_channel, RecvTimeoutError}, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; use cua_driver_core::video::{VideoBackend, VideoBackendFactory, VideoMetadata}; +use screencapturekit::cm::{CMSampleBufferExt, CMSampleBufferSCExt, SCFrameStatus}; use screencapturekit::prelude::{ - SCContentFilter, SCShareableContent, SCStream, SCStreamConfiguration, + CMSampleBuffer, SCContentFilter, SCShareableContent, SCStream, SCStreamConfiguration, + SCStreamOutputType, }; use screencapturekit::recording_output::{ SCRecordingOutput, SCRecordingOutputCodec, SCRecordingOutputConfiguration, SCRecordingOutputFileType, }; -use screencapturekit::{ - cm::{CMSampleBufferExt, CMSampleBufferSCExt, SCFrameStatus}, - screenshot_manager::SCScreenshotManager, -}; pub struct SckitVideoBackendFactory; @@ -84,16 +86,25 @@ impl Drop for TemporaryCapturePath { /// /// This is intentionally separate from the long-lived recording stream. A /// v2 observation needs one bounded sample whose pixels, frame status, -/// display timestamp and scale attachments are coherent. The old -/// `screencapture` CLI helper cannot provide that contract and is never called -/// here. +/// display timestamp and scale attachments are coherent. Apple's one-shot +/// `SCScreenshotManager.captureSampleBuffer` returns pixels without +/// `SCStreamFrameInfo` attachments on current macOS, so observations use one +/// exact-window `SCStream` callback instead. The old `screencapture` CLI +/// helper cannot provide this contract and is never called here. pub fn capture_window_sample( window_id: u32, expected_scale_factor: f64, + timeout: Duration, ) -> anyhow::Result { if !expected_scale_factor.is_finite() || expected_scale_factor <= 0.0 { anyhow::bail!("invalid expected scale factor for window {window_id}"); } + if timeout.is_zero() { + anyhow::bail!("ScreenCaptureKit window {window_id} sample deadline elapsed"); + } + let deadline = Instant::now() + .checked_add(timeout) + .ok_or_else(|| anyhow::anyhow!("invalid ScreenCaptureKit sample timeout"))?; let content = SCShareableContent::get() .map_err(|error| anyhow::anyhow!("SCShareableContent::get failed: {error}"))?; @@ -109,12 +120,50 @@ pub fn capture_window_sample( let configuration = SCStreamConfiguration::new() .with_width(width) .with_height(height) + .with_queue_depth(2) .with_ignores_shadows_single_window(true) .with_shows_cursor(false) .with_captures_audio(false); - let sample = SCScreenshotManager::capture_sample_buffer(&filter, &configuration) - .map_err(|error| anyhow::anyhow!("ScreenCaptureKit sample failed: {error}"))?; + let (sender, receiver) = sync_channel(2); + let mut stream = SCStream::new(&filter, &configuration); + stream + .add_output_handler( + move |sample, _| { + let completion_unix_ms = unix_ms_now(); + let _ = sender.try_send((sample, completion_unix_ms)); + }, + SCStreamOutputType::Screen, + ) + .ok_or_else(|| { + anyhow::anyhow!("ScreenCaptureKit failed to register output for window {window_id}") + })?; + stream + .start_capture() + .map_err(|error| anyhow::anyhow!("ScreenCaptureKit stream start failed: {error}"))?; + let remaining = deadline.saturating_duration_since(Instant::now()); + let received = if remaining.is_zero() { + Err(anyhow::anyhow!( + "ScreenCaptureKit window {window_id} sample deadline elapsed" + )) + } else { + receiver + .recv_timeout(remaining) + .map_err(|error| match error { + RecvTimeoutError::Timeout => anyhow::anyhow!( + "ScreenCaptureKit window {window_id} produced no sample before its deadline" + ), + RecvTimeoutError::Disconnected => anyhow::anyhow!( + "ScreenCaptureKit output for window {window_id} disconnected before a sample" + ), + }) + }; + let stop_result = stream.stop_capture().map_err(|error| { + anyhow::anyhow!("ScreenCaptureKit stream stop failed for window {window_id}: {error}") + }); + let (sample, completion_unix_ms) = received?; + stop_result?; + let frame_info = sample.frame_info(); let image = sample .cg_image() @@ -139,15 +188,17 @@ pub fn capture_window_sample( anyhow::bail!("ScreenCaptureKit produced an empty PNG for window {window_id}"); } - let completion_unix_ms = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap_or_default() - .as_millis() - .min(u128::from(u64::MAX)) as u64; let metadata = WindowFrameMetadata { completion_unix_ms, display_time: frame_info.as_ref().and_then(|info| info.display_time), - frame_status: frame_info.as_ref().and_then(|info| info.frame_status), + // screencapturekit 6.0.1 casts the attachment's NSNumber directly to + // Swift's SCFrameStatus enum and silently loses it. Read the numeric + // attachment through CoreFoundation until the dependency fixes that + // bridge; never infer status from pixels or timestamps. + frame_status: frame_info + .as_ref() + .and_then(|info| info.frame_status) + .or_else(|| frame_status_from_attachment(&sample)), scale_factor: frame_info.as_ref().and_then(|info| info.scale_factor), content_scale: frame_info.as_ref().and_then(|info| info.content_scale), content_rect: frame_info @@ -174,6 +225,65 @@ pub fn capture_window_sample( }) } +fn unix_ms_now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .min(u128::from(u64::MAX)) as u64 +} + +/// Decode `SCStreamFrameInfoStatus` from the sample attachment dictionary. +/// +/// ScreenCaptureKit stores the enum as a CFNumber/NSNumber. This deliberately +/// uses Apple's exported key constant rather than copying its string value. +fn frame_status_from_attachment(sample: &CMSampleBuffer) -> Option { + type CFIndex = isize; + type CFTypeId = usize; + const CF_NUMBER_SINT32_TYPE: i32 = 3; + + #[link(name = "CoreMedia", kind = "framework")] + unsafe extern "C" { + fn CMSampleBufferGetSampleAttachmentsArray( + sample_buffer: *mut c_void, + create_if_necessary: u8, + ) -> *const c_void; + } + #[link(name = "CoreFoundation", kind = "framework")] + unsafe extern "C" { + fn CFArrayGetCount(array: *const c_void) -> CFIndex; + fn CFArrayGetValueAtIndex(array: *const c_void, index: CFIndex) -> *const c_void; + fn CFDictionaryGetValue(dictionary: *const c_void, key: *const c_void) -> *const c_void; + fn CFGetTypeID(value: *const c_void) -> CFTypeId; + fn CFNumberGetTypeID() -> CFTypeId; + fn CFNumberGetValue(number: *const c_void, number_type: i32, value: *mut c_void) -> u8; + } + #[link(name = "ScreenCaptureKit", kind = "framework")] + unsafe extern "C" { + static SCStreamFrameInfoStatus: *const c_void; + } + + unsafe { + let attachments = CMSampleBufferGetSampleAttachmentsArray(sample.as_ptr(), 0); + if attachments.is_null() || CFArrayGetCount(attachments) < 1 { + return None; + } + let dictionary = CFArrayGetValueAtIndex(attachments, 0); + if dictionary.is_null() { + return None; + } + let number = CFDictionaryGetValue(dictionary, SCStreamFrameInfoStatus); + if number.is_null() || CFGetTypeID(number) != CFNumberGetTypeID() { + return None; + } + let mut raw = 0_i32; + if CFNumberGetValue(number, CF_NUMBER_SINT32_TYPE, (&mut raw as *mut i32).cast()) == 0 { + return None; + } + SCFrameStatus::from_raw(raw) + } +} + fn scaled_dimension(points: f64, scale: f64, name: &str) -> anyhow::Result { let pixels = (points * scale).round(); if !pixels.is_finite() || pixels < 1.0 || pixels > f64::from(u32::MAX) { diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs index 84c92c037..7d4af01c4 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs @@ -274,10 +274,20 @@ pub fn display_scale_for_bounds(bounds: &WindowBounds) -> Option { let right = (bounds.x + bounds.width).min(frame.origin.x + frame.size.width); let bottom = (bounds.y + bounds.height).min(frame.origin.y + frame.size.height); let area = (right - left).max(0.0) * (bottom - top).max(0.0); - if area <= 0.0 || frame.size.width <= 0.0 { + if area <= 0.0 { continue; } - let scale = display.pixels_wide() as f64 / frame.size.width; + // CGDisplayPixelsWide and CGDisplayBounds both report logical mode + // dimensions on Retina displays, so their ratio is incorrectly 1.0. + // The current display mode exposes both logical and physical widths; + // their ratio matches NSScreen.backingScaleFactor without requiring + // AppKit access from the registry worker. + let Some(mode) = display.display_mode() else { + continue; + }; + let Some(scale) = backing_scale(mode.width(), mode.pixel_width()) else { + continue; + }; if best.map_or(true, |(best_area, _)| area > best_area) { best = Some((area, scale)); } @@ -285,6 +295,14 @@ pub fn display_scale_for_bounds(bounds: &WindowBounds) -> Option { best.map(|(_, scale)| scale) } +fn backing_scale(logical_width: u64, physical_width: u64) -> Option { + if logical_width == 0 || physical_width == 0 { + return None; + } + let scale = physical_width as f64 / logical_width as f64; + (scale.is_finite() && scale > 0.0).then_some(scale) +} + type CopySpacesForWindowsFn = unsafe extern "C" fn( u32, u32, @@ -427,3 +445,14 @@ fn copy_current_space_ids(connection: u32) -> Option> { } Some(result) } + +#[cfg(test)] +mod tests { + use super::backing_scale; + + #[test] + fn backing_scale_uses_physical_display_mode_pixels_on_retina() { + assert_eq!(backing_scale(1_710, 3_420), Some(2.0)); + assert_eq!(backing_scale(1_920, 3_840), Some(2.0)); + } +}