diff --git a/libs/cua-driver/rust/Cargo.lock b/libs/cua-driver/rust/Cargo.lock index 1f5737aaba..c50d40941d 100644 --- a/libs/cua-driver/rust/Cargo.lock +++ b/libs/cua-driver/rust/Cargo.lock @@ -644,6 +644,7 @@ dependencies = [ "thiserror", "tokio", "tracing", + "uuid", "windows 0.58.0", ] diff --git a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml index 71d23bbca3..fb38c44edd 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml +++ b/libs/cua-driver/rust/crates/cua-driver-core/Cargo.toml @@ -11,6 +11,7 @@ anyhow = { workspace = true } thiserror = { workspace = true } tracing = { workspace = true } async-trait = "0.1" +uuid = { workspace = true } # Used by the shared `image_utils` module — pure PNG/JPEG/resize/crosshair # helpers extracted from `platform-{macos,windows,linux}/src/capture.rs` so # all three platforms call the same code path. diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/capabilities.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/capabilities.rs new file mode 100644 index 0000000000..6ee9f645e6 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/capabilities.rs @@ -0,0 +1,138 @@ +//! Conservative, explicit route capability reporting and preflight. + +use std::collections::BTreeMap; + +use serde::{Deserialize, Serialize}; + +use super::contracts::Route; + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PlatformName { + Macos, + Windows, + Linux, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ActionKind { + Click, + Drag, + Scroll, + PressKey, + TypeText, + SetValue, + SelectText, + PerformSecondaryAction, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AddressingMode { + Window, + Element, + CapturedPoint, + ObservedFocus, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Framework { + AppKit, + Chromium, + Electron, + WebKit, + Catalyst, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WindowStateKind { + Visible, + Occluded, + Minimized, + OffSpace, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CapabilityKey { + pub platform: PlatformName, + pub os_version: String, + pub action: ActionKind, + pub addressing: AddressingMode, + pub framework: Framework, + pub window_state: WindowStateKind, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "decision", rename_all = "snake_case", deny_unknown_fields)] +pub enum RouteDecision { + Supported { route: Route }, + Unsupported { reason: String }, +} + +impl RouteDecision { + pub fn route(&self) -> Option { + match self { + Self::Supported { route } => Some(*route), + Self::Unsupported { .. } => None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CapabilityCell { + pub key: CapabilityKey, + pub decision: RouteDecision, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CapabilityManifest { + pub platform: PlatformName, + pub driver_version: String, + pub protocol_version: String, + #[serde(default)] + pub permissions: BTreeMap, + #[serde(default)] + pub cells: Vec, +} + +#[derive(Debug, Default)] +pub struct CapabilityRegistry { + cells: BTreeMap, +} + +impl CapabilityRegistry { + pub fn from_cells(cells: impl IntoIterator) -> Self { + let mut registry = Self::default(); + for cell in cells { + assert!( + registry.cells.insert(cell.key, cell.decision).is_none(), + "platform capability source emitted a duplicate cell" + ); + } + registry + } + + pub fn decision(&self, key: &CapabilityKey) -> RouteDecision { + self.cells + .get(key) + .cloned() + .unwrap_or_else(|| RouteDecision::Unsupported { + reason: "capability cell has not been proven for background execution".to_owned(), + }) + } + + pub fn cells(&self) -> impl Iterator + '_ { + self.cells.iter().map(|(key, decision)| CapabilityCell { + key: key.clone(), + decision: decision.clone(), + }) + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/contracts.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/contracts.rs new file mode 100644 index 0000000000..c888769000 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/contracts.rs @@ -0,0 +1,743 @@ +//! Stable serde contracts for the background-only v2 API. + +use std::{collections::BTreeMap, fmt}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use uuid::Uuid; + +use super::{ + interaction::{NativeEvidence, PostureResult}, + menu::MenuState, + settlement::SettlementEvidence, +}; + +macro_rules! opaque_id { + ($name:ident) => { + #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize)] + #[serde(transparent)] + pub struct $name(String); + + impl $name { + pub fn new() -> Self { + Self(Uuid::new_v4().to_string()) + } + + pub fn parse(value: impl Into) -> Result { + let value = value.into(); + if value.trim().is_empty() { + return Err(concat!(stringify!($name), " cannot be empty").to_owned()); + } + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + + impl Default for $name { + fn default() -> Self { + Self::new() + } + } + + impl fmt::Display for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + Self::parse(value).map_err(serde::de::Error::custom) + } + } + }; +} + +opaque_id!(ClientId); +opaque_id!(AppId); +opaque_id!(WindowId); +opaque_id!(ObservationId); +opaque_id!(SurfaceId); +opaque_id!(ElementId); +opaque_id!(ActionId); +opaque_id!(AxRevision); +opaque_id!(MenuId); +opaque_id!(MenuRevision); +opaque_id!(GeometryRevision); +opaque_id!(CaptureRevision); + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct WindowGeneration(pub u64); + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum AppSelector { + Name { + name: String, + }, + BundleId { + bundle_id: String, + }, + Executable { + path: String, + #[serde(default)] + arguments: Vec, + }, +} + +impl AppSelector { + pub fn validate(&self) -> Result<(), String> { + match self { + Self::Name { name } => validate_non_empty("app name", name), + Self::BundleId { bundle_id } => validate_non_empty("bundle_id", bundle_id), + Self::Executable { path, .. } => validate_non_empty("executable path", path), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AppQuery { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name_contains: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub running: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AppRef { + pub id: AppId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pid: Option, + pub running: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WindowRef { + pub id: WindowId, + pub app: AppRef, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub title: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Point { + pub x: f64, + pub y: f64, +} + +impl Point { + pub fn validate(&self) -> Result<(), String> { + if self.x.is_finite() && self.y.is_finite() { + Ok(()) + } else { + Err("point coordinates must be finite".to_owned()) + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Size { + pub width: u32, + pub height: u32, +} + +impl Size { + pub fn validate(&self) -> Result<(), String> { + if self.width == 0 || self.height == 0 { + Err("size width and height must be nonzero".to_owned()) + } else { + Ok(()) + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Rect { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +impl Rect { + pub fn validate(&self) -> Result<(), String> { + if !self.x.is_finite() + || !self.y.is_finite() + || !self.width.is_finite() + || !self.height.is_finite() + { + return Err("rectangle geometry must be finite".to_owned()); + } + if self.width <= 0.0 || self.height <= 0.0 { + return Err("rectangle width and height must be positive".to_owned()); + } + Ok(()) + } + + pub fn contains(&self, point: Point) -> bool { + point.x >= self.x + && point.y >= self.y + && point.x < self.x + self.width + && point.y < self.y + self.height + } + + pub fn center(&self) -> Point { + Point { + x: self.x + self.width / 2.0, + y: self.y + self.height / 2.0, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SurfaceKind { + Window, + Menu, + Popover, + Sheet, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct CapturedSurface { + pub id: SurfaceId, + pub owner_window: WindowRef, + pub kind: SurfaceKind, + pub image_url: String, + pub size: Size, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub window_bounds: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ElementRef { + pub observation_id: ObservationId, + pub id: ElementId, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AccessibilityElement { + #[serde(rename = "ref")] + pub element_ref: ElementRef, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub label: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub value: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub bounds: Option, + #[serde(default)] + pub actions: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ReplaceAxLines { + pub start_line: usize, + pub delete_count: usize, + #[serde(default)] + pub lines: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum AxTreeUpdate { + Full { + revision: AxRevision, + tree: String, + }, + Diff { + base_revision: AxRevision, + revision: AxRevision, + operations: Vec, + }, +} + +impl AxTreeUpdate { + pub fn revision(&self) -> &AxRevision { + match self { + Self::Full { revision, .. } | Self::Diff { revision, .. } => revision, + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct AccessibilityState { + pub tree_update: AxTreeUpdate, + pub elements: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub focused_element: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub selected_text: Option, + #[serde(default)] + pub selected_elements: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub document_text: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WindowState { + pub observation_id: ObservationId, + pub window: WindowRef, + pub surfaces: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub accessibility: Option, + pub menu: MenuState, + pub settlement: SettlementEvidence, + pub captured_at_unix_ms: u64, + #[serde(default)] + pub warnings: Vec, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum MouseButton { + #[default] + Left, + Right, + Middle, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Modifier { + Shift, + Control, + Alt, + Meta, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ClickTarget { + Point { + observation_id: ObservationId, + surface_id: SurfaceId, + point: Point, + }, + Element { + element: ElementRef, + }, +} + +impl ClickTarget { + pub fn observation_id(&self) -> &ObservationId { + match self { + Self::Point { observation_id, .. } => observation_id, + Self::Element { element } => &element.observation_id, + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ClickRequest { + pub target: ClickTarget, + #[serde(default)] + pub button: MouseButton, + #[serde(default = "default_click_count")] + pub click_count: u8, + #[serde(default)] + pub modifiers: Vec, +} + +fn default_click_count() -> u8 { + 1 +} + +impl ClickRequest { + pub fn validate(&self) -> Result<(), String> { + if !(1..=3).contains(&self.click_count) { + return Err("click_count must be between 1 and 3".to_owned()); + } + if let ClickTarget::Point { point, .. } = &self.target { + point.validate()?; + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct DragRequest { + pub observation_id: ObservationId, + pub surface_id: SurfaceId, + pub start: Point, + pub end: Point, + #[serde(default)] + pub button: MouseButton, + #[serde(default)] + pub modifiers: Vec, + #[serde(default = "default_drag_duration_ms")] + pub duration_ms: u32, +} + +fn default_drag_duration_ms() -> u32 { + 300 +} + +impl DragRequest { + pub fn validate(&self) -> Result<(), String> { + if self.duration_ms > 10_000 { + return Err("duration_ms must be at most 10000".to_owned()); + } + self.start.validate()?; + self.end.validate()?; + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ScrollDirection { + Up, + Down, + Left, + Right, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum ScrollRequest { + Delta { + observation_id: ObservationId, + surface_id: SurfaceId, + point: Point, + #[serde(default)] + delta_x: f64, + #[serde(default)] + delta_y: f64, + }, + Element { + element: ElementRef, + direction: ScrollDirection, + #[serde(default = "default_pages")] + pages: u16, + }, +} + +fn default_pages() -> u16 { + 1 +} + +impl ScrollRequest { + pub fn observation_id(&self) -> &ObservationId { + match self { + Self::Delta { observation_id, .. } => observation_id, + Self::Element { element, .. } => &element.observation_id, + } + } + + pub fn validate(&self) -> Result<(), String> { + match self { + Self::Delta { + point, + delta_x, + delta_y, + .. + } => { + point.validate()?; + if !delta_x.is_finite() || !delta_y.is_finite() { + return Err("delta_x and delta_y must be finite".to_owned()); + } + if *delta_x == 0.0 && *delta_y == 0.0 { + return Err("delta_x and delta_y cannot both be zero".to_owned()); + } + Ok(()) + } + Self::Element { pages, .. } if !(1..=100).contains(pages) => { + Err("pages must be between 1 and 100".to_owned()) + } + _ => Ok(()), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct KeyStroke { + pub key: String, + #[serde(default)] + pub modifiers: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PressKeyRequest { + pub observation_id: ObservationId, + pub stroke: KeyStroke, +} + +impl PressKeyRequest { + pub fn validate(&self) -> Result<(), String> { + validate_non_empty("key", &self.stroke.key) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct TypeTextRequest { + pub observation_id: ObservationId, + pub text: String, +} + +impl TypeTextRequest { + pub fn validate(&self) -> Result<(), String> { + if self.text.is_empty() { + Err("text cannot be empty".to_owned()) + } else { + Ok(()) + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SetValueRequest { + pub element: ElementRef, + pub value: String, +} + +impl SetValueRequest { + /// Empty values are intentionally valid because they clear a control. + pub fn validate(&self) -> Result<(), String> { + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SelectionType { + #[default] + Text, + CursorBefore, + CursorAfter, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SelectTextRequest { + pub element: ElementRef, + pub text: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prefix: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub suffix: Option, + #[serde(default)] + pub selection_type: SelectionType, +} + +impl SelectTextRequest { + pub fn validate(&self) -> Result<(), String> { + if self.text.is_empty() { + return Err("selection text/anchor cannot be empty".to_owned()); + } + if self.prefix.as_ref().is_some_and(String::is_empty) + || self.suffix.as_ref().is_some_and(String::is_empty) + { + return Err("selection prefix/suffix cannot be an empty string".to_owned()); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SecondaryActionRequest { + pub element: ElementRef, + pub action: String, +} + +impl SecondaryActionRequest { + pub fn validate(&self) -> Result<(), String> { + validate_non_empty("secondary action", &self.action) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum Route { + Semantic, + TargetedPointer, + TargetedKeyboard, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum VerificationLevel { + EffectVerified, + DispatchVerified, + DispatchUnverified, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum EffectVerification { + EffectVerified, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ActionReceipt { + pub action_id: ActionId, + pub window: WindowRef, + pub consumed_observation_id: ObservationId, + pub route: Route, + pub verification: VerificationLevel, + pub posture: PostureResult, + pub settlement: SettlementEvidence, + #[serde(default)] + pub native_evidence: NativeEvidence, + #[serde(default)] + pub warnings: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LaunchResult { + pub action_id: ActionId, + pub app: AppRef, + pub windows: Vec, + pub reused_running_app: bool, + pub verification: EffectVerification, + pub posture: PostureResult, + pub settlement: SettlementEvidence, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PermissionState { + Granted, + Denied, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct Readiness { + pub ready: bool, + #[serde(default)] + pub permissions: BTreeMap, + #[serde(default)] + pub diagnostics: BTreeMap, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum AxTreeMode { + #[default] + DiffIfAvailable, + Full, +} + +macro_rules! empty_request { + ($name:ident) => { + #[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + pub struct $name {} + }; +} + +empty_request!(CheckReadinessRequest); +empty_request!(GetCapabilitiesRequest); + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ListAppsRequest { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub query: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct LaunchAppRequest { + pub app: AppSelector, +} + +impl LaunchAppRequest { + pub fn validate(&self) -> Result<(), String> { + self.app.validate() + } +} + +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ListWindowsRequest { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GetWindowRequest { + pub window_id: WindowId, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct GetWindowStateRequest { + pub window: WindowRef, + #[serde(default = "default_true")] + pub include_text: bool, + #[serde(default = "default_true")] + pub include_screenshots: bool, + #[serde(default)] + pub ax_tree_mode: AxTreeMode, +} + +fn default_true() -> bool { + true +} + +fn validate_non_empty(label: &str, value: &str) -> Result<(), String> { + if value.trim().is_empty() { + Err(format!("{label} cannot be empty")) + } else { + Ok(()) + } +} + +macro_rules! action_command { + ($name:ident, $request:ty) => { + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] + #[serde(deny_unknown_fields)] + pub struct $name { + pub window: WindowRef, + pub request: $request, + } + }; +} + +action_command!(ClickCommand, ClickRequest); +action_command!(DragCommand, DragRequest); +action_command!(ScrollCommand, ScrollRequest); +action_command!(PressKeyCommand, PressKeyRequest); +action_command!(TypeTextCommand, TypeTextRequest); +action_command!(SetValueCommand, SetValueRequest); +action_command!(SelectTextCommand, SelectTextRequest); +action_command!(SecondaryActionCommand, SecondaryActionRequest); diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/controller.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/controller.rs new file mode 100644 index 0000000000..8576b5e162 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/controller.rs @@ -0,0 +1,1479 @@ +//! Process-wide v2 controller and invariant-preserving action pipeline. + +use std::{ + collections::HashMap, + sync::Arc, + time::{Duration, Instant}, +}; + +use tokio::sync::RwLock; + +use super::{ + capabilities::{ + ActionKind, AddressingMode, CapabilityKey, CapabilityRegistry, Framework, PlatformName, + RouteDecision, WindowStateKind, + }, + contracts::{ + ActionId, ActionReceipt, AppRef, AxTreeMode, ClickCommand, ClickTarget, ClientId, + DragCommand, EffectVerification, GetWindowStateRequest, LaunchAppRequest, LaunchResult, + ListAppsRequest, ListWindowsRequest, ObservationId, PressKeyCommand, Readiness, Route, + ScrollCommand, ScrollRequest, SecondaryActionCommand, SelectTextCommand, SetValueCommand, + TypeTextCommand, VerificationLevel, WindowRef, WindowState, + }, + errors::{ErrorCode, ErrorPhase, NativeError, PartialEvidence, PartialNativeDispatch}, + interaction::{MutationDeadline, ScopePlan, ScopeRequirements}, + observation::{ + revision_accessibility, InvalidationReason, ObservationRecord, ResolvedDrag, + ResolvedScroll, ResolvedWindow, + }, + platform::{ + ClickSpec, ElementScrollSpec, InteractionProvider, KeyboardActionProvider, + LaunchPostureScope, LifecycleProvider, NativeDispatch, ObservationProvider, ObserveRequest, + PlatformDriver, PointerActionProvider, ResolvedAction, SelectionSpec, + SemanticActionProvider, WindowProvider, + }, + settlement::{ + PendingSettlementEvidence, SettlementAttempt, SettlementEvidence, SettlementProfile, + SettlementSignal, + }, + target::{ + ProcessMutationLockRegistry, TargetControllerRegistry, TargetControllerState, TargetKey, + }, +}; + +const DEFAULT_TARGET_IDLE_TTL: Duration = Duration::from_secs(300); +const DEFAULT_LOCK_TIMEOUT: Duration = Duration::from_secs(2); +const DEFAULT_MUTATION_WORK_TIMEOUT: Duration = Duration::from_secs(10); +const DEFAULT_MUTATION_TEARDOWN_TIMEOUT: Duration = Duration::from_secs(2); + +pub struct DriverController { + platform: Arc

, + platform_name: PlatformName, + os_version: String, + capabilities: RwLock, + pub targets: Arc>, + lock_timeout: Duration, + mutation_work_timeout: Duration, + mutation_teardown_timeout: Duration, +} + +impl DriverController

{ + pub fn new( + platform: Arc

, + platform_name: PlatformName, + os_version: impl Into, + ) -> Self { + let process_locks = Arc::new(ProcessMutationLockRegistry::default()); + let os_version = os_version.into(); + let capabilities = CapabilityRegistry::from_cells(platform.capability_cells(&os_version)); + Self { + platform, + platform_name, + os_version, + capabilities: RwLock::new(capabilities), + targets: Arc::new(TargetControllerRegistry::new( + process_locks, + DEFAULT_TARGET_IDLE_TTL, + )), + lock_timeout: DEFAULT_LOCK_TIMEOUT, + mutation_work_timeout: DEFAULT_MUTATION_WORK_TIMEOUT, + mutation_teardown_timeout: DEFAULT_MUTATION_TEARDOWN_TIMEOUT, + } + } + + /// Overrides the controller-owned mutation budget. + /// + /// `work_timeout` covers preflight, scope acquisition, native dispatch, + /// and settlement. `teardown_timeout` is an additional reserve used only + /// to release acquired leases while containment remains active. + pub fn with_mutation_timeouts( + mut self, + work_timeout: Duration, + teardown_timeout: Duration, + ) -> Self { + self.mutation_work_timeout = work_timeout; + self.mutation_teardown_timeout = teardown_timeout; + self + } + + pub fn start_invalidation_loop(&self) -> tokio::task::JoinHandle<()> { + let targets = Arc::clone(&self.targets); + let platform = Arc::clone(&self.platform); + let subscription = self.platform.subscribe_invalidations(); + tokio::spawn(async move { + targets.invalidation_loop(platform, subscription).await; + }) + } + + pub async fn check_readiness(&self) -> Result { + self.platform.lifecycle().readiness().await + } + + pub async fn get_capabilities( + &self, + ) -> Result { + let mut manifest = self.platform.lifecycle().capabilities().await?; + manifest.platform = self.platform_name.clone(); + manifest.cells = self.capabilities.read().await.cells().collect(); + Ok(manifest) + } + + pub async fn list_apps(&self, request: ListAppsRequest) -> Result, NativeError> { + self.platform + .lifecycle() + .list_apps(request.query.unwrap_or_default()) + .await + } + + pub async fn launch_app(&self, request: LaunchAppRequest) -> Result { + request.validate().map_err(NativeError::invalid)?; + let action_id = ActionId::new(); + let mut posture_scope = LaunchPostureScope::for_action(action_id.clone()); + let launch = match self + .platform + .lifecycle() + .launch_background(request.app, &mut posture_scope) + .await + { + Ok(launch) => launch, + Err(mut error) => { + if posture_scope.side_effect_started() { + error.partial_evidence = Some(Box::new(PartialEvidence::Launch { + action_id, + app: posture_scope.partial_app, + windows: posture_scope.partial_windows, + posture: posture_scope.posture, + native_evidence: posture_scope.native_evidence, + pending_settlement: posture_scope.pending_settlement.map(Box::new), + })); + } + return Err(error); + } + }; + if !launch.posture.held { + let mut error = NativeError::new( + ErrorCode::PostureViolated, + ErrorPhase::Verify, + false, + "background launch changed the user's foreground posture", + ) + .with_detail("action_id", action_id.to_string()); + error.partial_evidence = Some(Box::new(PartialEvidence::Launch { + action_id, + app: Some(launch.app), + windows: launch.windows, + posture: launch.posture, + native_evidence: posture_scope.native_evidence, + pending_settlement: None, + })); + return Err(error); + } + Ok(LaunchResult { + action_id, + app: launch.app, + windows: launch.windows, + reused_running_app: launch.reused_running_app, + verification: EffectVerification::EffectVerified, + posture: launch.posture, + settlement: launch.settlement, + }) + } + + pub async fn list_windows( + &self, + request: ListWindowsRequest, + ) -> Result, NativeError> { + self.platform + .windows() + .list_windows(request.app.as_ref()) + .await + } + + pub async fn get_window( + &self, + request: super::contracts::GetWindowRequest, + ) -> Result { + self.platform + .windows() + .rehydrate(&request.window_id, request.app.as_ref()) + .await + } + + pub async fn get_window_state( + &self, + client_id: &ClientId, + request: GetWindowStateRequest, + ) -> Result { + let resolved = self.platform.windows().resolve(&request.window).await?; + ensure_public_window_matches(&request.window, &resolved)?; + let key = TargetKey::from_window(client_id.clone(), &resolved); + let target = self + .targets + .get_or_create(&self.platform, key, resolved.clone()) + .await?; + let _process_guard = tokio::time::timeout( + self.lock_timeout, + Arc::clone(&target.mutation_lock).lock_owned(), + ) + .await + .map_err(|_| target_busy(&resolved))?; + target.ensure_valid()?; + let mut state = target.state.lock().await; + ensure_target_window_matches(&state.window, &resolved)?; + let settlement = self.settle_if_dirty(&mut state, true, None).await?; + let native = self + .platform + .observation() + .observe( + &mut state.platform, + &resolved, + ObserveRequest { + include_text: request.include_text, + include_screenshots: request.include_screenshots, + }, + ) + .await?; + native.window.geometry.validate()?; + // A coherent platform retry may legitimately return a newer geometry + // revision. Preserve every stable identity component and accept that + // refreshed geometry only after the platform completed A/capture/B. + ensure_target_window_matches(&resolved, &native.window)?; + let observed_window = native.window.clone(); + state.window = observed_window.clone(); + + let observation_id = ObservationId::new(); + let tree_bytes = native + .accessibility + .as_ref() + .map(|accessibility| accessibility.normalized_tree.len()) + .unwrap_or(0); + let revisioned = native + .accessibility + .map(|accessibility| { + revision_accessibility( + &state.ax_revisions, + &observation_id, + accessibility, + request.ax_tree_mode, + ) + }) + .transpose()?; + + let mut surfaces = HashMap::new(); + for surface in native.surfaces { + surface.validate_for_window(&observed_window)?; + if surfaces.insert(surface.id.clone(), surface).is_some() { + return Err(NativeError::invalid( + "native observation returned duplicate surface ids", + )); + } + } + if let super::menu::NativeMenuObservation::Open { surface_ids, .. } = &native.menu { + if surface_ids.iter().any(|id| !surfaces.contains_key(id)) { + return Err(NativeError::stale( + ErrorCode::MenuStateStale, + "native menu observation referenced a surface outside this observation", + )); + } + } + state + .menu + .reconcile_observation(native.menu, &observation_id)?; + let menu = state.menu.observation()?; + let public_surfaces = surfaces.values().map(|surface| surface.public()).collect(); + let accessibility = revisioned.as_ref().map(|state| state.public.clone()); + let approximate_bytes = tree_bytes + + surfaces + .values() + .map(|surface| surface.approximate_bytes) + .sum::(); + let public = WindowState { + observation_id: observation_id.clone(), + window: observed_window.public.clone(), + surfaces: public_surfaces, + accessibility, + menu: menu.clone(), + settlement: settlement.clone(), + captured_at_unix_ms: native.captured_at_unix_ms, + warnings: native.warnings, + }; + let accessibility_record = revisioned.as_ref().map(|state| state.record.clone()); + state.observations.insert(ObservationRecord { + id: observation_id, + window: observed_window.stamp(), + captured_at: Instant::now(), + surfaces, + accessibility: accessibility_record, + menu, + settlement, + state: super::observation::ObservationState::Current, + approximate_bytes, + artifacts: native.artifacts, + })?; + if let Some(revisioned) = revisioned { + state.ax_revisions.commit(revisioned.prepared_revision); + } + target.touch(); + Ok(public) + } + + pub async fn click( + &self, + client_id: &ClientId, + command: ClickCommand, + ) -> Result { + command.request.validate().map_err(NativeError::invalid)?; + let observation_id = command.request.target.observation_id().clone(); + let addressing = match command.request.target { + ClickTarget::Point { .. } => AddressingMode::CapturedPoint, + ClickTarget::Element { .. } => AddressingMode::Element, + }; + let menu_opening = command.request.button == super::contracts::MouseButton::Right; + self.mutate( + client_id, + command.window, + observation_id, + ActionKind::Click, + addressing, + menu_opening, + false, + move |state, window| match command.request.target { + ClickTarget::Point { + observation_id, + surface_id, + point, + } => Ok(ResolvedAction::PointClick { + point: state.observations.resolve_point( + window, + &observation_id, + &surface_id, + point, + )?, + spec: ClickSpec { + button: command.request.button, + click_count: command.request.click_count, + modifiers: command.request.modifiers, + }, + }), + ClickTarget::Element { element } => Ok(ResolvedAction::ElementClick { + source: element.clone(), + element: state.observations.resolve_element(window, &element)?, + spec: ClickSpec { + button: command.request.button, + click_count: command.request.click_count, + modifiers: command.request.modifiers, + }, + }), + }, + ) + .await + } + + pub async fn drag( + &self, + client_id: &ClientId, + command: DragCommand, + ) -> Result { + command.request.validate().map_err(NativeError::invalid)?; + let observation_id = command.request.observation_id.clone(); + self.mutate( + client_id, + command.window, + observation_id.clone(), + ActionKind::Drag, + AddressingMode::CapturedPoint, + false, + false, + move |state, window| { + let start = state.observations.resolve_point( + window, + &observation_id, + &command.request.surface_id, + command.request.start, + )?; + let end = state.observations.resolve_point( + window, + &observation_id, + &command.request.surface_id, + command.request.end, + )?; + Ok(ResolvedAction::Drag(ResolvedDrag { + start, + end, + duration_ms: command.request.duration_ms, + button: command.request.button, + modifiers: command.request.modifiers, + })) + }, + ) + .await + } + + pub async fn scroll( + &self, + client_id: &ClientId, + command: ScrollCommand, + ) -> Result { + command.request.validate().map_err(NativeError::invalid)?; + let observation_id = command.request.observation_id().clone(); + let addressing = match command.request { + ScrollRequest::Delta { .. } => AddressingMode::CapturedPoint, + ScrollRequest::Element { .. } => AddressingMode::Element, + }; + self.mutate( + client_id, + command.window, + observation_id, + ActionKind::Scroll, + addressing, + false, + false, + move |state, window| match command.request { + ScrollRequest::Delta { + observation_id, + surface_id, + point, + delta_x, + delta_y, + } => Ok(ResolvedAction::DeltaScroll(ResolvedScroll { + point: state.observations.resolve_point( + window, + &observation_id, + &surface_id, + point, + )?, + delta_x, + delta_y, + })), + ScrollRequest::Element { + element, + direction, + pages, + } => Ok(ResolvedAction::ElementScroll { + element: state.observations.resolve_element(window, &element)?, + spec: ElementScrollSpec { direction, pages }, + }), + }, + ) + .await + } + + pub async fn press_key( + &self, + client_id: &ClientId, + command: PressKeyCommand, + ) -> Result { + command.request.validate().map_err(NativeError::invalid)?; + let observation_id = command.request.observation_id.clone(); + self.mutate( + client_id, + command.window, + observation_id.clone(), + ActionKind::PressKey, + AddressingMode::ObservedFocus, + false, + false, + move |state, window| { + let focus = state.observations.validate_focus(window, &observation_id)?; + Ok(ResolvedAction::PressKey { + focus, + stroke: command.request.stroke, + }) + }, + ) + .await + } + + pub async fn type_text( + &self, + client_id: &ClientId, + command: TypeTextCommand, + ) -> Result { + command.request.validate().map_err(NativeError::invalid)?; + let observation_id = command.request.observation_id.clone(); + self.mutate( + client_id, + command.window, + observation_id.clone(), + ActionKind::TypeText, + AddressingMode::ObservedFocus, + false, + false, + move |state, window| { + let focus = state.observations.validate_focus(window, &observation_id)?; + Ok(ResolvedAction::TypeText { + focus, + text: command.request.text, + }) + }, + ) + .await + } + + pub async fn set_value( + &self, + client_id: &ClientId, + command: SetValueCommand, + ) -> Result { + command.request.validate().map_err(NativeError::invalid)?; + let observation_id = command.request.element.observation_id.clone(); + self.mutate( + client_id, + command.window, + observation_id, + ActionKind::SetValue, + AddressingMode::Element, + false, + true, + move |state, window| { + Ok(ResolvedAction::SetValue { + element: state + .observations + .resolve_element(window, &command.request.element)?, + value: command.request.value, + }) + }, + ) + .await + } + + pub async fn select_text( + &self, + client_id: &ClientId, + command: SelectTextCommand, + ) -> Result { + command.request.validate().map_err(NativeError::invalid)?; + let observation_id = command.request.element.observation_id.clone(); + self.mutate( + client_id, + command.window, + observation_id, + ActionKind::SelectText, + AddressingMode::Element, + false, + true, + move |state, window| { + Ok(ResolvedAction::SelectText { + element: state + .observations + .resolve_element(window, &command.request.element)?, + selection: SelectionSpec { + text: command.request.text, + prefix: command.request.prefix, + suffix: command.request.suffix, + selection_type: command.request.selection_type, + }, + }) + }, + ) + .await + } + + pub async fn perform_secondary_action( + &self, + client_id: &ClientId, + command: SecondaryActionCommand, + ) -> Result { + command.request.validate().map_err(NativeError::invalid)?; + let observation_id = command.request.element.observation_id.clone(); + let menu_opening = is_menu_action(&command.request.action); + self.mutate( + client_id, + command.window, + observation_id, + ActionKind::PerformSecondaryAction, + AddressingMode::Element, + menu_opening, + false, + move |state, window| { + let element = state + .observations + .resolve_element(window, &command.request.element)?; + if !element + .actions + .iter() + .any(|action| action == &command.request.action) + { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "secondary action was not exposed by the observed element", + )); + } + Ok(ResolvedAction::Secondary { + element, + action: command.request.action, + }) + }, + ) + .await + } + + pub async fn close_connection(&self, client_id: &ClientId) -> Result { + self.targets + .close_connection(&self.platform, client_id) + .await + } + + pub async fn expire_idle_targets(&self) -> Result { + self.targets.expire_idle(&self.platform).await + } + + #[allow(clippy::too_many_arguments)] + async fn mutate( + &self, + client_id: &ClientId, + public_window: WindowRef, + observation_id: ObservationId, + action: ActionKind, + addressing: AddressingMode, + menu_opening: bool, + requires_effect_verification: bool, + prepare: F, + ) -> Result + where + F: FnOnce( + &mut TargetControllerState

, + &ResolvedWindow, + ) -> Result, + { + let resolved = self.platform.windows().resolve(&public_window).await?; + ensure_public_window_matches(&public_window, &resolved)?; + let key = TargetKey::from_window(client_id.clone(), &resolved); + let target = self.targets.get(&key).await?; + let _process_guard = tokio::time::timeout( + self.lock_timeout, + Arc::clone(&target.mutation_lock).lock_owned(), + ) + .await + .map_err(|_| target_busy(&resolved))?; + target.ensure_valid()?; + let mut state = target.state.lock().await; + ensure_target_window_matches(&state.window, &resolved)?; + if state.settlement.settled_evidence().is_none() { + let mut error = NativeError::new( + ErrorCode::UiNotSettled, + ErrorPhase::Settle, + true, + "target is still dirty; observe it to resume settlement before another mutation", + ); + error.pending_settlement = state.settlement.pending_evidence().map(Box::new); + return Err(error); + } + // Freshness and handle resolution precede capability routing so an + // unsupported cell cannot hide a stale/cross-window request. + let prepared = prepare(&mut state, &resolved)?; + validate_current_menu_target(&state, &prepared)?; + let targeted_menu_id = resolved_action_menu_id(&prepared).cloned(); + let capability_key = CapabilityKey { + platform: self.platform_name.clone(), + os_version: self.os_version.clone(), + action: action.clone(), + addressing, + framework: resolved.framework.clone(), + window_state: resolved.state.clone(), + }; + let route = match self.capabilities.read().await.decision(&capability_key) { + RouteDecision::Supported { route } => route, + RouteDecision::Unsupported { reason } => { + return Err(NativeError::unsupported(reason) + .with_detail("action", format!("{action:?}")) + .with_detail("framework", format!("{:?}", resolved.framework))) + } + }; + let prepared = adapt_action_route(&mut state, &resolved, route, prepared)?; + let action_id = ActionId::new(); + let deadline = + mutation_deadline(self.mutation_work_timeout, self.mutation_teardown_timeout)?; + let mut requirements = ScopeRequirements::for_route(route); + requirements.menu_dismissal = menu_opening; + let preflight_result = { + let TargetControllerState { + platform, focus, .. + } = &mut *state; + tokio::time::timeout_at( + deadline.work.into(), + self.platform.interaction().preflight( + platform, + focus, + &action_id, + &resolved, + route, + &prepared, + deadline, + requirements, + ), + ) + .await + }; + let mut scope_plan = match preflight_result { + Ok(result) => result?, + Err(_) => { + return Err(mutation_deadline_error( + &action_id, + ErrorPhase::Preflight, + "preflight", + )); + } + }; + ensure_scope_plan_matches(&scope_plan, &action_id, &resolved, route, deadline)?; + let pending_menu_id = menu_opening.then(|| { + state + .menu + .begin_open(action_id.clone(), resolved.public.clone(), resolved.stamp()) + }); + if let Some(menu_id) = pending_menu_id.clone() { + scope_plan.bind_opening_menu(menu_id); + } + let cursor = state.logical_cursor.clone(); + let scope_result = { + let TargetControllerState { + platform, focus, .. + } = &mut *state; + tokio::time::timeout_at( + deadline.work.into(), + self.platform + .interaction() + .acquire_scope(platform, focus, scope_plan, cursor), + ) + .await + }; + let mut scope = match scope_result { + Ok(Ok(scope)) => scope, + Ok(Err(error)) => { + if menu_opening { + state.menu.close(); + } + return Err(error); + } + Err(_) => { + if menu_opening { + state.menu.close(); + } + // The timed-out acquisition future has been dropped, so all + // provider-local RAII guards have had their cleanup chance. + // Core has no complete teardown evidence, though, and cannot + // safely reuse this target controller. + target.invalidate(); + drop(state); + let mut error = + mutation_deadline_error(&action_id, ErrorPhase::Preflight, "scope_acquisition") + .with_detail("cleanup_state", "uncertain"); + match tokio::time::timeout_at( + deadline.teardown.into(), + self.targets.remove_invalid_target(&self.platform, &key), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(removal_error)) => error = error.with_related(&removal_error), + Err(_) => { + error = error.with_related(&mutation_deadline_error( + &action_id, + ErrorPhase::Verify, + "poisoned_target_teardown", + )); + } + } + return Err(error); + } + }; + scope.bind_target_validity(target.validity_handle()); + let semantic_plan = if route == Route::Semantic { + let prepare_result = { + let platform = &mut state.platform; + tokio::time::timeout_at( + deadline.work.into(), + self.platform + .semantic() + .prepare(platform, &mut scope, &prepared), + ) + .await + }; + match prepare_result { + Ok(Ok(plan)) => Some(plan), + Ok(Err(error)) => { + let teardown = scope.release(); + let teardown_failed = !teardown.failures.is_empty(); + if teardown_failed { + target.invalidate(); + } + let mut failures = vec![error]; + failures.extend(teardown.failures); + if let Some(error) = NativeError::from_posture(&scope.posture) { + failures.push(error); + } + let scope_evidence = serde_json::to_value(&scope.native_evidence) + .expect("typed scope evidence must serialize"); + if teardown_failed { + drop(state); + match tokio::time::timeout_at( + deadline.teardown.into(), + self.targets.remove_invalid_target(&self.platform, &key), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => failures.push(error), + Err(_) => failures.push(mutation_deadline_error( + &action_id, + ErrorPhase::Verify, + "poisoned_target_teardown", + )), + } + } + let mut primary = NativeError::primary(failures) + .expect("semantic prepare failure is nonempty"); + primary + .details + .insert("interaction_scope".to_owned(), scope_evidence); + return Err(primary); + } + Err(_) => { + let mut failures = vec![mutation_deadline_error( + &action_id, + ErrorPhase::Preflight, + "semantic_prepare", + )]; + let teardown = scope.release(); + let teardown_failed = !teardown.failures.is_empty(); + if teardown_failed { + target.invalidate(); + } + failures.extend(teardown.failures); + if let Some(error) = NativeError::from_posture(&scope.posture) { + failures.push(error); + } + if teardown_failed { + drop(state); + match tokio::time::timeout_at( + deadline.teardown.into(), + self.targets.remove_invalid_target(&self.platform, &key), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => failures.push(error), + Err(_) => failures.push(mutation_deadline_error( + &action_id, + ErrorPhase::Verify, + "poisoned_target_teardown", + )), + } + } + return Err(NativeError::primary(failures) + .expect("semantic prepare timeout failure is nonempty")); + } + } + } else { + None + }; + if !menu_opening { + if let Some(menu_id) = &targeted_menu_id { + if let Err(error) = state.menu.begin_target(menu_id, action_id.clone()) { + let teardown = scope.release(); + let teardown_failed = !teardown.failures.is_empty(); + if teardown_failed { + target.invalidate(); + } + let mut failures = vec![error]; + failures.extend(teardown.failures); + if let Some(error) = NativeError::from_posture(&scope.posture) { + failures.push(error); + } + let scope_evidence = serde_json::to_value(&scope.native_evidence) + .expect("typed scope evidence must serialize"); + if teardown_failed { + drop(state); + match tokio::time::timeout_at( + deadline.teardown.into(), + self.targets.remove_invalid_target(&self.platform, &key), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => failures.push(error), + Err(_) => failures.push(mutation_deadline_error( + &action_id, + ErrorPhase::Verify, + "poisoned_target_teardown", + )), + } + } + let mut primary = NativeError::primary(failures).expect("non-empty failures"); + primary + .details + .insert("interaction_scope".to_owned(), scope_evidence); + return Err(primary); + } + } + } + + // Core owns the dispatch boundary. Once provider code is entered the + // observation is consumed and target dirty even if the task is + // cancelled while awaiting native dispatch. + state + .observations + .consume(&observation_id, action_id.clone())?; + state + .observations + .invalidate_all(InvalidationReason::MutationDispatched); + let profile = settlement_profile(&action, menu_opening, requires_effect_verification); + state.settlement.mark_dirty(action_id.clone(), profile)?; + let dispatch_result = { + let platform = &mut state.platform; + tokio::time::timeout_at( + deadline.work.into(), + self.dispatch(platform, &mut scope, prepared, semantic_plan), + ) + .await + }; + let mut failures = Vec::new(); + let mut dispatch_timed_out = false; + let dispatch = match dispatch_result { + Ok(result) => { + // This signal proves that provider control returned. A timed + // out/cancelled provider must never manufacture it. + state + .settlement + .record_signal(SettlementSignal::DispatchComplete)?; + match result { + Ok(dispatch) => Some(dispatch), + Err(error) => { + if menu_opening || targeted_menu_id.is_some() { + state.menu.close(); + } + failures.push(error); + None + } + } + } + Err(_) => { + dispatch_timed_out = true; + if menu_opening || targeted_menu_id.is_some() { + state.menu.close(); + } + failures.push(mutation_deadline_error( + &action_id, + ErrorPhase::Dispatch, + "dispatch", + )); + None + } + }; + + if let Some(menu_evidence) = dispatch + .as_ref() + .and_then(|dispatch| dispatch.menu.as_ref()) + { + if let Err(error) = + state + .menu + .record_dispatch_evidence(&action_id, &resolved.stamp(), menu_evidence) + { + state.menu.close(); + failures.push(error); + } + } else if (menu_opening || targeted_menu_id.is_some()) && dispatch.is_some() { + state.menu.close(); + failures.push(NativeError::stale( + ErrorCode::MenuStateStale, + "menu dispatch returned no exact native menu/action/owner identity evidence", + )); + } + + let mut settlement = None; + let mut pending_settlement = if dispatch_timed_out { + state.settlement.pending_evidence().map(Box::new) + } else { + None + }; + if !dispatch_timed_out { + match self + .settle_if_dirty(&mut state, false, Some(deadline.work)) + .await + { + Ok(evidence) => settlement = Some(evidence), + Err(error) => { + pending_settlement = error.pending_settlement.clone(); + failures.push(error); + } + } + } + + let teardown = scope.release(); + let teardown_failed = !teardown.failures.is_empty(); + if teardown_failed { + target.invalidate(); + } + failures.extend(teardown.failures); + if let Some(error) = NativeError::from_posture(&scope.posture) { + failures.push(error); + } + if requires_effect_verification + && dispatch + .as_ref() + .is_some_and(|dispatch| dispatch.verification != VerificationLevel::EffectVerified) + { + failures.push(NativeError::new( + ErrorCode::VerificationFailed, + ErrorPhase::Verify, + true, + "semantic mutation did not provide its promised exact readback", + )); + } + + let mut native_evidence = scope.native_evidence.clone(); + if let Some(dispatch) = &dispatch { + native_evidence + .fields + .extend(dispatch.evidence.fields.clone()); + } + let partial = PartialEvidence::Action { + action_id: action_id.clone(), + window: resolved.public.clone(), + consumed_observation_id: observation_id.clone(), + route, + dispatch: dispatch.as_ref().map(|dispatch| PartialNativeDispatch { + verification: dispatch.verification, + native_evidence: dispatch.evidence.clone(), + warnings: dispatch.warnings.clone(), + }), + posture: scope.posture.clone(), + native_evidence: scope.native_evidence.clone(), + pending_settlement, + }; + if teardown_failed { + drop(state); + match tokio::time::timeout_at( + deadline.teardown.into(), + self.targets.remove_invalid_target(&self.platform, &key), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => failures.push(error), + Err(_) => failures.push(mutation_deadline_error( + &action_id, + ErrorPhase::Verify, + "poisoned_target_teardown", + )), + } + } + if let Some(mut error) = NativeError::primary(failures) { + error.partial_evidence = Some(Box::new(partial)); + return Err(error); + } + + let receipt = match (dispatch, settlement) { + (Some(dispatch), Some(settlement)) => Some(ActionReceipt { + action_id, + window: resolved.public, + consumed_observation_id: observation_id, + route, + verification: dispatch.verification, + posture: scope.posture.clone(), + settlement, + native_evidence, + warnings: dispatch.warnings, + }), + _ => None, + }; + target.touch(); + receipt.ok_or_else(|| { + NativeError::new( + ErrorCode::Internal, + ErrorPhase::Verify, + false, + "mutation completed without either a receipt or a typed failure", + ) + }) + } + + async fn settle_if_dirty( + &self, + state: &mut TargetControllerState

, + resumed_from_prior_call: bool, + mutation_work_deadline: Option, + ) -> Result { + let Some(dirty) = state.settlement.begin(resumed_from_prior_call) else { + return state + .settlement + .settled_evidence() + .cloned() + .ok_or_else(|| NativeError::invalid("settlement state has no settled evidence")); + }; + let profile_deadline = Instant::now() + Duration::from_millis(dirty.profile.deadline_ms); + let deadline = mutation_work_deadline + .map(|mutation_deadline| profile_deadline.min(mutation_deadline)) + .unwrap_or(profile_deadline); + let settlement_result = tokio::time::timeout_at( + deadline.into(), + self.platform + .observation() + .settle(&mut state.platform, &dirty, deadline), + ) + .await; + match settlement_result { + Err(_) => { + let pending: PendingSettlementEvidence = dirty.pending_evidence(); + state.settlement.preserve_pending(&pending)?; + let mut error = NativeError::new( + ErrorCode::UiNotSettled, + ErrorPhase::Settle, + true, + "native settlement exceeded the controller-owned deadline", + ) + .with_detail("deadline_stage", "settlement"); + error.pending_settlement = Some(Box::new(pending)); + Err(error) + } + Ok(Ok(SettlementAttempt::Settled(evidence))) => state.settlement.complete(evidence), + Ok(Ok(SettlementAttempt::Pending(pending))) => { + state.settlement.preserve_pending(&pending)?; + let mut error = NativeError::new( + ErrorCode::UiNotSettled, + ErrorPhase::Settle, + true, + "target did not settle before the action-specific deadline", + ); + error.pending_settlement = Some(Box::new(pending)); + Err(error) + } + Ok(Err(mut error)) => { + let pending: PendingSettlementEvidence = dirty.pending_evidence(); + state.settlement.preserve_pending(&pending)?; + error = NativeError::new( + ErrorCode::UiNotSettled, + ErrorPhase::Settle, + true, + format!("native settlement failed: {}", error.message), + ) + .with_related(&error); + error.pending_settlement = Some(Box::new(pending)); + Err(error) + } + } + } + + async fn dispatch( + &self, + target: &mut P::TargetState, + scope: &mut super::interaction::InteractionScope, + mutation: ResolvedAction, + semantic_plan: Option< + >::PreparedAction, + >, + ) -> Result { + if scope.route == Route::Semantic { + let plan = semantic_plan.ok_or_else(|| { + NativeError::new( + ErrorCode::Internal, + ErrorPhase::Dispatch, + false, + "semantic dispatch entered without its retained prepare plan", + ) + })?; + return self.platform.semantic().dispatch(target, scope, plan).await; + } + if semantic_plan.is_some() { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Dispatch, + false, + "non-semantic dispatch received a semantic prepare plan", + )); + } + match mutation { + ResolvedAction::PointClick { point, spec } => { + self.platform.pointer().click(scope, point, spec).await + } + ResolvedAction::Drag(drag) => self.platform.pointer().drag(scope, drag).await, + ResolvedAction::DeltaScroll(scroll) => { + self.platform.pointer().scroll(scope, scroll).await + } + ResolvedAction::PressKey { focus, stroke } => { + self.platform + .keyboard() + .press_key(scope, &focus, stroke) + .await + } + ResolvedAction::TypeText { focus, text } => { + self.platform + .keyboard() + .type_text(scope, &focus, &text) + .await + } + ResolvedAction::ElementClick { .. } + | ResolvedAction::ElementScroll { .. } + | ResolvedAction::SetValue { .. } + | ResolvedAction::SelectText { .. } + | ResolvedAction::Secondary { .. } => Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Dispatch, + false, + "element semantic action reached a non-semantic dispatch route", + )), + } + } +} + +fn adapt_action_route( + state: &mut TargetControllerState

, + window: &ResolvedWindow, + route: Route, + action: ResolvedAction, +) -> Result { + match action { + ResolvedAction::ElementClick { + source, + element, + spec, + } if route == Route::Semantic => Ok(ResolvedAction::ElementClick { + source, + element, + spec, + }), + ResolvedAction::ElementClick { source, spec, .. } if route == Route::TargetedPointer => { + Ok(ResolvedAction::PointClick { + point: state.observations.resolve_element_point(window, &source)?, + spec, + }) + } + ResolvedAction::PointClick { .. } | ResolvedAction::Drag(_) + if route == Route::TargetedPointer => + { + Ok(action) + } + ResolvedAction::ElementScroll { .. } + | ResolvedAction::SetValue { .. } + | ResolvedAction::SelectText { .. } + | ResolvedAction::Secondary { .. } + if route == Route::Semantic => + { + Ok(action) + } + ResolvedAction::DeltaScroll(_) + if matches!(route, Route::Semantic | Route::TargetedPointer) => + { + Ok(action) + } + ResolvedAction::PressKey { .. } if route == Route::TargetedKeyboard => Ok(action), + ResolvedAction::TypeText { .. } + if matches!(route, Route::Semantic | Route::TargetedKeyboard) => + { + Ok(action) + } + _ => Err(NativeError::unsupported(format!( + "resolved action is incompatible with selected route {route:?}" + ))), + } +} + +fn validate_current_menu_target( + state: &TargetControllerState

, + action: &ResolvedAction, +) -> Result<(), NativeError> { + let menu_id = match action { + ResolvedAction::ElementClick { element, .. } + | ResolvedAction::ElementScroll { element, .. } + | ResolvedAction::SetValue { element, .. } + | ResolvedAction::SelectText { element, .. } + | ResolvedAction::Secondary { element, .. } => element.menu_id.as_ref(), + _ => None, + }; + if let Some(menu_id) = menu_id { + state.menu.validate_current_menu_id(menu_id)?; + } + Ok(()) +} + +fn resolved_action_menu_id(action: &ResolvedAction) -> Option<&super::contracts::MenuId> { + match action { + ResolvedAction::ElementClick { element, .. } + | ResolvedAction::ElementScroll { element, .. } + | ResolvedAction::SetValue { element, .. } + | ResolvedAction::SelectText { element, .. } + | ResolvedAction::Secondary { element, .. } => element.menu_id.as_ref(), + _ => None, + } +} + +fn ensure_public_window_matches( + requested: &WindowRef, + resolved: &ResolvedWindow, +) -> Result<(), NativeError> { + if requested.id != resolved.public.id || requested.app.id != resolved.public.app.id { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "resolved window does not match the supplied app/window identity", + )); + } + resolved.geometry.validate()?; + Ok(()) +} + +fn ensure_target_window_matches( + target: &ResolvedWindow, + resolved: &ResolvedWindow, +) -> Result<(), NativeError> { + if target.public.id != resolved.public.id + || target.public.app.id != resolved.public.app.id + || target.generation != resolved.generation + || target.process != resolved.process + || target.native != resolved.native + { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "live target no longer matches the target controller identity/generation", + )); + } + Ok(()) +} + +fn ensure_scope_plan_matches( + plan: &ScopePlan, + action_id: &ActionId, + window: &ResolvedWindow, + route: Route, + deadline: MutationDeadline, +) -> Result<(), NativeError> { + if &plan.action_id != action_id + || plan.window.stamp() != window.stamp() + || plan.route != route + || plan.deadline != deadline + { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "interaction preflight returned a plan for a different action, route, target, or deadline", + ) + .with_detail("expected_action_id", action_id.to_string()) + .with_detail("planned_action_id", plan.action_id.to_string()) + .with_detail("expected_route", format!("{route:?}")) + .with_detail("planned_route", format!("{:?}", plan.route))); + } + Ok(()) +} + +fn mutation_deadline( + work_timeout: Duration, + teardown_timeout: Duration, +) -> Result { + let started_at = Instant::now(); + let work = started_at.checked_add(work_timeout).ok_or_else(|| { + NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "configured mutation work timeout exceeds the platform clock range", + ) + })?; + let teardown = work.checked_add(teardown_timeout).ok_or_else(|| { + NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "configured mutation teardown timeout exceeds the platform clock range", + ) + })?; + MutationDeadline::new(work, teardown) +} + +fn mutation_deadline_error( + action_id: &ActionId, + phase: ErrorPhase, + stage: &'static str, +) -> NativeError { + NativeError::new( + ErrorCode::DispatchFailed, + phase, + false, + format!("mutation exceeded the controller-owned deadline during {stage}"), + ) + .with_detail("action_id", action_id.to_string()) + .with_detail("deadline_stage", stage) +} + +fn settlement_profile( + action: &ActionKind, + menu_opening: bool, + exact_readback: bool, +) -> SettlementProfile { + if exact_readback { + return SettlementProfile::requiring( + format!("{action:?}_exact_readback").to_lowercase(), + [SettlementSignal::VerificationReadbackComplete], + ) + .with_relevant_signals([ + SettlementSignal::AxValueChanged, + SettlementSignal::FocusChanged, + SettlementSignal::VerificationReadbackComplete, + ]); + } + if menu_opening { + return SettlementProfile::requiring( + format!("{action:?}_menu_open").to_lowercase(), + [SettlementSignal::MenuOpened], + ) + .with_relevant_signals([ + SettlementSignal::MenuOpened, + SettlementSignal::MenuDismissed, + SettlementSignal::WindowListChanged, + SettlementSignal::FocusChanged, + ]); + } + let relevant = match action { + ActionKind::Click => vec![ + SettlementSignal::AxAction, + SettlementSignal::AxValueChanged, + SettlementSignal::FocusChanged, + SettlementSignal::WindowListChanged, + SettlementSignal::MenuOpened, + SettlementSignal::MenuDismissed, + ], + ActionKind::Drag => vec![ + SettlementSignal::AxAction, + SettlementSignal::AxValueChanged, + SettlementSignal::WindowGeometryChanged, + ], + ActionKind::Scroll => vec![ + SettlementSignal::ScrollChanged, + SettlementSignal::AxValueChanged, + ], + ActionKind::PressKey | ActionKind::TypeText | ActionKind::SelectText => vec![ + SettlementSignal::AxValueChanged, + SettlementSignal::FocusChanged, + ], + ActionKind::SetValue => vec![ + SettlementSignal::AxValueChanged, + SettlementSignal::VerificationReadbackComplete, + ], + ActionKind::PerformSecondaryAction => vec![ + SettlementSignal::AxAction, + SettlementSignal::MenuOpened, + SettlementSignal::MenuDismissed, + SettlementSignal::WindowListChanged, + ], + }; + SettlementProfile::dispatch_only(format!("{action:?}").to_lowercase()) + .with_relevant_signals(relevant) +} + +fn target_busy(window: &ResolvedWindow) -> NativeError { + NativeError::new( + ErrorCode::TargetBusy, + ErrorPhase::Preflight, + true, + "timed out acquiring the bounded process mutation lock", + ) + .with_detail("window_id", window.public.id.to_string()) + .with_detail("app_id", window.public.app.id.to_string()) +} + +fn is_menu_action(action: &str) -> bool { + matches!(action, "show_menu" | "AXShowMenu") +} + +#[allow(dead_code)] +fn _type_assertions(_: Framework, _: WindowStateKind, _: AxTreeMode) {} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/errors.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/errors.rs new file mode 100644 index 0000000000..6b0f7ab40d --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/errors.rs @@ -0,0 +1,241 @@ +//! Stable, phase-aware v2 error envelope. + +use std::{collections::BTreeMap, fmt}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::{ + contracts::{ActionId, AppRef, ObservationId, Route, VerificationLevel, WindowRef}, + interaction::{NativeEvidence, PostureResult}, + settlement::PendingSettlementEvidence, +}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ErrorPhase { + Validate, + Preflight, + Dispatch, + Settle, + Verify, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ErrorCode { + ProtocolMismatch, + InvalidRequest, + PermissionDenied, + AppNotFound, + AppLaunchFailed, + WindowNotFound, + WindowIdentityChanged, + ObservationStale, + ObservationRaced, + SurfaceStale, + ElementStale, + AxRevisionMismatch, + UiNotSettled, + MenuStateStale, + UnsupportedInBackground, + TargetBusy, + DispatchFailed, + VerificationFailed, + /// A required posture witness was unavailable, incomplete, or lagged, + /// and no foreground, key-window, or physical-cursor disturbance was + /// observed. This is distinct from an observed posture violation. + PostureUnverifiable, + PostureViolated, + Internal, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NativeError { + pub code: ErrorCode, + pub phase: ErrorPhase, + pub retryable: bool, + pub message: String, + #[serde(default)] + pub details: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub partial_evidence: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub pending_settlement: Option>, + #[serde(default)] + pub related_failures: Vec, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NativeErrorSummary { + pub code: ErrorCode, + pub phase: ErrorPhase, + pub message: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum PartialEvidence { + Action { + action_id: ActionId, + window: WindowRef, + consumed_observation_id: ObservationId, + route: Route, + #[serde(default, skip_serializing_if = "Option::is_none")] + dispatch: Option, + #[serde(default)] + posture: PostureResult, + #[serde(default)] + native_evidence: NativeEvidence, + #[serde(default, skip_serializing_if = "Option::is_none")] + pending_settlement: Option>, + }, + Launch { + action_id: ActionId, + #[serde(default, skip_serializing_if = "Option::is_none")] + app: Option, + #[serde(default)] + windows: Vec, + #[serde(default)] + posture: PostureResult, + #[serde(default)] + native_evidence: NativeEvidence, + #[serde(default, skip_serializing_if = "Option::is_none")] + pending_settlement: Option>, + }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PartialNativeDispatch { + pub verification: VerificationLevel, + #[serde(default)] + pub native_evidence: NativeEvidence, + #[serde(default)] + pub warnings: Vec, +} + +impl NativeError { + pub fn new( + code: ErrorCode, + phase: ErrorPhase, + retryable: bool, + message: impl Into, + ) -> Self { + Self { + code, + phase, + retryable, + message: message.into(), + details: BTreeMap::new(), + partial_evidence: None, + pending_settlement: None, + related_failures: Vec::new(), + } + } + + pub fn invalid(message: impl Into) -> Self { + Self::new( + ErrorCode::InvalidRequest, + ErrorPhase::Validate, + false, + message, + ) + } + + pub fn stale(code: ErrorCode, message: impl Into) -> Self { + Self::new(code, ErrorPhase::Preflight, true, message) + } + + pub fn unsupported(reason: impl Into) -> Self { + Self::new( + ErrorCode::UnsupportedInBackground, + ErrorPhase::Preflight, + false, + reason, + ) + } + + pub fn from_posture(posture: &PostureResult) -> Option { + let observed_disturbance = posture.frontmost_changed + || posture.key_window_changed + || posture.physical_cursor_moved + || posture.restored_after_violation; + if observed_disturbance { + return Some(Self::new( + ErrorCode::PostureViolated, + ErrorPhase::Verify, + false, + "interaction changed the user's foreground, key window, or physical cursor posture", + )); + } + if !posture.held { + return Some(Self::new( + ErrorCode::PostureUnverifiable, + ErrorPhase::Verify, + false, + "required posture witness was unavailable, incomplete, or lagged without an observed disturbance", + )); + } + None + } + + pub fn with_detail(mut self, key: impl Into, value: impl Into) -> Self { + self.details.insert(key.into(), value.into()); + self + } + + pub fn with_related(mut self, failure: &NativeError) -> Self { + self.related_failures.push(NativeErrorSummary { + code: failure.code, + phase: failure.phase, + message: failure.message.clone(), + }); + self + } + + pub fn precedence(&self) -> u8 { + match self.code { + ErrorCode::PostureViolated => 5, + ErrorCode::PostureUnverifiable => 4, + ErrorCode::DispatchFailed => 3, + ErrorCode::UiNotSettled => 2, + ErrorCode::VerificationFailed => 1, + _ => 0, + } + } + + pub fn primary(mut failures: Vec) -> Option { + if failures.is_empty() { + return None; + } + let primary_index = failures + .iter() + .enumerate() + .max_by_key(|(_, error)| error.precedence()) + .map(|(index, _)| index)?; + let mut primary = failures.remove(primary_index); + primary + .related_failures + .extend(failures.iter().map(|failure| NativeErrorSummary { + code: failure.code, + phase: failure.phase, + message: failure.message.clone(), + })); + Some(primary) + } +} + +impl fmt::Display for NativeError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "{:?}/{:?}: {}", + self.code, self.phase, self.message + ) + } +} + +impl std::error::Error for NativeError {} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/interaction.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/interaction.rs new file mode 100644 index 0000000000..a773ec4b3a --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/interaction.rs @@ -0,0 +1,361 @@ +//! Portable per-mutation interaction scope and evidence. + +use std::{ + collections::BTreeMap, + sync::{Arc, Mutex}, + time::Instant, +}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use super::{ + contracts::{ActionId, MenuId, Point, Route}, + errors::NativeError, + observation::{ResolvedWindow, ResolvedWindowStamp}, + target::TargetValidityHandle, +}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PostureResult { + pub held: bool, + pub frontmost_changed: bool, + pub key_window_changed: bool, + pub physical_cursor_moved: bool, + pub restored_after_violation: bool, +} + +impl Default for PostureResult { + fn default() -> Self { + Self { + held: true, + frontmost_changed: false, + key_window_changed: false, + physical_cursor_moved: false, + restored_after_violation: false, + } + } +} + +#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct NativeEvidence { + #[serde(default)] + pub fields: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub interaction_scope: Option, +} + +impl NativeEvidence { + pub fn merge(&mut self, other: Self) { + self.fields.extend(other.fields); + if other.interaction_scope.is_some() { + self.interaction_scope = other.interaction_scope; + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LeaseDecision { + Acquired, + NotApplicable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum LeaseTeardownStatus { + Released, + NotApplicable, + Failed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScopeLeaseAcquisition { + pub posture_witness: LeaseDecision, + pub accessibility: LeaseDecision, + pub containment: LeaseDecision, + pub menu_dismissal: LeaseDecision, + pub target_belief: LeaseDecision, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct ScopeLeaseTeardown { + pub posture_witness: LeaseTeardownStatus, + pub accessibility: LeaseTeardownStatus, + pub containment: LeaseTeardownStatus, + pub menu_dismissal: LeaseTeardownStatus, + pub target_belief: LeaseTeardownStatus, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct InteractionScopeEvidence { + pub acquisition: ScopeLeaseAcquisition, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub teardown: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ScopeRequirements { + pub target_belief: bool, + pub containment: bool, + pub accessibility: bool, + pub menu_dismissal: bool, +} + +/// Controller-owned time boundaries for one mutation. +/// +/// Native work must stop by `work`; the later `teardown` boundary is reserved +/// for releasing every acquired lease while containment is still active. +/// Providers must carry this exact value through preflight and acquisition -- +/// they do not get to create an independent native timeout. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MutationDeadline { + pub work: Instant, + pub teardown: Instant, +} + +impl MutationDeadline { + pub fn new(work: Instant, teardown: Instant) -> Result { + if teardown < work { + return Err(NativeError::invalid( + "mutation teardown deadline must not precede its work deadline", + )); + } + Ok(Self { work, teardown }) + } +} + +impl ScopeRequirements { + pub fn for_route(route: Route) -> Self { + Self { + target_belief: !matches!(route, Route::Semantic), + containment: true, + accessibility: true, + menu_dismissal: false, + } + } +} + +/// Action-aware output from the final fallible interaction preflight. +/// +/// `NativePlan` keeps the selected platform recipe statically typed. Core +/// carries this exact value into acquisition instead of asking a provider to +/// recover recipe state from a mutable side channel. +#[derive(Debug)] +pub struct ScopePlan { + pub action_id: ActionId, + pub window: ResolvedWindow, + pub route: Route, + pub deadline: MutationDeadline, + pub requirements: ScopeRequirements, + pub opening_menu_id: Option, + pub native: NativePlan, +} + +impl ScopePlan { + pub fn new( + action_id: ActionId, + window: ResolvedWindow, + route: Route, + deadline: MutationDeadline, + requirements: ScopeRequirements, + native: NativePlan, + ) -> Self { + Self { + action_id, + window, + route, + deadline, + requirements, + opening_menu_id: None, + native, + } + } + + pub fn bind_opening_menu(&mut self, menu_id: MenuId) { + self.opening_menu_id = Some(menu_id); + } + + pub fn into_scope( + self, + acquisition: ScopeLeaseAcquisition, + logical_cursor: TargetCursorHandle, + native_evidence: NativeEvidence, + cleanup: Box, + ) -> InteractionScope { + InteractionScope::new( + self.window, + self.route, + self.action_id, + self.deadline, + self.opening_menu_id, + acquisition, + logical_cursor, + native_evidence, + cleanup, + ) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq)] +pub struct LogicalCursor { + pub point: Option, +} + +#[derive(Debug, Clone, Default)] +pub struct TargetCursorHandle(Arc>); + +impl TargetCursorHandle { + pub fn position(&self) -> Option { + self.0.lock().expect("logical cursor lock poisoned").point + } + + pub fn update(&self, point: Point) { + self.0.lock().expect("logical cursor lock poisoned").point = Some(point); + } +} + +pub struct InteractionScope { + pub window: ResolvedWindow, + pub route: Route, + pub deadline: MutationDeadline, + pub leases: ScopeLeaseAcquisition, + pub action_id: ActionId, + pub owner: ResolvedWindowStamp, + pub opening_menu_id: Option, + pub logical_cursor: TargetCursorHandle, + pub posture: PostureResult, + pub native_evidence: NativeEvidence, + cleanup: Option>, + teardown: Option, + target_validity: Option, +} + +impl std::fmt::Debug for InteractionScope { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("InteractionScope") + .field("window", &self.window) + .field("route", &self.route) + .field("deadline", &self.deadline) + .field("leases", &self.leases) + .field("action_id", &self.action_id) + .field("owner", &self.owner) + .field("opening_menu_id", &self.opening_menu_id) + .field("posture", &self.posture) + .field("native_evidence", &self.native_evidence) + .field("teardown", &self.teardown) + .field("target_validity_bound", &self.target_validity.is_some()) + .finish_non_exhaustive() + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ScopeTeardownOutcome { + pub posture: PostureResult, + pub native_evidence: NativeEvidence, + pub leases: ScopeLeaseTeardown, + pub failures: Vec, +} + +pub trait ScopeCleanup: Send { + /// Attempts every bounded native release and returns the complete result. + /// Implementations must not stop after the first failure. Core invokes the + /// native cleanup at most once and caches this outcome for repeat release. + /// `deadline` is the controller's absolute teardown cutoff: callback + /// barriers and joins must be bounded by it, and already-expired cleanup + /// must still perform every immediate best-effort release without waiting. + fn cleanup(&mut self, deadline: Instant) -> ScopeTeardownOutcome; +} + +impl InteractionScope { + #[allow(clippy::too_many_arguments)] + fn new( + window: ResolvedWindow, + route: Route, + action_id: ActionId, + deadline: MutationDeadline, + opening_menu_id: Option, + leases: ScopeLeaseAcquisition, + logical_cursor: TargetCursorHandle, + mut native_evidence: NativeEvidence, + cleanup: Box, + ) -> Self { + native_evidence.interaction_scope = Some(InteractionScopeEvidence { + acquisition: leases.clone(), + teardown: None, + }); + let owner = window.stamp(); + Self { + window, + route, + deadline, + leases, + action_id, + owner, + opening_menu_id, + logical_cursor, + posture: PostureResult::default(), + native_evidence, + cleanup: Some(cleanup), + teardown: None, + target_validity: None, + } + } + + pub(crate) fn bind_target_validity(&mut self, target_validity: TargetValidityHandle) { + self.target_validity = Some(target_validity); + } + + pub fn release(&mut self) -> ScopeTeardownOutcome { + if let Some(outcome) = &self.teardown { + return outcome.clone(); + } + let mut cleanup = self + .cleanup + .take() + .expect("active interaction scope must retain its cleanup"); + let mut outcome = cleanup.cleanup(self.deadline.teardown); + if Instant::now() > self.deadline.teardown { + outcome.failures.push( + NativeError::new( + super::errors::ErrorCode::VerificationFailed, + super::errors::ErrorPhase::Verify, + false, + "interaction scope teardown exceeded the controller-owned deadline", + ) + .with_detail("deadline_stage", "teardown"), + ); + } + if !outcome.failures.is_empty() { + if let Some(target_validity) = &self.target_validity { + target_validity.invalidate(); + } + } + self.posture = outcome.posture.clone(); + self.native_evidence.merge(outcome.native_evidence.clone()); + self.native_evidence.interaction_scope = Some(InteractionScopeEvidence { + acquisition: self.leases.clone(), + teardown: Some(outcome.leases.clone()), + }); + self.teardown = Some(outcome.clone()); + outcome + } +} + +impl Drop for InteractionScope { + fn drop(&mut self) { + if self.cleanup.is_none() { + return; + } + let outcome = self.release(); + for error in outcome.failures { + tracing::error!(error = %error, "failed to release v2 interaction scope during drop"); + } + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/menu.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/menu.rs new file mode 100644 index 0000000000..569ec0fe48 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/menu.rs @@ -0,0 +1,515 @@ +//! Durable menu lifecycle state with revisioned public observations. + +use serde::{Deserialize, Serialize}; + +use super::contracts::{ + ActionId, ElementId, ElementRef, MenuId, MenuRevision, ObservationId, SurfaceId, + WindowGeneration, WindowRef, +}; +use super::errors::{ErrorCode, NativeError}; +use super::observation::{NativeProcessHandle, NativeWindowHandle, ResolvedWindowStamp}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct NativeMenuIdentity { + pub process: NativeProcessHandle, + pub window: NativeWindowHandle, + pub generation: WindowGeneration, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum NativeMenuEvidence { + Opened { + menu_id: MenuId, + opened_by_action_id: ActionId, + owner: ResolvedWindowStamp, + identity: NativeMenuIdentity, + surface_ids: Vec, + focused_item: Option, + }, + Targeted { + menu_id: MenuId, + action_id: ActionId, + owner: ResolvedWindowStamp, + identity: NativeMenuIdentity, + }, + Dismissed { + menu_id: MenuId, + action_id: ActionId, + owner: ResolvedWindowStamp, + identity: NativeMenuIdentity, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Default)] +#[allow(clippy::large_enum_variant)] // Exact owner/action/native evidence stays one causal value. +pub enum NativeMenuObservation { + #[default] + Unchanged, + Closed { + identity: NativeMenuIdentity, + }, + Open { + menu_id: MenuId, + opened_by_action_id: ActionId, + owner: ResolvedWindowStamp, + identity: NativeMenuIdentity, + surface_ids: Vec, + focused_item: Option, + }, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[allow(clippy::large_enum_variant)] // Wire shape intentionally matches the public tagged union. +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum MenuState { + Closed { + revision: MenuRevision, + }, + Open { + revision: MenuRevision, + id: MenuId, + opened_by_action_id: ActionId, + owner_window: WindowRef, + #[serde(default)] + surface_ids: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + focused_item: Option, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum MenuLifecycle { + Closed, + Opening { + id: MenuId, + action_id: ActionId, + owner: WindowRef, + owner_native: ResolvedWindowStamp, + }, + Open { + id: MenuId, + opened_by_action_id: ActionId, + owner: WindowRef, + surface_ids: Vec, + focused_item: Option, + native_identity: NativeMenuIdentity, + owner_native: ResolvedWindowStamp, + }, + Targeting { + id: MenuId, + opened_by_action_id: ActionId, + owner: WindowRef, + action_id: ActionId, + surface_ids: Vec, + focused_item: Option, + native_identity: NativeMenuIdentity, + owner_native: ResolvedWindowStamp, + }, + Dismissing { + id: MenuId, + action_id: ActionId, + native_identity: NativeMenuIdentity, + owner_native: ResolvedWindowStamp, + }, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct MenuControllerState { + revision: MenuRevision, + lifecycle: MenuLifecycle, +} + +impl Default for MenuControllerState { + fn default() -> Self { + Self { + revision: MenuRevision::new(), + lifecycle: MenuLifecycle::Closed, + } + } +} + +impl MenuControllerState { + pub fn lifecycle(&self) -> &MenuLifecycle { + &self.lifecycle + } + + pub fn begin_open( + &mut self, + action_id: ActionId, + owner: WindowRef, + owner_native: ResolvedWindowStamp, + ) -> MenuId { + let id = MenuId::new(); + self.revision = MenuRevision::new(); + self.lifecycle = MenuLifecycle::Opening { + id: id.clone(), + action_id, + owner, + owner_native, + }; + id + } + + #[allow(clippy::too_many_arguments)] // Each causal field is validated independently here. + pub fn record_open( + &mut self, + id: &MenuId, + evidence_menu_id: &MenuId, + evidence_action_id: &ActionId, + evidence_owner: &ResolvedWindowStamp, + native_identity: NativeMenuIdentity, + surface_ids: Vec, + focused_item: Option, + ) -> Result<(), NativeError> { + let (opened_by_action_id, owner, owner_native) = match &self.lifecycle { + MenuLifecycle::Opening { + id: current, + action_id, + owner, + owner_native, + } if current == id + && current == evidence_menu_id + && action_id == evidence_action_id + && owner_native == evidence_owner => + { + (action_id.clone(), owner.clone(), owner_native.clone()) + } + _ => { + return Err(menu_stale( + "opened menu does not match the active opening state", + )) + } + }; + self.revision = MenuRevision::new(); + self.lifecycle = MenuLifecycle::Open { + id: id.clone(), + opened_by_action_id, + owner, + surface_ids, + focused_item, + native_identity, + owner_native, + }; + Ok(()) + } + + pub fn record_dispatch_evidence( + &mut self, + action_id: &ActionId, + owner: &ResolvedWindowStamp, + evidence: &NativeMenuEvidence, + ) -> Result<(), NativeError> { + match evidence { + NativeMenuEvidence::Opened { + menu_id, + opened_by_action_id, + owner: evidence_owner, + identity, + surface_ids, + focused_item: _, + } => { + if opened_by_action_id != action_id || evidence_owner != owner { + return Err(menu_stale( + "native menu-open evidence does not match the dispatch action/owner", + )); + } + self.record_open( + menu_id, + menu_id, + opened_by_action_id, + evidence_owner, + identity.clone(), + surface_ids.clone(), + None, + ) + } + NativeMenuEvidence::Dismissed { + menu_id, + action_id: evidence_action_id, + owner: evidence_owner, + identity, + } => { + self.validate_dispatch_identity( + menu_id, + evidence_action_id, + evidence_owner, + identity, + action_id, + owner, + )?; + self.close(); + Ok(()) + } + NativeMenuEvidence::Targeted { + menu_id, + action_id: evidence_action_id, + owner: evidence_owner, + identity, + } => { + self.validate_dispatch_identity( + menu_id, + evidence_action_id, + evidence_owner, + identity, + action_id, + owner, + )?; + self.finish_target()?; + Ok(()) + } + } + } + + pub fn begin_target(&mut self, id: &MenuId, action_id: ActionId) -> Result<(), NativeError> { + let MenuLifecycle::Open { + id: current, + opened_by_action_id, + owner, + surface_ids, + focused_item, + native_identity, + owner_native, + } = &self.lifecycle + else { + return Err(menu_stale("no open menu is available for targeting")); + }; + if current != id { + return Err(menu_stale("menu id does not match the currently open menu")); + } + self.lifecycle = MenuLifecycle::Targeting { + id: current.clone(), + opened_by_action_id: opened_by_action_id.clone(), + owner: owner.clone(), + action_id, + surface_ids: surface_ids.clone(), + focused_item: focused_item.clone(), + native_identity: native_identity.clone(), + owner_native: owner_native.clone(), + }; + Ok(()) + } + + pub fn begin_dismiss(&mut self, action_id: ActionId) -> Result<(), NativeError> { + let (id, native_identity, owner_native) = match &self.lifecycle { + MenuLifecycle::Open { + id, + native_identity, + owner_native, + .. + } + | MenuLifecycle::Targeting { + id, + native_identity, + owner_native, + .. + } => (id.clone(), native_identity.clone(), owner_native.clone()), + _ => return Err(menu_stale("no active menu is available for dismissal")), + }; + self.lifecycle = MenuLifecycle::Dismissing { + id, + action_id, + native_identity, + owner_native, + }; + Ok(()) + } + + pub fn reconcile_observation( + &mut self, + observation: NativeMenuObservation, + observation_id: &ObservationId, + ) -> Result<(), NativeError> { + match observation { + NativeMenuObservation::Unchanged => Ok(()), + NativeMenuObservation::Closed { identity } => { + if self.native_identity() != Some(&identity) { + return Err(menu_stale( + "closed native menu identity does not match the active menu", + )); + } + self.close(); + Ok(()) + } + NativeMenuObservation::Open { + menu_id, + opened_by_action_id, + owner, + identity, + surface_ids, + focused_item, + } => match &mut self.lifecycle { + MenuLifecycle::Opening { id, .. } => { + let id = id.clone(); + self.record_open( + &id, + &menu_id, + &opened_by_action_id, + &owner, + identity, + surface_ids, + focused_item.map(|id| ElementRef { + observation_id: observation_id.clone(), + id, + }), + ) + } + MenuLifecycle::Open { + id, + opened_by_action_id: current_action, + owner_native, + native_identity, + surface_ids: current_surfaces, + focused_item: current_focus, + .. + } if id == &menu_id + && current_action == &opened_by_action_id + && owner_native == &owner + && native_identity == &identity => + { + *current_surfaces = surface_ids; + *current_focus = focused_item.map(|id| ElementRef { + observation_id: observation_id.clone(), + id, + }); + self.revision = MenuRevision::new(); + Ok(()) + } + _ => Err(menu_stale( + "observed native menu identity has no matching controller lifecycle", + )), + }, + } + } + + pub fn native_identity(&self) -> Option<&NativeMenuIdentity> { + match &self.lifecycle { + MenuLifecycle::Open { + native_identity, .. + } + | MenuLifecycle::Targeting { + native_identity, .. + } + | MenuLifecycle::Dismissing { + native_identity, .. + } => Some(native_identity), + _ => None, + } + } + + fn validate_dispatch_identity( + &self, + menu_id: &MenuId, + evidence_action_id: &ActionId, + evidence_owner: &ResolvedWindowStamp, + identity: &NativeMenuIdentity, + action_id: &ActionId, + owner: &ResolvedWindowStamp, + ) -> Result<(), NativeError> { + let (current_id, current_action, current_owner, current_identity) = match &self.lifecycle { + MenuLifecycle::Targeting { + id, + action_id, + owner_native, + native_identity, + .. + } + | MenuLifecycle::Dismissing { + id, + action_id, + owner_native, + native_identity, + } => (id, action_id, owner_native, native_identity), + _ => { + return Err(menu_stale( + "native menu evidence has no matching controller transition", + )) + } + }; + if current_id != menu_id + || current_action != evidence_action_id + || current_action != action_id + || current_owner != evidence_owner + || current_owner != owner + || current_identity != identity + { + return Err(menu_stale( + "native menu evidence does not match the current menu/action/owner identity", + )); + } + Ok(()) + } + + fn finish_target(&mut self) -> Result<(), NativeError> { + let MenuLifecycle::Targeting { + id, + opened_by_action_id, + owner, + surface_ids, + focused_item, + native_identity, + owner_native, + .. + } = &self.lifecycle + else { + return Err(menu_stale("no menu targeting transition is active")); + }; + self.lifecycle = MenuLifecycle::Open { + id: id.clone(), + opened_by_action_id: opened_by_action_id.clone(), + owner: owner.clone(), + surface_ids: surface_ids.clone(), + focused_item: focused_item.clone(), + native_identity: native_identity.clone(), + owner_native: owner_native.clone(), + }; + Ok(()) + } + + pub fn validate_current_menu_id(&self, menu_id: &MenuId) -> Result<(), NativeError> { + match &self.lifecycle { + MenuLifecycle::Open { id, .. } | MenuLifecycle::Targeting { id, .. } + if id == menu_id => + { + Ok(()) + } + _ => Err(menu_stale( + "menu element does not match the controller's current MenuId", + )), + } + } + + pub fn close(&mut self) { + self.revision = MenuRevision::new(); + self.lifecycle = MenuLifecycle::Closed; + } + + pub fn observation(&self) -> Result { + match &self.lifecycle { + MenuLifecycle::Closed => Ok(MenuState::Closed { + revision: self.revision.clone(), + }), + MenuLifecycle::Open { + id, + opened_by_action_id, + owner, + surface_ids, + focused_item, + .. + } => Ok(MenuState::Open { + revision: self.revision.clone(), + id: id.clone(), + opened_by_action_id: opened_by_action_id.clone(), + owner_window: owner.clone(), + surface_ids: surface_ids.clone(), + focused_item: focused_item.clone(), + }), + _ => Err(menu_stale( + "menu lifecycle is transitional and cannot be published as settled state", + )), + } + } +} + +fn menu_stale(message: impl Into) -> NativeError { + NativeError::stale(ErrorCode::MenuStateStale, message) +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/mod.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/mod.rs new file mode 100644 index 0000000000..81333f1c43 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/mod.rs @@ -0,0 +1,26 @@ +//! Typed, background-only driver v2 contracts and orchestration. +//! +//! This module is additive. The MCP/v1 surface remains available while v2 is +//! built and verified independently. + +pub mod capabilities; +pub mod contracts; +pub mod controller; +pub mod errors; +pub mod interaction; +pub mod menu; +pub mod observation; +pub mod platform; +pub mod settlement; +pub mod target; + +pub use capabilities::*; +pub use contracts::*; +pub use controller::*; +pub use errors::*; +pub use interaction::*; +pub use menu::*; +pub use observation::*; +pub use platform::*; +pub use settlement::*; +pub use target::*; diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/observation.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/observation.rs new file mode 100644 index 0000000000..70add43a05 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/observation.rs @@ -0,0 +1,1379 @@ +//! Observation identity, AX revisions, and the only perception-to-action bridge. + +use std::{ + collections::{HashMap, VecDeque}, + sync::{Arc, Mutex}, + time::{Duration, Instant}, +}; + +use super::{ + capabilities::{Framework, WindowStateKind}, + contracts::{ + AccessibilityElement, AccessibilityState, AxRevision, AxTreeMode, AxTreeUpdate, + CaptureRevision, CapturedSurface, ElementId, ElementRef, GeometryRevision, MenuId, + ObservationId, Point, Rect, ReplaceAxLines, Size, SurfaceId, SurfaceKind, WindowGeneration, + WindowRef, + }, + errors::{ErrorCode, ErrorPhase, NativeError}, + menu::{MenuState, NativeMenuObservation}, + settlement::SettlementEvidence, +}; + +macro_rules! native_handle { + ($name:ident) => { + #[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord)] + pub struct $name(String); + + impl $name { + pub fn new(value: impl Into) -> Result { + let value = value.into(); + if value.trim().is_empty() { + return Err(NativeError::invalid(concat!( + stringify!($name), + " cannot be empty" + ))); + } + Ok(Self(value)) + } + + pub fn as_str(&self) -> &str { + &self.0 + } + } + }; +} + +native_handle!(NativeWindowHandle); +native_handle!(NativeProcessHandle); +native_handle!(NativeElementHandle); + +#[derive(Debug, Clone, PartialEq)] +pub struct WindowGeometry { + pub bounds: Rect, + pub scale_factor: f64, + pub revision: GeometryRevision, +} + +impl WindowGeometry { + pub fn validate(&self) -> Result<(), NativeError> { + self.bounds.validate().map_err(invalid_native_geometry)?; + if !self.scale_factor.is_finite() || self.scale_factor <= 0.0 { + return Err(invalid_native_geometry( + "window scale_factor must be finite and positive", + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub struct ResolvedWindow { + pub public: WindowRef, + pub native: NativeWindowHandle, + pub process: NativeProcessHandle, + pub framework: Framework, + pub geometry: WindowGeometry, + pub generation: WindowGeneration, + pub state: WindowStateKind, +} + +impl ResolvedWindow { + pub fn stamp(&self) -> ResolvedWindowStamp { + ResolvedWindowStamp { + app_id: self.public.app.id.clone(), + window_id: self.public.id.clone(), + generation: self.generation, + geometry_revision: self.geometry.revision.clone(), + native_window: self.native.clone(), + process: self.process.clone(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedWindowStamp { + pub app_id: super::contracts::AppId, + pub window_id: super::contracts::WindowId, + pub generation: WindowGeneration, + pub geometry_revision: GeometryRevision, + pub native_window: NativeWindowHandle, + pub process: NativeProcessHandle, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct SurfaceToWindowTransform { + pub scale_x: f64, + pub scale_y: f64, + pub offset_x: f64, + pub offset_y: f64, +} + +impl SurfaceToWindowTransform { + pub fn transform(&self, point: Point) -> Point { + Point { + x: point.x * self.scale_x + self.offset_x, + y: point.y * self.scale_y + self.offset_y, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum CaptureFreshness { + Fresh, + ReusedWithFreshCompletion, + Frozen, + Unavailable, +} + +impl CaptureFreshness { + pub fn action_safe(self) -> bool { + matches!(self, Self::Fresh | Self::ReusedWithFreshCompletion) + } +} + +#[derive(Debug, Clone)] +pub struct SurfaceRecord { + pub id: SurfaceId, + pub kind: SurfaceKind, + pub owner_window: WindowRef, + pub image_url: String, + /// Native artifact bytes retained by this surface. The public URL length + /// is not a useful proxy for observation-store memory/disk pressure. + pub approximate_bytes: usize, + pub raster_size: Size, + pub window_bounds: Option, + pub capture_revision: CaptureRevision, + pub transform: SurfaceToWindowTransform, + pub freshness: CaptureFreshness, + /// Exact native ownership for the pixels. Related transient surfaces are + /// action-safe only while their recorded parent is the current target. + pub owner: SurfaceOwner, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SurfaceOwner { + Target(ResolvedWindowStamp), + RelatedTransient { + owner: ResolvedWindowStamp, + parent: ResolvedWindowStamp, + }, +} + +impl SurfaceRecord { + pub fn validate(&self) -> Result<(), NativeError> { + self.raster_size + .validate() + .map_err(invalid_native_geometry)?; + if let Some(bounds) = self.window_bounds { + bounds.validate().map_err(invalid_native_geometry)?; + } + let transform = self.transform; + if !transform.scale_x.is_finite() + || !transform.scale_y.is_finite() + || !transform.offset_x.is_finite() + || !transform.offset_y.is_finite() + || transform.scale_x == 0.0 + || transform.scale_y == 0.0 + { + return Err(invalid_native_geometry( + "surface transform must be finite and invertible", + )); + } + Ok(()) + } + + pub fn validate_for_window(&self, window: &ResolvedWindow) -> Result<(), NativeError> { + self.validate()?; + validate_surface_owner(self, window) + } + + pub fn public(&self) -> CapturedSurface { + CapturedSurface { + id: self.id.clone(), + owner_window: self.owner_window.clone(), + kind: self.kind, + image_url: self.image_url.clone(), + size: self.raster_size, + window_bounds: self.window_bounds, + } + } +} + +#[derive(Debug, Clone)] +pub struct NativeAccessibilityElement { + pub id: ElementId, + pub native: NativeElementHandle, + /// Exact target or related-transient window that owned this AX element + /// when the observation was captured. + pub owner: ResolvedWindowStamp, + pub role: Option, + pub label: Option, + pub value: Option, + pub bounds: Option, + pub actions: Vec, + /// Native attribution for elements published by a related menu surface. + pub menu_id: Option, +} + +#[derive(Debug, Clone)] +pub struct NativeAccessibilityUpdate { + pub normalized_tree: String, + pub elements: Vec, + pub focused_element: Option, + pub selected_text: Option, + pub selected_elements: Vec, + pub document_text: Option, +} + +#[derive(Debug, Clone)] +pub struct NativeObservationUpdate { + pub window: ResolvedWindow, + pub surfaces: Vec, + pub accessibility: Option, + pub menu: NativeMenuObservation, + pub captured_at_unix_ms: u64, + pub warnings: Vec, + /// Observation-owned native resources (for example screenshot files). + /// Cleanup runs exactly once when the final stored handle is dropped. + pub artifacts: Vec, +} + +type ArtifactCleanup = Box Result<(), NativeError> + Send + 'static>; + +struct ObservationArtifactInner { + label: String, + cleanup: Mutex>, +} + +impl Drop for ObservationArtifactInner { + fn drop(&mut self) { + let cleanup = self + .cleanup + .get_mut() + .expect("observation artifact cleanup lock poisoned") + .take(); + if let Some(cleanup) = cleanup { + if let Err(error) = cleanup() { + tracing::error!(artifact = %self.label, error = %error, "failed to clean observation artifact"); + } + } + } +} + +/// Cloneable ownership token whose final drop performs artifact cleanup. +#[derive(Clone)] +pub struct ObservationArtifactHandle(Arc); + +impl ObservationArtifactHandle { + pub fn new( + label: impl Into, + cleanup: impl FnOnce() -> Result<(), NativeError> + Send + 'static, + ) -> Self { + Self(Arc::new(ObservationArtifactInner { + label: label.into(), + cleanup: Mutex::new(Some(Box::new(cleanup))), + })) + } + + pub fn label(&self) -> &str { + &self.0.label + } +} + +impl std::fmt::Debug for ObservationArtifactHandle { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_tuple("ObservationArtifactHandle") + .field(&self.0.label) + .finish() + } +} + +#[derive(Debug, Clone)] +pub struct ElementRecord { + pub id: ElementId, + pub native: NativeElementHandle, + pub owner: ResolvedWindowStamp, + pub role: Option, + pub bounds: Option, + pub actions: Vec, + pub ax_revision: AxRevision, + pub menu_id: Option, +} + +#[derive(Debug, Clone)] +pub struct AccessibilityRecord { + pub revision: AxRevision, + pub elements: HashMap, + pub focused_element: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum InvalidationReason { + Superseded, + MutationDispatched, + WindowChanged, + AppTerminated, + DisplayChanged, + /// Ordinary value/focus/content changed. Current observations are stale, + /// but the last delivered AX tree remains a valid diff base. + ContentChanged, + AccessibilityInvalidated, + TransientDismissed, + MenuChanged, + DiffBaseInvalidated, + Evicted, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ObservationState { + Current, + Consumed { + action_id: super::contracts::ActionId, + }, + Invalidated { + reason: InvalidationReason, + }, + Expired, +} + +#[derive(Debug, Clone)] +pub struct ObservationRecord { + pub id: ObservationId, + pub window: ResolvedWindowStamp, + pub captured_at: Instant, + pub surfaces: HashMap, + pub accessibility: Option, + pub menu: MenuState, + pub settlement: SettlementEvidence, + pub state: ObservationState, + pub approximate_bytes: usize, + pub artifacts: Vec, +} + +#[derive(Debug, Clone)] +pub struct ResolvedPoint { + pub window: ResolvedWindow, + pub surface_id: SurfaceId, + pub surface_owner: SurfaceOwner, + pub capture_revision: CaptureRevision, + pub surface_point: Point, + pub window_point: Point, + pub screen_point: Point, + pub geometry_revision: GeometryRevision, +} + +#[derive(Debug, Clone)] +pub struct ResolvedDrag { + pub start: ResolvedPoint, + pub end: ResolvedPoint, + pub duration_ms: u32, + pub button: super::contracts::MouseButton, + pub modifiers: Vec, +} + +#[derive(Debug, Clone)] +pub struct ResolvedScroll { + pub point: ResolvedPoint, + pub delta_x: f64, + pub delta_y: f64, +} + +#[derive(Debug, Clone)] +pub struct ResolvedElement { + pub window: ResolvedWindow, + pub observation_id: ObservationId, + pub element_id: ElementId, + pub native: NativeElementHandle, + pub owner: ResolvedWindowStamp, + pub ax_revision: AxRevision, + pub role: Option, + pub bounds: Option, + pub actions: Vec, + pub menu_id: Option, +} + +#[derive(Debug, Clone)] +pub struct ResolvedFocus { + pub window: ResolvedWindow, + pub observation_id: ObservationId, + pub focused_element: Option, + pub ax_revision: Option, +} + +#[derive(Debug)] +pub struct ObservationStore { + records: HashMap, + /// Least-recently-used at the front, most-recently-used at the back. + access_order: VecDeque, + max_count: usize, + max_bytes: usize, + ttl: Duration, + current_bytes: usize, +} + +impl ObservationStore { + pub fn new(max_count: usize, max_bytes: usize, ttl: Duration) -> Self { + Self { + records: HashMap::new(), + access_order: VecDeque::new(), + max_count, + max_bytes, + ttl, + current_bytes: 0, + } + } + + pub fn insert(&mut self, record: ObservationRecord) -> Result<(), NativeError> { + self.expire(); + let id = record.id.clone(); + if self.max_count == 0 || record.approximate_bytes > self.max_bytes { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Verify, + false, + "observation exceeds the configured native store capacity", + ) + .with_detail("observation_bytes", record.approximate_bytes) + .with_detail("max_bytes", self.max_bytes)); + } + let replaced_bytes = self + .records + .get(&id) + .map_or(0, |existing| existing.approximate_bytes); + let projected_bytes = self + .current_bytes + .saturating_sub(replaced_bytes) + .checked_add(record.approximate_bytes) + .ok_or_else(|| { + NativeError::new( + ErrorCode::Internal, + ErrorPhase::Verify, + false, + "observation store byte accounting overflowed", + ) + })?; + self.invalidate_all(InvalidationReason::Superseded); + if self.records.insert(id.clone(), record).is_some() { + self.access_order.retain(|candidate| candidate != &id); + } + self.current_bytes = projected_bytes; + self.access_order.push_back(id.clone()); + self.evict(&id); + debug_assert!(self.records.contains_key(&id)); + Ok(()) + } + + pub fn current( + &mut self, + observation_id: &ObservationId, + window: &ResolvedWindow, + ) -> Result<&ObservationRecord, NativeError> { + self.expire(); + { + let record = self.records.get(observation_id).ok_or_else(|| { + NativeError::stale( + ErrorCode::ObservationStale, + "observation is missing or was evicted from the native store", + ) + .with_detail("observation_id", observation_id.to_string()) + })?; + ensure_window_stamp(&record.window, window)?; + if record.state != ObservationState::Current { + return Err(NativeError::stale( + ErrorCode::ObservationStale, + format!("observation is not current: {:?}", record.state), + ) + .with_detail("observation_id", observation_id.to_string())); + } + } + self.touch(observation_id); + Ok(self + .records + .get(observation_id) + .expect("touched observation remains in store")) + } + + pub fn consume( + &mut self, + observation_id: &ObservationId, + action_id: super::contracts::ActionId, + ) -> Result<(), NativeError> { + self.expire(); + { + let record = self.records.get_mut(observation_id).ok_or_else(|| { + NativeError::stale( + ErrorCode::ObservationStale, + "cannot consume an observation that is not in the native store", + ) + })?; + if record.state != ObservationState::Current { + return Err(NativeError::stale( + ErrorCode::ObservationStale, + "observation was already consumed or invalidated", + )); + } + record.state = ObservationState::Consumed { action_id }; + } + self.touch(observation_id); + Ok(()) + } + + /// Remove an observation immediately. Dropping its final artifact handles + /// performs deterministic native-file cleanup. + pub fn remove(&mut self, observation_id: &ObservationId) -> bool { + self.remove_record(observation_id).is_some() + } + + pub fn len(&self) -> usize { + self.records.len() + } + + pub fn is_empty(&self) -> bool { + self.records.is_empty() + } + + pub fn current_bytes(&self) -> usize { + self.current_bytes + } + + pub fn invalidate_all(&mut self, reason: InvalidationReason) { + for record in self.records.values_mut() { + if record.state == ObservationState::Current { + record.state = ObservationState::Invalidated { + reason: reason.clone(), + }; + } + } + } + + pub fn resolve_point( + &mut self, + window: &ResolvedWindow, + observation_id: &ObservationId, + surface_id: &SurfaceId, + point: Point, + ) -> Result { + let record = self.current(observation_id, window)?; + let surface = record.surfaces.get(surface_id).ok_or_else(|| { + NativeError::stale( + ErrorCode::SurfaceStale, + "surface does not belong to the supplied observation", + ) + .with_detail("surface_id", surface_id.to_string()) + })?; + validate_surface_owner(surface, window)?; + if !surface.freshness.action_safe() { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "surface capture is not fresh enough for a point action", + )); + } + if point.x < 0.0 + || point.y < 0.0 + || point.x >= f64::from(surface.raster_size.width) + || point.y >= f64::from(surface.raster_size.height) + { + return Err(NativeError::invalid( + "point lies outside the captured surface raster", + )); + } + let window_point = surface.transform.transform(point); + let screen_point = Point { + x: window.geometry.bounds.x + window_point.x, + y: window.geometry.bounds.y + window_point.y, + }; + Ok(ResolvedPoint { + window: window.clone(), + surface_id: surface_id.clone(), + surface_owner: surface.owner.clone(), + capture_revision: surface.capture_revision.clone(), + surface_point: point, + window_point, + screen_point, + geometry_revision: record.window.geometry_revision.clone(), + }) + } + + pub fn resolve_element( + &mut self, + window: &ResolvedWindow, + element: &ElementRef, + ) -> Result { + let record = self.current(&element.observation_id, window)?; + let accessibility = record.accessibility.as_ref().ok_or_else(|| { + NativeError::stale( + ErrorCode::ElementStale, + "observation did not contain accessibility state", + ) + })?; + let resolved = accessibility.elements.get(&element.id).ok_or_else(|| { + NativeError::stale( + ErrorCode::ElementStale, + "element does not belong to the supplied observation", + ) + .with_detail("element_id", element.id.to_string()) + })?; + if let Some(menu_id) = &resolved.menu_id { + match &record.menu { + MenuState::Open { id, .. } if id == menu_id => {} + _ => { + return Err(NativeError::stale( + ErrorCode::MenuStateStale, + "menu element no longer matches the observation's active menu", + )) + } + } + } + Ok(ResolvedElement { + window: window.clone(), + observation_id: element.observation_id.clone(), + element_id: resolved.id.clone(), + native: resolved.native.clone(), + owner: resolved.owner.clone(), + ax_revision: resolved.ax_revision.clone(), + role: resolved.role.clone(), + bounds: resolved.bounds, + actions: resolved.actions.clone(), + menu_id: resolved.menu_id.clone(), + }) + } + + pub fn resolve_element_point( + &mut self, + window: &ResolvedWindow, + element: &ElementRef, + ) -> Result { + let resolved = self.resolve_element(window, element)?; + let center = resolved + .bounds + .ok_or_else(|| { + NativeError::stale( + ErrorCode::ElementStale, + "element has no current bounds for targeted pointer dispatch", + ) + })? + .center(); + let record = self.current(&element.observation_id, window)?; + let surface = record + .surfaces + .values() + .find(|surface| { + surface_owner_stamp(surface) == &resolved.owner && surface.freshness.action_safe() + }) + .ok_or_else(|| { + NativeError::stale( + ErrorCode::SurfaceStale, + "observation has no fresh surface for the element's exact owning window", + ) + })?; + if surface.transform.scale_x == 0.0 || surface.transform.scale_y == 0.0 { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "surface transform is not invertible", + )); + } + let surface_point = Point { + x: (center.x - surface.transform.offset_x) / surface.transform.scale_x, + y: (center.y - surface.transform.offset_y) / surface.transform.scale_y, + }; + let surface_id = surface.id.clone(); + self.resolve_point(window, &element.observation_id, &surface_id, surface_point) + } + + pub fn validate_focus( + &mut self, + window: &ResolvedWindow, + observation_id: &ObservationId, + ) -> Result { + let record = self.current(observation_id, window)?; + Ok(ResolvedFocus { + window: window.clone(), + observation_id: observation_id.clone(), + focused_element: record + .accessibility + .as_ref() + .and_then(|accessibility| accessibility.focused_element.clone()), + ax_revision: record + .accessibility + .as_ref() + .map(|accessibility| accessibility.revision.clone()), + }) + } + + fn expire(&mut self) { + let ttl = self.ttl; + let expired: Vec<_> = self + .records + .iter() + .filter(|(_, record)| record.captured_at.elapsed() > ttl) + .map(|(id, _)| id.clone()) + .collect(); + for id in expired { + self.remove_record(&id); + } + } + + fn evict(&mut self, protected: &ObservationId) { + while self.records.len() > self.max_count || self.current_bytes > self.max_bytes { + let candidate = self + .access_order + .iter() + .find(|id| { + *id != protected + && self.records.get(*id).is_some_and(|record| { + !matches!(record.state, ObservationState::Consumed { .. }) + }) + }) + .or_else(|| self.access_order.iter().find(|id| *id != protected)) + .cloned(); + let Some(id) = candidate else { + break; + }; + self.remove_record(&id); + } + } + + fn touch(&mut self, observation_id: &ObservationId) { + self.access_order + .retain(|candidate| candidate != observation_id); + self.access_order.push_back(observation_id.clone()); + } + + fn remove_record(&mut self, observation_id: &ObservationId) -> Option { + self.access_order + .retain(|candidate| candidate != observation_id); + let record = self.records.remove(observation_id)?; + self.current_bytes = self.current_bytes.saturating_sub(record.approximate_bytes); + Some(record) + } +} + +fn validate_surface_owner( + surface: &SurfaceRecord, + window: &ResolvedWindow, +) -> Result<(), NativeError> { + let target = window.stamp(); + let owner = match &surface.owner { + SurfaceOwner::Target(owner) if owner == &target => owner, + SurfaceOwner::RelatedTransient { owner, parent } if parent == &target => owner, + SurfaceOwner::Target(_) | SurfaceOwner::RelatedTransient { .. } => { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "surface native ownership no longer matches the current target", + )) + } + }; + if owner.app_id != surface.owner_window.app.id || owner.window_id != surface.owner_window.id { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "surface public owner does not match its native owner stamp", + )); + } + Ok(()) +} + +fn surface_owner_stamp(surface: &SurfaceRecord) -> &ResolvedWindowStamp { + match &surface.owner { + SurfaceOwner::Target(owner) | SurfaceOwner::RelatedTransient { owner, .. } => owner, + } +} + +fn invalid_native_geometry(message: impl Into) -> NativeError { + NativeError::new( + ErrorCode::Internal, + super::errors::ErrorPhase::Verify, + false, + message, + ) +} + +impl Default for ObservationStore { + fn default() -> Self { + Self::new(32, 128 * 1024 * 1024, Duration::from_secs(120)) + } +} + +fn ensure_window_stamp( + stamp: &ResolvedWindowStamp, + window: &ResolvedWindow, +) -> Result<(), NativeError> { + if stamp.app_id != window.public.app.id + || stamp.window_id != window.public.id + || stamp.generation != window.generation + || stamp.native_window != window.native + || stamp.process != window.process + { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "observation target no longer matches the resolved window identity/generation", + )); + } + if stamp.geometry_revision != window.geometry.revision { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "window geometry changed after observation", + )); + } + Ok(()) +} + +#[derive(Debug, Default)] +pub struct AxRevisionState { + last_delivered: Option<(AxRevision, String)>, + force_full: bool, +} + +impl AxRevisionState { + pub fn invalidate_base(&mut self) { + self.force_full = true; + } + + pub fn reset(&mut self) { + self.last_delivered = None; + self.force_full = false; + } + + pub fn prepare( + &self, + current_tree: &str, + mode: AxTreeMode, + ) -> Result { + let normalized = normalize_tree(current_tree); + let revision = AxRevision::new(); + let update = match (&self.last_delivered, mode, self.force_full) { + (Some((base_revision, base_tree)), AxTreeMode::DiffIfAvailable, false) => { + let operations = diff_lines(base_tree, &normalized); + let reconstructed = apply_ax_diff(base_tree, &operations)?; + if reconstructed != normalized { + return Err(NativeError::new( + ErrorCode::AxRevisionMismatch, + super::errors::ErrorPhase::Verify, + true, + "generated AX diff did not reconstruct the normalized current tree", + )); + } + AxTreeUpdate::Diff { + base_revision: base_revision.clone(), + revision: revision.clone(), + operations, + } + } + _ => AxTreeUpdate::Full { + revision: revision.clone(), + tree: normalized.clone(), + }, + }; + Ok(PreparedAxRevision { + expected_base: self + .last_delivered + .as_ref() + .map(|(revision, _)| revision.clone()), + revision, + normalized_tree: normalized, + update, + }) + } + + /// Commits a revision only after the complete observation has been + /// validated and inserted into the observation store. The target state + /// lock makes an intervening AX commit impossible. + pub fn commit(&mut self, prepared: PreparedAxRevision) { + debug_assert_eq!( + self.last_delivered.as_ref().map(|(revision, _)| revision), + prepared.expected_base.as_ref() + ); + self.last_delivered = Some((prepared.revision, prepared.normalized_tree)); + self.force_full = false; + } + + pub fn last_revision(&self) -> Option<&AxRevision> { + self.last_delivered.as_ref().map(|(revision, _)| revision) + } +} + +pub struct PreparedAxRevision { + expected_base: Option, + revision: AxRevision, + normalized_tree: String, + pub update: AxTreeUpdate, +} + +pub fn normalize_tree(tree: &str) -> String { + tree.replace("\r\n", "\n") + .replace('\r', "\n") + .lines() + .map(str::trim_end) + .collect::>() + .join("\n") +} + +pub fn diff_lines(base: &str, current: &str) -> Vec { + let base_lines: Vec<_> = if base.is_empty() { + Vec::new() + } else { + base.split('\n').collect() + }; + let current_lines: Vec<_> = if current.is_empty() { + Vec::new() + } else { + current.split('\n').collect() + }; + let prefix = base_lines + .iter() + .zip(current_lines.iter()) + .take_while(|(left, right)| left == right) + .count(); + let max_suffix = base_lines + .len() + .min(current_lines.len()) + .saturating_sub(prefix); + let suffix = (0..max_suffix) + .take_while(|offset| { + base_lines[base_lines.len() - 1 - offset] + == current_lines[current_lines.len() - 1 - offset] + }) + .count(); + if prefix == base_lines.len() && prefix == current_lines.len() { + return Vec::new(); + } + vec![ReplaceAxLines { + start_line: prefix, + delete_count: base_lines.len() - prefix - suffix, + lines: current_lines[prefix..current_lines.len() - suffix] + .iter() + .map(|line| (*line).to_owned()) + .collect(), + }] +} + +pub fn apply_ax_diff(base: &str, operations: &[ReplaceAxLines]) -> Result { + let mut base_lines: Vec = if base.is_empty() { + Vec::new() + } else { + base.split('\n').map(str::to_owned).collect() + }; + let mut previous_end = 0; + for operation in operations { + let end = operation + .start_line + .checked_add(operation.delete_count) + .ok_or_else(|| ax_diff_error("AX diff range overflow"))?; + if operation.start_line < previous_end || end > base_lines.len() { + return Err(ax_diff_error( + "AX diff operations overlap, are unsorted, or exceed the base tree", + )); + } + previous_end = end; + } + for operation in operations.iter().rev() { + let end = operation.start_line + operation.delete_count; + base_lines.splice(operation.start_line..end, operation.lines.clone()); + } + Ok(base_lines.join("\n")) +} + +fn ax_diff_error(message: impl Into) -> NativeError { + NativeError::new( + ErrorCode::AxRevisionMismatch, + super::errors::ErrorPhase::Verify, + true, + message, + ) +} + +pub struct RevisionedAccessibility { + pub public: AccessibilityState, + pub record: AccessibilityRecord, + pub prepared_revision: PreparedAxRevision, +} + +pub fn revision_accessibility( + state: &AxRevisionState, + observation_id: &ObservationId, + native: NativeAccessibilityUpdate, + mode: AxTreeMode, +) -> Result { + let mut element_ids = std::collections::HashSet::with_capacity(native.elements.len()); + for element in &native.elements { + if let Some(bounds) = element.bounds { + bounds.validate().map_err(ax_manifest_error)?; + } + if !element_ids.insert(element.id.clone()) { + return Err(ax_manifest_error( + "native accessibility update returned a duplicate element id", + )); + } + } + if native + .focused_element + .as_ref() + .is_some_and(|id| !element_ids.contains(id)) + { + return Err(ax_manifest_error( + "native accessibility focus references an element outside the manifest", + )); + } + if native + .selected_elements + .iter() + .any(|id| !element_ids.contains(id)) + { + return Err(ax_manifest_error( + "native accessibility selection references an element outside the manifest", + )); + } + let prepared_revision = state.prepare(&native.normalized_tree, mode)?; + let tree_update = prepared_revision.update.clone(); + let revision = tree_update.revision().clone(); + let mut records = HashMap::new(); + let mut elements = Vec::with_capacity(native.elements.len()); + for element in native.elements { + let element_ref = ElementRef { + observation_id: observation_id.clone(), + id: element.id.clone(), + }; + elements.push(AccessibilityElement { + element_ref, + role: element.role.clone(), + label: element.label, + value: element.value, + bounds: element.bounds, + actions: element.actions.clone(), + }); + records.insert( + element.id.clone(), + ElementRecord { + id: element.id, + native: element.native, + owner: element.owner, + role: element.role, + bounds: element.bounds, + actions: element.actions, + ax_revision: revision.clone(), + menu_id: element.menu_id, + }, + ); + } + let focused_element = native.focused_element.as_ref().map(|id| ElementRef { + observation_id: observation_id.clone(), + id: id.clone(), + }); + let selected_elements = native + .selected_elements + .iter() + .map(|id| ElementRef { + observation_id: observation_id.clone(), + id: id.clone(), + }) + .collect(); + Ok(RevisionedAccessibility { + public: AccessibilityState { + tree_update, + elements, + focused_element, + selected_text: native.selected_text, + selected_elements, + document_text: native.document_text, + }, + record: AccessibilityRecord { + revision, + elements: records, + focused_element: native.focused_element, + }, + prepared_revision, + }) +} + +fn ax_manifest_error(message: impl Into) -> NativeError { + NativeError::new( + ErrorCode::AxRevisionMismatch, + super::errors::ErrorPhase::Verify, + false, + message, + ) +} + +#[cfg(test)] +mod store_tests { + use std::sync::atomic::{AtomicUsize, Ordering}; + + use super::*; + use crate::api::contracts::{AppId, AppRef, WindowId}; + + fn window() -> ResolvedWindow { + let public = WindowRef { + id: WindowId::parse("window-1").unwrap(), + app: AppRef { + id: AppId::parse("app-1").unwrap(), + name: Some("Fixture".to_owned()), + pid: Some(100), + running: true, + }, + title: Some("Fixture".to_owned()), + }; + ResolvedWindow { + public, + native: NativeWindowHandle::new("native-window-1").unwrap(), + process: NativeProcessHandle::new("native-process-1").unwrap(), + framework: Framework::AppKit, + geometry: WindowGeometry { + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + scale_factor: 2.0, + revision: GeometryRevision::parse("geometry-1").unwrap(), + }, + generation: WindowGeneration(1), + state: WindowStateKind::Visible, + } + } + + fn record( + id: &str, + bytes: usize, + captured_at: Instant, + cleaned: Arc, + ) -> ObservationRecord { + ObservationRecord { + id: ObservationId::parse(id).unwrap(), + window: window().stamp(), + captured_at, + surfaces: HashMap::new(), + accessibility: None, + menu: MenuState::Closed { + revision: super::super::contracts::MenuRevision::new(), + }, + settlement: SettlementEvidence::initial(), + state: ObservationState::Current, + approximate_bytes: bytes, + artifacts: vec![ObservationArtifactHandle::new(id, move || { + cleaned.fetch_add(1, Ordering::SeqCst); + Ok(()) + })], + } + } + + #[test] + fn count_and_byte_eviction_share_accounting_and_cleanup() { + let cleaned = Arc::new(AtomicUsize::new(0)); + let mut store = ObservationStore::new(2, 15, Duration::from_secs(60)); + store + .insert(record("one", 5, Instant::now(), cleaned.clone())) + .unwrap(); + store + .insert(record("two", 5, Instant::now(), cleaned.clone())) + .unwrap(); + store + .insert(record("three", 10, Instant::now(), cleaned.clone())) + .unwrap(); + + assert_eq!(store.len(), 2); + assert_eq!(store.current_bytes(), 15); + assert_eq!(cleaned.load(Ordering::SeqCst), 1); + assert!(store.remove(&ObservationId::parse("three").unwrap())); + assert_eq!(store.len(), 1); + assert_eq!(store.current_bytes(), 5); + assert_eq!(cleaned.load(Ordering::SeqCst), 2); + assert!(store.remove(&ObservationId::parse("two").unwrap())); + assert!(store.is_empty()); + assert_eq!(store.current_bytes(), 0); + assert_eq!(cleaned.load(Ordering::SeqCst), 3); + } + + #[test] + fn ttl_removal_drops_artifacts_instead_of_retaining_expired_records() { + let cleaned = Arc::new(AtomicUsize::new(0)); + let mut store = ObservationStore::new(4, 1_000, Duration::from_millis(5)); + let id = ObservationId::parse("expired").unwrap(); + store + .insert(record( + "expired", + 100, + Instant::now() - Duration::from_secs(1), + cleaned.clone(), + )) + .unwrap(); + + let error = store.current(&id, &window()).unwrap_err(); + assert_eq!(error.code, ErrorCode::ObservationStale); + assert!(store.is_empty()); + assert_eq!(store.current_bytes(), 0); + assert_eq!(cleaned.load(Ordering::SeqCst), 1); + } + + #[test] + fn insert_expires_old_records_before_capacity_eviction() { + let cleaned = Arc::new(AtomicUsize::new(0)); + let mut store = ObservationStore::new(1, 1_000, Duration::from_secs(1)); + store + .insert(record( + "expired-before-insert", + 900, + Instant::now() - Duration::from_secs(2), + cleaned.clone(), + )) + .unwrap(); + store + .insert(record( + "fresh-after-expiry", + 100, + Instant::now(), + cleaned.clone(), + )) + .unwrap(); + + assert_eq!(store.len(), 1); + assert_eq!(store.current_bytes(), 100); + assert!(store + .records + .contains_key(&ObservationId::parse("fresh-after-expiry").unwrap())); + assert_eq!(cleaned.load(Ordering::SeqCst), 1); + } + + #[test] + fn oversize_insert_fails_before_invalidating_or_evicting_current() { + let cleaned = Arc::new(AtomicUsize::new(0)); + let mut store = ObservationStore::new(2, 10, Duration::from_secs(60)); + let current_id = ObservationId::parse("retained-current").unwrap(); + store + .insert(record( + "retained-current", + 5, + Instant::now(), + cleaned.clone(), + )) + .unwrap(); + + let error = store + .insert(record("oversize", 11, Instant::now(), cleaned.clone())) + .unwrap_err(); + assert_eq!(error.code, ErrorCode::Internal); + assert_eq!(store.len(), 1); + assert_eq!(store.current_bytes(), 5); + assert!(store.current(¤t_id, &window()).is_ok()); + assert_eq!(cleaned.load(Ordering::SeqCst), 1); + } + + #[test] + fn eviction_prefers_invalidated_lru_over_consumed_evidence() { + let cleaned = Arc::new(AtomicUsize::new(0)); + let mut store = ObservationStore::new(2, 1_000, Duration::from_secs(60)); + let consumed = ObservationId::parse("consumed-evidence").unwrap(); + let invalidated = ObservationId::parse("invalidated-evidence").unwrap(); + let newest = ObservationId::parse("new-current").unwrap(); + store + .insert(record( + "consumed-evidence", + 10, + Instant::now(), + cleaned.clone(), + )) + .unwrap(); + store + .consume(&consumed, crate::api::contracts::ActionId::new()) + .unwrap(); + store + .insert(record( + "invalidated-evidence", + 10, + Instant::now(), + cleaned.clone(), + )) + .unwrap(); + store + .insert(record("new-current", 10, Instant::now(), cleaned.clone())) + .unwrap(); + + assert!(store.records.contains_key(&consumed)); + assert!(!store.records.contains_key(&invalidated)); + assert!(store.records.contains_key(&newest)); + } + + #[test] + fn element_point_uses_the_surface_owned_by_the_exact_related_window() { + let target = window(); + let mut related = target.stamp(); + related.window_id = WindowId::parse("related-1").unwrap(); + related.generation = WindowGeneration(2); + related.geometry_revision = GeometryRevision::parse("related-geometry-1").unwrap(); + related.native_window = NativeWindowHandle::new("native-related-1").unwrap(); + let related_window = WindowRef { + id: related.window_id.clone(), + app: target.public.app.clone(), + title: Some("Popover".to_owned()), + }; + let observation_id = ObservationId::parse("related-observation").unwrap(); + let element_id = ElementId::parse("related-element").unwrap(); + let related_surface_id = SurfaceId::parse("related-surface").unwrap(); + let cleaned = Arc::new(AtomicUsize::new(0)); + let mut observation = record(observation_id.as_str(), 1, Instant::now(), cleaned); + observation.surfaces.insert( + related_surface_id.clone(), + SurfaceRecord { + id: related_surface_id.clone(), + kind: SurfaceKind::Popover, + owner_window: related_window, + image_url: "file:///tmp/related.png".to_owned(), + raster_size: Size { + width: 100, + height: 100, + }, + window_bounds: None, + capture_revision: CaptureRevision::parse("related-capture").unwrap(), + transform: SurfaceToWindowTransform { + scale_x: 0.5, + scale_y: 0.5, + offset_x: 20.0, + offset_y: 30.0, + }, + freshness: CaptureFreshness::Fresh, + owner: SurfaceOwner::RelatedTransient { + owner: related.clone(), + parent: target.stamp(), + }, + approximate_bytes: 1, + }, + ); + observation.accessibility = Some(AccessibilityRecord { + revision: AxRevision::parse("ax-related").unwrap(), + elements: HashMap::from([( + element_id.clone(), + ElementRecord { + id: element_id.clone(), + native: NativeElementHandle::new("native-related-element").unwrap(), + owner: related, + role: Some("AXButton".to_owned()), + bounds: Some(Rect { + x: 30.0, + y: 40.0, + width: 10.0, + height: 10.0, + }), + actions: vec!["AXPress".to_owned()], + ax_revision: AxRevision::parse("ax-related").unwrap(), + menu_id: None, + }, + )]), + focused_element: None, + }); + let mut store = ObservationStore::default(); + store.insert(observation).unwrap(); + + let point = store + .resolve_element_point( + &target, + &ElementRef { + observation_id, + id: element_id, + }, + ) + .unwrap(); + assert_eq!(point.surface_id, related_surface_id); + assert_eq!(point.surface_point, Point { x: 30.0, y: 30.0 }); + assert_eq!(point.window_point, Point { x: 35.0, y: 45.0 }); + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/platform.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/platform.rs new file mode 100644 index 0000000000..21e24b9805 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/platform.rs @@ -0,0 +1,380 @@ +//! Portable platform-provider seams for the v2 controller. + +use std::time::Instant; + +use async_trait::async_trait; + +use super::{ + capabilities::CapabilityManifest, + contracts::{ + AppId, AppQuery, AppRef, AppSelector, ElementRef, KeyStroke, Modifier, MouseButton, + Readiness, ScrollDirection, SelectionType, WindowGeneration, WindowId, WindowRef, + }, + errors::NativeError, + interaction::{ + InteractionScope, MutationDeadline, NativeEvidence, PostureResult, ScopePlan, + ScopeRequirements, TargetCursorHandle, + }, + menu::NativeMenuEvidence, + observation::{ + InvalidationReason, NativeObservationUpdate, NativeProcessHandle, ResolvedDrag, + ResolvedElement, ResolvedFocus, ResolvedPoint, ResolvedScroll, ResolvedWindow, + }, + settlement::{DirtyState, SettlementAttempt, SettlementEvidence}, + Route, VerificationLevel, +}; + +#[derive(Debug, Clone)] +pub struct NativeLaunch { + pub app: AppRef, + pub windows: Vec, + pub reused_running_app: bool, + pub posture: PostureResult, + pub settlement: SettlementEvidence, +} + +#[derive(Debug, Clone, Default)] +pub struct LaunchPostureScope { + pub posture: PostureResult, + pub native_evidence: NativeEvidence, + pub partial_app: Option, + pub partial_windows: Vec, + pub pending_settlement: Option, + action_id: Option, + side_effect_started: bool, +} + +impl LaunchPostureScope { + pub fn for_action(action_id: super::contracts::ActionId) -> Self { + Self { + action_id: Some(action_id), + ..Self::default() + } + } + + pub fn action_id(&self) -> Option<&super::contracts::ActionId> { + self.action_id.as_ref() + } + + pub fn begin_launch(&mut self) { + self.side_effect_started = true; + } + + pub fn side_effect_started(&self) -> bool { + self.side_effect_started + } + + pub fn record_partial_result(&mut self, app: AppRef, windows: Vec) { + self.partial_app = Some(app); + self.partial_windows = windows; + } +} + +#[derive(Debug, Clone, Copy)] +pub struct ObserveRequest { + pub include_text: bool, + pub include_screenshots: bool, +} + +#[derive(Debug, Clone)] +pub struct ClickSpec { + pub button: MouseButton, + pub click_count: u8, + pub modifiers: Vec, +} + +#[derive(Debug, Clone)] +pub struct ElementScrollSpec { + pub direction: ScrollDirection, + pub pages: u16, +} + +#[derive(Debug, Clone)] +pub struct SelectionSpec { + pub text: String, + pub prefix: Option, + pub suffix: Option, + pub selection_type: SelectionType, +} + +#[derive(Debug, Clone)] +pub struct NativeDispatch { + pub verification: VerificationLevel, + pub evidence: NativeEvidence, + pub warnings: Vec, + pub menu: Option, +} + +#[derive(Debug, Clone)] +pub enum ResolvedAction { + ElementClick { + source: ElementRef, + element: ResolvedElement, + spec: ClickSpec, + }, + PointClick { + point: ResolvedPoint, + spec: ClickSpec, + }, + Drag(ResolvedDrag), + ElementScroll { + element: ResolvedElement, + spec: ElementScrollSpec, + }, + DeltaScroll(ResolvedScroll), + PressKey { + focus: ResolvedFocus, + stroke: KeyStroke, + }, + TypeText { + focus: ResolvedFocus, + text: String, + }, + SetValue { + element: ResolvedElement, + value: String, + }, + SelectText { + element: ResolvedElement, + selection: SelectionSpec, + }, + Secondary { + element: ResolvedElement, + action: String, + }, +} + +impl NativeDispatch { + pub fn dispatch_verified() -> Self { + Self { + verification: VerificationLevel::DispatchVerified, + evidence: NativeEvidence::default(), + warnings: Vec::new(), + menu: None, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum TargetInvalidation { + /// The native event source overran, so core cannot prove which cached + /// target identities remain valid. Every target must be torn down and + /// explicitly re-observed from an exact platform snapshot. + NativeStateResyncRequired, + ProcessExited { + process: NativeProcessHandle, + }, + WindowGenerationChanged { + app_id: AppId, + window_id: WindowId, + previous: WindowGeneration, + current: WindowGeneration, + }, + /// Native state changed without destroying the target controller. Core + /// invalidates perception state under the target lock; the platform + /// remains the producer of the signal, not a second state-machine owner. + ObservationChanged { + app_id: AppId, + window_id: WindowId, + generation: WindowGeneration, + reason: InvalidationReason, + }, +} + +#[async_trait] +pub trait InvalidationSubscription: Send { + async fn next(&mut self) -> Option; +} + +#[async_trait] +pub trait TargetFocusCoordinator: Send { + async fn shutdown(&mut self) -> Result<(), NativeError>; +} + +#[async_trait] +pub trait LifecycleProvider: Send + Sync { + async fn readiness(&self) -> Result; + async fn capabilities(&self) -> Result; + async fn list_apps(&self, query: AppQuery) -> Result, NativeError>; + async fn launch_background( + &self, + app: AppSelector, + posture: &mut LaunchPostureScope, + ) -> Result; +} + +#[async_trait] +pub trait WindowProvider: Send + Sync { + async fn list_windows(&self, app: Option<&AppRef>) -> Result, NativeError>; + async fn rehydrate( + &self, + id: &WindowId, + app: Option<&AppRef>, + ) -> Result; + async fn resolve(&self, window: &WindowRef) -> Result; +} + +#[async_trait] +pub trait ObservationProvider: Send + Sync { + async fn settle( + &self, + target: &mut TargetState, + dirty: &DirtyState, + deadline: Instant, + ) -> Result; + async fn observe( + &self, + target: &mut TargetState, + window: &ResolvedWindow, + request: ObserveRequest, + ) -> Result; +} + +/// Two-phase semantic operations under the locked native target state. +/// +/// `prepare` is side-effect free and runs after scope acquisition but before +/// observation consumption or dirty marking. It owns every fallible shape, +/// identity, exposure, settable, selection, and route proof and returns a +/// retained typed plan. `dispatch` is the controller-owned dispatch boundary: +/// method entry consumes the observation and permits only the selected native +/// operation and its post-dispatch verification readback. +#[async_trait] +pub trait SemanticActionProvider: Send + Sync { + type PreparedAction: Send; + + async fn prepare( + &self, + target: &mut TargetState, + scope: &mut InteractionScope, + action: &ResolvedAction, + ) -> Result; + + async fn dispatch( + &self, + target: &mut TargetState, + scope: &mut InteractionScope, + action: Self::PreparedAction, + ) -> Result; +} + +/// Dispatch-only targeted pointer operations. +/// +/// Method entry is the controller-owned dispatch boundary. Implementations do +/// not perform a second preflight and must treat cancellation as a potentially +/// partial dispatch. +#[async_trait] +pub trait PointerActionProvider: Send + Sync { + async fn click( + &self, + scope: &mut InteractionScope, + point: ResolvedPoint, + click: ClickSpec, + ) -> Result; + async fn drag( + &self, + scope: &mut InteractionScope, + drag: ResolvedDrag, + ) -> Result; + async fn scroll( + &self, + scope: &mut InteractionScope, + scroll: ResolvedScroll, + ) -> Result; +} + +/// Dispatch-only keyboard operations against a core-resolved focus token. +/// +/// Method entry occurs after observation consumption and dirty marking. Any +/// fallible focus or route preparation belongs in `InteractionProvider`. +#[async_trait] +pub trait KeyboardActionProvider: Send + Sync { + async fn press_key( + &self, + scope: &mut InteractionScope, + focus: &ResolvedFocus, + stroke: KeyStroke, + ) -> Result; + async fn type_text( + &self, + scope: &mut InteractionScope, + focus: &ResolvedFocus, + text: &str, + ) -> Result; +} + +#[async_trait] +pub trait InteractionProvider: Send + Sync { + type NativeScopePlan: Send; + + /// Route-specific native preflight. This is the final fallible preparation + /// seam and runs before scope acquisition, observation consumption, or + /// dirty marking. The returned native plan is action-aware and is carried + /// by value into acquisition; providers must not use mutable pending + /// recipe state to connect these calls. Preflight must be side-effect free: + /// it may not acquire native resources or dispatch, and cancellation at + /// its controller-owned deadline must leave the target reusable. + #[allow(clippy::too_many_arguments)] // Each action/target/route invariant is explicit. + async fn preflight( + &self, + target: &mut TargetState, + focus: &mut Focus, + action_id: &super::contracts::ActionId, + window: &ResolvedWindow, + route: Route, + action: &ResolvedAction, + deadline: MutationDeadline, + requirements: ScopeRequirements, + ) -> Result, NativeError>; + + /// Acquires the leases in `plan`. Every partially acquired resource must + /// be owned by a cancellation-safe RAII guard until the complete scope is + /// returned; a cancelled acquisition causes core to poison the target + /// because complete teardown evidence is unavailable. + async fn acquire_scope( + &self, + target: &mut TargetState, + focus: &mut Focus, + plan: ScopePlan, + logical_cursor: TargetCursorHandle, + ) -> Result; +} + +#[async_trait] +pub trait PlatformDriver: Send + Sync + 'static { + type TargetState: Send; + type TargetFocusCoordinator: TargetFocusCoordinator; + type Lifecycle: LifecycleProvider; + type Windows: WindowProvider; + type Observation: ObservationProvider; + type Semantic: SemanticActionProvider; + type Pointer: PointerActionProvider; + type Keyboard: KeyboardActionProvider; + type Interaction: InteractionProvider; + type Invalidations: InvalidationSubscription; + + async fn create_target_state( + &self, + window: &ResolvedWindow, + ) -> Result<(Self::TargetState, Self::TargetFocusCoordinator), NativeError>; + + async fn destroy_target_state( + &self, + target: &mut Self::TargetState, + focus: &mut Self::TargetFocusCoordinator, + ) -> Result<(), NativeError> { + let _ = target; + focus.shutdown().await + } + + fn lifecycle(&self) -> &Self::Lifecycle; + fn windows(&self) -> &Self::Windows; + fn observation(&self) -> &Self::Observation; + fn semantic(&self) -> &Self::Semantic; + fn pointer(&self) -> &Self::Pointer; + fn keyboard(&self) -> &Self::Keyboard; + fn interaction(&self) -> &Self::Interaction; + /// Complete, conservative capability matrix for this platform/version. + /// Core seeds its immutable routing registry exclusively from these cells. + fn capability_cells(&self, os_version: &str) -> Vec; + fn subscribe_invalidations(&self) -> Self::Invalidations; +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/settlement.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/settlement.rs new file mode 100644 index 0000000000..e61f718279 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/settlement.rs @@ -0,0 +1,312 @@ +//! Notification-driven dirty and settlement state. + +use std::{collections::BTreeSet, time::Instant}; + +use serde::{Deserialize, Serialize}; + +use super::{contracts::ActionId, errors::NativeError}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SettlementSignal { + DispatchStarted, + DispatchComplete, + AxAction, + AxValueChanged, + FocusChanged, + MenuOpened, + MenuDismissed, + WindowListChanged, + WindowGeometryChanged, + ScrollChanged, + FreshFrame, + /// A promised verification readback reached a terminal result. The + /// signal records completion, not success: an exact mismatch still lets + /// settlement finish so `VerificationFailed` remains the primary error. + VerificationReadbackComplete, + RunLoopDrained, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SettlementProfile { + pub name: String, + pub required_terminal_signals: BTreeSet, + /// Native target signals that reset the quiet window after terminal + /// evidence exists. Keeping this profile-specific prevents unrelated or + /// continuously animated state from holding every action dirty. + pub relevant_signals: BTreeSet, + pub quiet_window_ms: u64, + pub deadline_ms: u64, +} + +impl SettlementProfile { + pub fn dispatch_only(name: impl Into) -> Self { + Self { + name: name.into(), + required_terminal_signals: BTreeSet::new(), + relevant_signals: BTreeSet::new(), + quiet_window_ms: 30, + deadline_ms: 2_000, + } + } + + pub fn requiring( + name: impl Into, + required_terminal_signals: impl IntoIterator, + ) -> Self { + let required_terminal_signals: BTreeSet<_> = + required_terminal_signals.into_iter().collect(); + Self { + name: name.into(), + relevant_signals: required_terminal_signals.clone(), + required_terminal_signals, + quiet_window_ms: 30, + deadline_ms: 2_000, + } + } + + pub fn with_relevant_signals( + mut self, + relevant_signals: impl IntoIterator, + ) -> Self { + self.relevant_signals = relevant_signals.into_iter().collect(); + self + } +} + +#[derive(Debug, Clone)] +pub struct DirtyState { + pub action_id: ActionId, + pub profile: SettlementProfile, + pub since: Instant, + pub observed_signals: BTreeSet, + pub resumed_from_prior_call: bool, +} + +impl DirtyState { + pub fn pending_evidence(&self) -> PendingSettlementEvidence { + let mut missing_signals: Vec<_> = self + .profile + .required_terminal_signals + .difference(&self.observed_signals) + .copied() + .collect(); + if !self + .observed_signals + .contains(&SettlementSignal::DispatchComplete) + { + missing_signals.push(SettlementSignal::DispatchComplete); + } + PendingSettlementEvidence { + state: PendingSettlementState::Pending, + trigger_action_id: self.action_id.clone(), + profile: self.profile.name.clone(), + elapsed_ms: self.since.elapsed().as_millis().min(u128::from(u64::MAX)) as u64, + observed_signals: self.observed_signals.iter().copied().collect(), + missing_signals, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SettledState { + Settled, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct SettlementEvidence { + pub state: SettledState, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub trigger_action_id: Option, + pub profile: String, + pub elapsed_ms: u64, + #[serde(default)] + pub observed_signals: Vec, + pub terminal_signal: String, + pub quiet_window_ms: u64, + #[serde(default)] + pub resumed_from_prior_call: bool, +} + +impl SettlementEvidence { + pub fn initial() -> Self { + Self { + state: SettledState::Settled, + trigger_action_id: None, + profile: "initial".to_owned(), + elapsed_ms: 0, + observed_signals: Vec::new(), + terminal_signal: "already_settled".to_owned(), + quiet_window_ms: 0, + resumed_from_prior_call: false, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum PendingSettlementState { + Pending, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct PendingSettlementEvidence { + pub state: PendingSettlementState, + pub trigger_action_id: ActionId, + pub profile: String, + pub elapsed_ms: u64, + #[serde(default)] + pub observed_signals: Vec, + #[serde(default)] + pub missing_signals: Vec, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SettlementAttempt { + Settled(SettlementEvidence), + Pending(PendingSettlementEvidence), +} + +#[derive(Debug, Clone)] +pub enum SettlementState { + Settled(SettlementEvidence), + Dirty(DirtyState), + Settling(DirtyState), +} + +impl Default for SettlementState { + fn default() -> Self { + Self::Settled(SettlementEvidence::initial()) + } +} + +impl SettlementState { + pub fn mark_dirty( + &mut self, + action_id: ActionId, + profile: SettlementProfile, + ) -> Result<(), NativeError> { + if !matches!(self, Self::Settled(_)) { + return Err(NativeError::invalid( + "cannot dispatch a new mutation while prior target settlement is pending", + )); + } + let mut observed_signals = BTreeSet::new(); + observed_signals.insert(SettlementSignal::DispatchStarted); + *self = Self::Dirty(DirtyState { + action_id, + profile, + since: Instant::now(), + observed_signals, + resumed_from_prior_call: false, + }); + Ok(()) + } + + pub fn begin(&mut self, resumed_from_prior_call: bool) -> Option { + let current = match self { + Self::Dirty(dirty) | Self::Settling(dirty) => dirty.clone(), + Self::Settled(_) => return None, + }; + let mut settling = current; + settling.resumed_from_prior_call |= resumed_from_prior_call; + *self = Self::Settling(settling.clone()); + Some(settling) + } + + pub fn record_signal(&mut self, signal: SettlementSignal) -> Result<(), NativeError> { + match self { + Self::Dirty(dirty) | Self::Settling(dirty) => { + dirty.observed_signals.insert(signal); + Ok(()) + } + Self::Settled(_) => Err(NativeError::invalid( + "cannot record a settlement signal for a clean target", + )), + } + } + + pub fn complete( + &mut self, + evidence: SettlementEvidence, + ) -> Result { + let dirty = match self { + Self::Dirty(dirty) | Self::Settling(dirty) => dirty, + Self::Settled(_) => { + return Err(NativeError::invalid( + "cannot complete settlement for a target that is already settled", + )) + } + }; + if evidence.trigger_action_id.as_ref() != Some(&dirty.action_id) + || evidence.profile != dirty.profile.name + { + return Err(NativeError::invalid( + "settlement evidence does not match the dirty action/profile", + )); + } + let observed: BTreeSet<_> = evidence.observed_signals.iter().copied().collect(); + if !observed.contains(&SettlementSignal::DispatchComplete) { + return Err(NativeError::invalid( + "settlement evidence cannot complete before dispatch completion", + )); + } + if !dirty.profile.required_terminal_signals.is_subset(&observed) { + return Err(NativeError::invalid( + "settlement evidence is missing required terminal signals", + )); + } + *self = Self::Settled(evidence.clone()); + Ok(evidence) + } + + pub fn preserve_dirty_after_timeout(&mut self) { + if let Self::Settling(dirty) = self { + let mut dirty = dirty.clone(); + dirty.resumed_from_prior_call = true; + *self = Self::Dirty(dirty); + } + } + + pub fn preserve_pending( + &mut self, + pending: &PendingSettlementEvidence, + ) -> Result<(), NativeError> { + let dirty = match self { + Self::Dirty(dirty) | Self::Settling(dirty) => dirty, + Self::Settled(_) => { + return Err(NativeError::invalid( + "cannot preserve progress for an already-settled target", + )) + } + }; + if pending.trigger_action_id != dirty.action_id || pending.profile != dirty.profile.name { + return Err(NativeError::invalid( + "pending settlement evidence does not match the dirty action/profile", + )); + } + let mut next = dirty.clone(); + next.observed_signals + .extend(pending.observed_signals.iter().copied()); + next.resumed_from_prior_call = true; + *self = Self::Dirty(next); + Ok(()) + } + + pub fn settled_evidence(&self) -> Option<&SettlementEvidence> { + match self { + Self::Settled(evidence) => Some(evidence), + _ => None, + } + } + + pub fn pending_evidence(&self) -> Option { + match self { + Self::Dirty(dirty) | Self::Settling(dirty) => Some(dirty.pending_evidence()), + Self::Settled(_) => None, + } + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/target.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/target.rs new file mode 100644 index 0000000000..e9220d42b2 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/target.rs @@ -0,0 +1,419 @@ +//! Long-lived per-client/window target ownership and process mutation locks. + +use std::{ + collections::HashMap, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, Weak, + }, + time::{Duration, Instant}, +}; + +use tokio::sync::Mutex as AsyncMutex; + +use super::{ + contracts::{AppId, ClientId, WindowGeneration, WindowId}, + errors::{ErrorCode, NativeError}, + interaction::TargetCursorHandle, + menu::MenuControllerState, + observation::{ + AxRevisionState, InvalidationReason, NativeProcessHandle, ObservationStore, ResolvedWindow, + }, + platform::{InvalidationSubscription, PlatformDriver, TargetInvalidation}, + settlement::SettlementState, +}; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct TargetKey { + pub client_id: ClientId, + pub app_id: AppId, + pub window_id: WindowId, + pub window_generation: WindowGeneration, +} + +impl TargetKey { + pub fn from_window(client_id: ClientId, window: &ResolvedWindow) -> Self { + Self { + client_id, + app_id: window.public.app.id.clone(), + window_id: window.public.id.clone(), + window_generation: window.generation, + } + } +} + +pub type SharedProcessMutationLock = Arc>; + +#[derive(Debug, Clone)] +pub(crate) struct TargetValidityHandle(Arc); + +impl TargetValidityHandle { + fn new() -> Self { + Self(Arc::new(AtomicBool::new(true))) + } + + pub(crate) fn invalidate(&self) { + self.0.store(false, Ordering::Release); + } + + fn is_valid(&self) -> bool { + self.0.load(Ordering::Acquire) + } +} + +#[derive(Debug, Default)] +pub struct ProcessMutationLockRegistry { + locks: Mutex>>>, +} + +impl ProcessMutationLockRegistry { + pub fn lock_for(&self, process: &NativeProcessHandle) -> SharedProcessMutationLock { + let mut locks = self.locks.lock().expect("process lock registry poisoned"); + if let Some(lock) = locks.get(process).and_then(Weak::upgrade) { + return lock; + } + let lock = Arc::new(AsyncMutex::new(())); + locks.insert(process.clone(), Arc::downgrade(&lock)); + lock + } +} + +pub struct TargetControllerState { + pub window: ResolvedWindow, + pub platform: P::TargetState, + pub focus: P::TargetFocusCoordinator, + pub ax_revisions: AxRevisionState, + pub observations: ObservationStore, + pub logical_cursor: TargetCursorHandle, + pub menu: MenuControllerState, + pub settlement: SettlementState, +} + +pub struct TargetController { + pub key: TargetKey, + pub process: NativeProcessHandle, + pub mutation_lock: SharedProcessMutationLock, + pub state: AsyncMutex>, + last_used: Mutex, + validity: TargetValidityHandle, + torn_down: AtomicBool, +} + +impl TargetController

{ + fn new( + key: TargetKey, + window: ResolvedWindow, + platform: P::TargetState, + focus: P::TargetFocusCoordinator, + mutation_lock: SharedProcessMutationLock, + ) -> Self { + Self { + key, + process: window.process.clone(), + mutation_lock, + state: AsyncMutex::new(TargetControllerState { + window, + platform, + focus, + ax_revisions: AxRevisionState::default(), + observations: ObservationStore::default(), + logical_cursor: TargetCursorHandle::default(), + menu: MenuControllerState::default(), + settlement: SettlementState::default(), + }), + last_used: Mutex::new(Instant::now()), + validity: TargetValidityHandle::new(), + torn_down: AtomicBool::new(false), + } + } + + pub fn touch(&self) { + *self + .last_used + .lock() + .expect("target last-used lock poisoned") = Instant::now(); + } + + pub fn idle_for(&self) -> Duration { + self.last_used + .lock() + .expect("target last-used lock poisoned") + .elapsed() + } + + pub fn ensure_valid(&self) -> Result<(), NativeError> { + if self.validity.is_valid() { + Ok(()) + } else { + Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "target controller was invalidated by native lifecycle state", + )) + } + } + + pub fn invalidate(&self) { + self.validity.invalidate(); + } + + pub(crate) fn validity_handle(&self) -> TargetValidityHandle { + self.validity.clone() + } + + pub async fn teardown(&self, platform: &P) -> Result<(), NativeError> { + self.invalidate(); + if self.torn_down.swap(true, Ordering::AcqRel) { + return Ok(()); + } + let mut state = self.state.lock().await; + state + .observations + .invalidate_all(InvalidationReason::WindowChanged); + state.ax_revisions.reset(); + state.menu.close(); + let TargetControllerState { + platform: target_state, + focus, + .. + } = &mut *state; + platform.destroy_target_state(target_state, focus).await + } +} + +pub struct TargetControllerRegistry { + targets: AsyncMutex>>>, + process_locks: Arc, + idle_ttl: Duration, +} + +impl TargetControllerRegistry

{ + pub fn new(process_locks: Arc, idle_ttl: Duration) -> Self { + Self { + targets: AsyncMutex::new(HashMap::new()), + process_locks, + idle_ttl, + } + } + + pub async fn get_or_create( + &self, + platform: &P, + key: TargetKey, + window: ResolvedWindow, + ) -> Result>, NativeError> { + let existing = { self.targets.lock().await.get(&key).cloned() }; + if let Some(target) = existing { + if target.ensure_valid().is_ok() { + target.touch(); + return Ok(target); + } + let invalid = self.remove_matching(|candidate, _| candidate == &key).await; + teardown_all(platform, invalid).await?; + } + + let superseded = self + .remove_matching(|candidate, _| { + candidate.client_id == key.client_id + && candidate.app_id == key.app_id + && candidate.window_id == key.window_id + && candidate.window_generation != key.window_generation + }) + .await; + teardown_all(platform, superseded).await?; + + // Native creation/teardown never runs while the registry mutex is + // held. A racing creator is resolved by discarding this extra state. + let (mut target_state, mut focus) = platform.create_target_state(&window).await?; + let mutation_lock = self.process_locks.lock_for(&window.process); + let mut targets = self.targets.lock().await; + if let Some(target) = targets.get(&key).cloned() { + drop(targets); + platform + .destroy_target_state(&mut target_state, &mut focus) + .await?; + target.ensure_valid()?; + target.touch(); + return Ok(target); + } + let target = Arc::new(TargetController::new( + key.clone(), + window, + target_state, + focus, + mutation_lock, + )); + targets.insert(key, Arc::clone(&target)); + Ok(target) + } + + pub async fn get(&self, key: &TargetKey) -> Result>, NativeError> { + let targets = self.targets.lock().await; + let target = targets.get(key).cloned().ok_or_else(|| { + NativeError::stale( + ErrorCode::ObservationStale, + "target controller does not exist; observe the window before mutating it", + ) + })?; + target.ensure_valid()?; + target.touch(); + Ok(target) + } + + /// Removes and tears down one exact target after a native lease failure. + /// The target is invalidated before callers release their state lock, so + /// no racing request can reuse it between failure detection and removal. + pub async fn remove_invalid_target( + &self, + platform: &P, + key: &TargetKey, + ) -> Result { + let target = self.targets.lock().await.remove(key); + let Some(target) = target else { + return Ok(false); + }; + target.teardown(platform).await?; + Ok(true) + } + + pub async fn close_connection( + &self, + platform: &P, + client_id: &ClientId, + ) -> Result { + let targets = self + .remove_matching(|key, _| &key.client_id == client_id) + .await; + teardown_all(platform, targets).await + } + + pub async fn expire_idle(&self, platform: &P) -> Result { + let idle_ttl = self.idle_ttl; + let targets = self + .remove_matching(|_, target| target.idle_for() >= idle_ttl) + .await; + teardown_all(platform, targets).await + } + + pub async fn handle_invalidation( + &self, + platform: &P, + invalidation: TargetInvalidation, + ) -> Result { + if let TargetInvalidation::ObservationChanged { + app_id, + window_id, + generation, + reason, + } = &invalidation + { + let targets: Vec<_> = { + let registry = self.targets.lock().await; + registry + .iter() + .filter(|(key, _)| { + &key.app_id == app_id + && &key.window_id == window_id + && &key.window_generation == generation + }) + .map(|(_, target)| Arc::clone(target)) + .collect() + }; + for target in &targets { + let mut state = target.state.lock().await; + state.observations.invalidate_all(reason.clone()); + if matches!( + reason, + &InvalidationReason::AccessibilityInvalidated + | &InvalidationReason::DiffBaseInvalidated + | &InvalidationReason::DisplayChanged + ) { + state.ax_revisions.invalidate_base(); + } + if matches!( + reason, + &InvalidationReason::TransientDismissed | &InvalidationReason::MenuChanged + ) { + state.menu.close(); + } + } + return Ok(targets.len()); + } + let targets = match invalidation { + TargetInvalidation::NativeStateResyncRequired => { + self.remove_matching(|_, _| true).await + } + TargetInvalidation::ProcessExited { process } => { + self.remove_matching(|_, target| target.process == process) + .await + } + TargetInvalidation::WindowGenerationChanged { + app_id, + window_id, + previous, + .. + } => { + self.remove_matching(|key, _| { + key.app_id == app_id + && key.window_id == window_id + && key.window_generation == previous + }) + .await + } + TargetInvalidation::ObservationChanged { .. } => unreachable!( + "observation invalidations return before destructive invalidation matching" + ), + }; + teardown_all(platform, targets).await + } + + pub async fn invalidation_loop( + self: Arc, + platform: Arc

, + mut subscription: P::Invalidations, + ) { + while let Some(invalidation) = subscription.next().await { + if let Err(error) = self.handle_invalidation(&platform, invalidation).await { + tracing::error!(error = %error, "failed to tear down invalidated v2 target"); + } + } + } + + pub async fn len(&self) -> usize { + self.targets.lock().await.len() + } + + pub async fn is_empty(&self) -> bool { + self.targets.lock().await.is_empty() + } + + async fn remove_matching( + &self, + predicate: impl Fn(&TargetKey, &TargetController

) -> bool, + ) -> Vec>> { + let mut targets = self.targets.lock().await; + let keys: Vec<_> = targets + .iter() + .filter_map(|(key, target)| predicate(key, target).then_some(key.clone())) + .collect(); + keys.into_iter() + .filter_map(|key| targets.remove(&key)) + .collect() + } +} + +async fn teardown_all( + platform: &P, + targets: Vec>>, +) -> Result { + let count = targets.len(); + let mut failures = Vec::new(); + for target in targets { + if let Err(error) = target.teardown(platform).await { + failures.push(error); + } + } + if let Some(error) = NativeError::primary(failures) { + return Err(error); + } + Ok(count) +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs index 18b3d4ef72..a8955da539 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/lib.rs @@ -32,6 +32,7 @@ pub fn embedded_mode() -> bool { std::env::var_os(EMBEDDED_ENV).is_some_and(|v| v == "1") } +pub mod api; pub mod capture_mode; pub mod cdp; pub mod element_cache; diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs index ff024628bf..174d9e1a7c 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs @@ -3,6 +3,13 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; +use crate::api::{ + CheckReadinessRequest, ClickCommand, DragCommand, GetCapabilitiesRequest, GetWindowRequest, + GetWindowStateRequest, LaunchAppRequest, ListAppsRequest, ListWindowsRequest, NativeError, + PressKeyCommand, ScrollCommand, SecondaryActionCommand, SelectTextCommand, SetValueCommand, + TypeTextCommand, +}; + // ── Request ────────────────────────────────────────────────────────────────── /// An incoming JSON-RPC 2.0 request or notification. @@ -41,6 +48,128 @@ pub struct ToolCall { pub args: Value, } +// ── Background driver v2 ──────────────────────────────────────────────────── + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct V2ProtocolVersion { + pub major: u16, + pub minor: u16, +} + +pub const V2_PROTOCOL_VERSION: V2ProtocolVersion = V2ProtocolVersion { major: 2, minor: 0 }; + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(tag = "method", content = "params", deny_unknown_fields)] +pub enum V2Command { + #[serde(rename = "driver.v2.check_readiness")] + CheckReadiness(CheckReadinessRequest), + #[serde(rename = "driver.v2.get_capabilities")] + GetCapabilities(GetCapabilitiesRequest), + #[serde(rename = "driver.v2.list_apps")] + ListApps(ListAppsRequest), + #[serde(rename = "driver.v2.launch_app")] + LaunchApp(LaunchAppRequest), + #[serde(rename = "driver.v2.list_windows")] + ListWindows(ListWindowsRequest), + #[serde(rename = "driver.v2.get_window")] + GetWindow(GetWindowRequest), + #[serde(rename = "driver.v2.get_window_state")] + GetWindowState(GetWindowStateRequest), + #[serde(rename = "driver.v2.click")] + Click(ClickCommand), + #[serde(rename = "driver.v2.drag")] + Drag(DragCommand), + #[serde(rename = "driver.v2.scroll")] + Scroll(ScrollCommand), + #[serde(rename = "driver.v2.press_key")] + PressKey(PressKeyCommand), + #[serde(rename = "driver.v2.type_text")] + TypeText(TypeTextCommand), + #[serde(rename = "driver.v2.set_value")] + SetValue(SetValueCommand), + #[serde(rename = "driver.v2.select_text")] + SelectText(SelectTextCommand), + #[serde(rename = "driver.v2.perform_secondary_action")] + PerformSecondaryAction(SecondaryActionCommand), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct V2RequestEnvelope { + pub request_id: String, + pub protocol_version: V2ProtocolVersion, + #[serde(flatten)] + pub command: V2Command, +} + +impl V2RequestEnvelope { + pub fn validate_version(&self) -> Result<(), NativeError> { + if self.protocol_version != V2_PROTOCOL_VERSION { + return Err(NativeError::new( + crate::api::ErrorCode::ProtocolMismatch, + crate::api::ErrorPhase::Validate, + false, + format!( + "unsupported v2 protocol {}.{}; driver requires {}.{}", + self.protocol_version.major, + self.protocol_version.minor, + V2_PROTOCOL_VERSION.major, + V2_PROTOCOL_VERSION.minor + ), + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct V2HandshakeRequest { + pub request_id: String, + pub minimum_version: V2ProtocolVersion, + pub maximum_version: V2ProtocolVersion, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct V2HandshakeResponse { + pub request_id: String, + pub minimum_version: V2ProtocolVersion, + pub maximum_version: V2ProtocolVersion, + pub driver_name: String, + pub driver_version: String, + pub build: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct V2ResponseEnvelope { + pub request_id: String, + pub protocol_version: V2ProtocolVersion, + #[serde(flatten)] + pub body: V2ResponseBody, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum V2ResponseBody { + Result(V2Success), + Error(V2Failure), +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct V2Success { + pub result: T, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct V2Failure { + pub error: NativeError, +} + // ── Response ───────────────────────────────────────────────────────────────── #[derive(Debug, Serialize)] diff --git a/libs/cua-driver/rust/crates/cua-driver-core/tests/api_v2_contracts.rs b/libs/cua-driver/rust/crates/cua-driver-core/tests/api_v2_contracts.rs new file mode 100644 index 0000000000..717c408722 --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/tests/api_v2_contracts.rs @@ -0,0 +1,2168 @@ +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, Mutex, + }, + time::{Duration, Instant}, +}; + +use async_trait::async_trait; +use cua_driver_core::{ + api::*, + protocol::{ + V2Command, V2Failure, V2RequestEnvelope, V2ResponseBody, V2ResponseEnvelope, + V2_PROTOCOL_VERSION, + }, +}; +use tokio::sync::Notify; + +fn id(value: &str, parse: impl FnOnce(String) -> Result) -> T { + parse(value.to_owned()).expect("valid test id") +} + +fn test_mutation_deadline() -> MutationDeadline { + let work = Instant::now() + Duration::from_secs(30); + MutationDeadline::new(work, work + Duration::from_secs(5)).unwrap() +} + +fn app_ref() -> AppRef { + AppRef { + id: id("app-1", AppId::parse), + name: Some("Fixture".to_owned()), + pid: Some(42), + running: true, + } +} + +fn window_ref() -> WindowRef { + WindowRef { + id: id("window-1", WindowId::parse), + app: app_ref(), + title: Some("Fixture window".to_owned()), + } +} + +fn resolved_window() -> ResolvedWindow { + ResolvedWindow { + public: window_ref(), + native: NativeWindowHandle::new("native-window-1").unwrap(), + process: NativeProcessHandle::new("process-42").unwrap(), + framework: Framework::AppKit, + geometry: WindowGeometry { + bounds: Rect { + x: 100.0, + y: 200.0, + width: 640.0, + height: 480.0, + }, + scale_factor: 1.0, + revision: id("geometry-1", GeometryRevision::parse), + }, + generation: WindowGeneration(1), + state: WindowStateKind::Visible, + } +} + +#[test] +fn v2_protocol_rejects_schema_drift_and_version_mismatch() { + let valid = serde_json::json!({ + "request_id": "request-1", + "protocol_version": {"major": 2, "minor": 0}, + "method": "driver.v2.click", + "params": { + "window": window_ref(), + "request": { + "target": { + "kind": "point", + "observation_id": "observation-1", + "surface_id": "surface-1", + "point": {"x": 10.0, "y": 20.0} + }, + "button": "left", + "click_count": 1, + "modifiers": [] + } + } + }); + let envelope: V2RequestEnvelope = + serde_json::from_value(valid.clone()).expect("valid strict command"); + assert!(matches!(envelope.command, V2Command::Click(_))); + envelope.validate_version().expect("supported version"); + + let mut unknown = valid.clone(); + unknown["params"]["request"]["execution_mode"] = serde_json::json!("foreground"); + assert!(serde_json::from_value::(unknown).is_err()); + + let mut ambiguous_union = valid.clone(); + ambiguous_union["params"]["request"]["target"]["element"] = serde_json::json!({ + "observation_id": "observation-1", + "id": "element-1" + }); + assert!(serde_json::from_value::(ambiguous_union).is_err()); + + let mut wrong_version = valid; + wrong_version["protocol_version"]["minor"] = serde_json::json!(1); + let mismatch: V2RequestEnvelope = serde_json::from_value(wrong_version).unwrap(); + let error = mismatch.validate_version().unwrap_err(); + assert_eq!(error.code, ErrorCode::ProtocolMismatch); + assert_eq!(error.phase, ErrorPhase::Validate); + assert_eq!( + serde_json::to_value(&error).unwrap()["code"], + serde_json::json!("protocol_mismatch") + ); + + let both_result_and_error = serde_json::json!({ + "request_id": "request-1", + "protocol_version": {"major": 2, "minor": 0}, + "result": {}, + "error": serde_json::to_value(&error).unwrap(), + }); + assert!( + serde_json::from_value::>(both_result_and_error) + .is_err() + ); + + let launch = LaunchResult { + action_id: id("launch-action", ActionId::parse), + app: app_ref(), + windows: vec![window_ref()], + reused_running_app: false, + verification: EffectVerification::EffectVerified, + posture: PostureResult::default(), + settlement: SettlementEvidence::initial(), + }; + let mut invalid_launch = serde_json::to_value(launch).unwrap(); + invalid_launch["verification"] = serde_json::json!("dispatch_verified"); + assert!(serde_json::from_value::(invalid_launch).is_err()); +} + +#[test] +fn posture_unverifiable_has_a_distinct_wire_code_and_safety_precedence() { + let unverifiable = NativeError::new( + ErrorCode::PostureUnverifiable, + ErrorPhase::Verify, + false, + "required posture witness was incomplete without an observed disturbance", + ); + let response = V2ResponseEnvelope:: { + request_id: "posture-request".to_owned(), + protocol_version: V2_PROTOCOL_VERSION, + body: V2ResponseBody::Error(V2Failure { + error: unverifiable.clone(), + }), + }; + let encoded = serde_json::to_value(&response).unwrap(); + assert_eq!( + encoded["error"]["code"], + serde_json::json!("posture_unverifiable") + ); + let decoded: NativeError = serde_json::from_value(encoded["error"].clone()).unwrap(); + assert_eq!(decoded, unverifiable); + + let dispatch = NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "dispatch failed", + ); + let primary = NativeError::primary(vec![unverifiable.clone(), dispatch]).unwrap(); + assert_eq!(primary.code, ErrorCode::PostureUnverifiable); + let violation = NativeError::new( + ErrorCode::PostureViolated, + ErrorPhase::Verify, + false, + "posture changed", + ); + let primary = NativeError::primary(vec![violation, unverifiable]).unwrap(); + assert_eq!(primary.code, ErrorCode::PostureViolated); + + let incomplete = PostureResult { + held: false, + ..PostureResult::default() + }; + assert_eq!( + NativeError::from_posture(&incomplete).unwrap().code, + ErrorCode::PostureUnverifiable + ); + let observed = PostureResult { + held: false, + physical_cursor_moved: true, + ..PostureResult::default() + }; + assert_eq!( + NativeError::from_posture(&observed).unwrap().code, + ErrorCode::PostureViolated + ); + assert!(NativeError::from_posture(&PostureResult::default()).is_none()); +} + +#[test] +fn ax_diff_round_trip_is_exact_and_malformed_ranges_fail_closed() { + let base = "root id=1\n button id=2 label=Old\n text id=3"; + let current = "root id=1\n button id=2 label=New\n text id=3\n text id=4"; + let operations = diff_lines(base, current); + assert_eq!(apply_ax_diff(base, &operations).unwrap(), current); + + let malformed = vec![ + ReplaceAxLines { + start_line: 0, + delete_count: 2, + lines: vec![], + }, + ReplaceAxLines { + start_line: 1, + delete_count: 1, + lines: vec!["overlap".to_owned()], + }, + ]; + let error = apply_ax_diff(base, &malformed).unwrap_err(); + assert_eq!(error.code, ErrorCode::AxRevisionMismatch); + + let mut revisions = AxRevisionState::default(); + let prepared = revisions + .prepare(current, AxTreeMode::DiffIfAvailable) + .unwrap(); + assert!( + revisions.last_revision().is_none(), + "preparing an observation must not advance the delivered revision" + ); + revisions.commit(prepared); + assert!(revisions.last_revision().is_some()); + let no_op = revisions + .prepare(current, AxTreeMode::DiffIfAvailable) + .unwrap(); + let AxTreeUpdate::Diff { + base_revision, + revision, + operations, + } = &no_op.update + else { + panic!("a delivered base must produce a diff by default"); + }; + assert!(operations.is_empty()); + assert_ne!(base_revision, revision, "even a no-op gets a new revision"); + revisions.commit(no_op); + revisions.invalidate_base(); + assert!(matches!( + revisions + .prepare(current, AxTreeMode::DiffIfAvailable) + .unwrap() + .update, + AxTreeUpdate::Full { .. } + )); + + let malformed_native = NativeAccessibilityUpdate { + normalized_tree: current.to_owned(), + elements: vec![NativeAccessibilityElement { + id: id("element-1", ElementId::parse), + native: NativeElementHandle::new("native-element-1").unwrap(), + owner: resolved_window().stamp(), + role: Some("button".to_owned()), + label: None, + value: None, + bounds: Some(Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }), + actions: Vec::new(), + menu_id: None, + }], + focused_element: Some(id("missing-element", ElementId::parse)), + selected_text: None, + selected_elements: Vec::new(), + document_text: None, + }; + let error = revision_accessibility( + &AxRevisionState::default(), + &id("observation-malformed", ObservationId::parse), + malformed_native, + AxTreeMode::DiffIfAvailable, + ) + .err() + .expect("dangling AX references must fail"); + assert_eq!(error.code, ErrorCode::AxRevisionMismatch); +} + +#[test] +fn request_validation_rejects_unsafe_noops_and_non_finite_geometry() { + let observation_id = id("observation-1", ObservationId::parse); + let surface_id = id("surface-1", SurfaceId::parse); + assert!(ClickRequest { + target: ClickTarget::Point { + observation_id: observation_id.clone(), + surface_id: surface_id.clone(), + point: Point { + x: f64::NAN, + y: 1.0, + }, + }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + } + .validate() + .is_err()); + assert!(ScrollRequest::Delta { + observation_id, + surface_id, + point: Point { x: 1.0, y: 1.0 }, + delta_x: f64::INFINITY, + delta_y: 0.0, + } + .validate() + .is_err()); + assert!(PressKeyRequest { + observation_id: id("observation-2", ObservationId::parse), + stroke: KeyStroke { + key: " ".to_owned(), + modifiers: Vec::new(), + }, + } + .validate() + .is_err()); + assert!(SelectTextRequest { + element: ElementRef { + observation_id: id("observation-3", ObservationId::parse), + id: id("element-1", ElementId::parse), + }, + text: String::new(), + prefix: Some(String::new()), + suffix: None, + selection_type: SelectionType::Text, + } + .validate() + .is_err()); + assert!(SecondaryActionRequest { + element: ElementRef { + observation_id: id("observation-4", ObservationId::parse), + id: id("element-2", ElementId::parse), + }, + action: String::new(), + } + .validate() + .is_err()); + assert!(Rect { + x: 0.0, + y: 0.0, + width: 0.0, + height: 10.0, + } + .validate() + .is_err()); +} + +#[derive(Default)] +struct FakeTargetState { + semantic_dispatches: usize, + verification_readback_complete: bool, +} + +struct FakeFocus { + shutdowns: Arc, +} + +#[async_trait] +impl TargetFocusCoordinator for FakeFocus { + async fn shutdown(&mut self) -> Result<(), NativeError> { + self.shutdowns.fetch_add(1, Ordering::SeqCst); + Ok(()) + } +} + +#[derive(Default)] +struct EmptyInvalidations; + +#[async_trait] +impl InvalidationSubscription for EmptyInvalidations { + async fn next(&mut self) -> Option { + None + } +} + +#[derive(Default)] +struct FakePlatform { + observation_count: AtomicUsize, + dispatch_count: AtomicUsize, + target_creations: AtomicUsize, + shutdowns: Arc, + ordering: Arc>>, + cleanup_count: Arc, + preflight_action_ids: Arc>>, + acquired_action_ids: Arc>>, + block_dispatch: std::sync::atomic::AtomicBool, + dispatch_fail: std::sync::atomic::AtomicBool, + semantic_prepare_fail: std::sync::atomic::AtomicBool, + semantic_verification_fail: std::sync::atomic::AtomicBool, + observation_role: Mutex>, + cleanup_failure_count: AtomicUsize, + cleanup_posture_violated: std::sync::atomic::AtomicBool, + settle_pending: std::sync::atomic::AtomicBool, + launch_fail: std::sync::atomic::AtomicBool, + refresh_geometry_during_observe: std::sync::atomic::AtomicBool, + dispatch_entered: Notify, +} + +enum FakePreparedSemantic { + Click, + SetValue, + Secondary, +} + +struct FakeScopeCleanup { + ordering: Arc>>, + cleanup_count: Arc, + failure_count: usize, + posture_violated: bool, + leases: ScopeLeaseTeardown, +} + +impl ScopeCleanup for FakeScopeCleanup { + fn cleanup(&mut self, _deadline: Instant) -> ScopeTeardownOutcome { + self.cleanup_count.fetch_add(1, Ordering::SeqCst); + self.ordering.lock().unwrap().push("scope_released"); + let mut posture = PostureResult::default(); + if self.posture_violated { + posture.held = false; + posture.frontmost_changed = true; + posture.restored_after_violation = true; + } + let failures = (0..self.failure_count) + .map(|index| { + NativeError::new( + if index == 0 { + ErrorCode::VerificationFailed + } else { + ErrorCode::Internal + }, + ErrorPhase::Verify, + false, + format!("fake cleanup failure {index}"), + ) + }) + .collect(); + ScopeTeardownOutcome { + posture, + native_evidence: NativeEvidence { + fields: BTreeMap::from([( + "fake_cleanup".to_owned(), + serde_json::json!("complete"), + )]), + ..NativeEvidence::default() + }, + leases: self.leases.clone(), + failures, + } + } +} + +impl FakePlatform { + fn record(&self, event: &'static str) { + self.ordering.lock().unwrap().push(event); + } +} + +#[async_trait] +impl LifecycleProvider for FakePlatform { + async fn readiness(&self) -> Result { + Ok(Readiness { + ready: true, + permissions: BTreeMap::new(), + diagnostics: BTreeMap::new(), + }) + } + + async fn capabilities(&self) -> Result { + Ok(CapabilityManifest { + platform: PlatformName::Macos, + driver_version: "test".to_owned(), + protocol_version: "2.0".to_owned(), + permissions: BTreeMap::new(), + cells: Vec::new(), + }) + } + + async fn list_apps(&self, _query: AppQuery) -> Result, NativeError> { + Ok(vec![app_ref()]) + } + + async fn launch_background( + &self, + _app: AppSelector, + posture: &mut LaunchPostureScope, + ) -> Result { + posture.begin_launch(); + posture.record_partial_result(app_ref(), vec![window_ref()]); + posture.posture.held = true; + posture.pending_settlement = Some(PendingSettlementEvidence { + state: PendingSettlementState::Pending, + trigger_action_id: posture + .action_id() + .cloned() + .expect("controller launch scopes carry their action id"), + profile: "fake_launch".to_owned(), + elapsed_ms: 1, + observed_signals: vec![SettlementSignal::DispatchComplete], + missing_signals: Vec::new(), + }); + if self.launch_fail.load(Ordering::SeqCst) { + return Err(NativeError::new( + ErrorCode::AppLaunchFailed, + ErrorPhase::Dispatch, + true, + "fake post-dispatch launch failure", + )); + } + Ok(NativeLaunch { + app: app_ref(), + windows: vec![window_ref()], + reused_running_app: false, + posture: posture.posture.clone(), + settlement: SettlementEvidence::initial(), + }) + } +} + +#[tokio::test] +async fn post_dispatch_launch_failure_keeps_exact_partial_and_pending_evidence() { + let platform = Arc::new(FakePlatform::default()); + platform.launch_fail.store(true, Ordering::SeqCst); + let controller = DriverController::new(platform, PlatformName::Macos, "test-os"); + let error = controller + .launch_app(LaunchAppRequest { + app: AppSelector::BundleId { + bundle_id: "com.example.fixture".to_owned(), + }, + }) + .await + .unwrap_err(); + + let Some(PartialEvidence::Launch { + action_id, + app, + windows, + posture, + pending_settlement: Some(pending), + .. + }) = error.partial_evidence.as_deref() + else { + panic!("post-dispatch launch failure must keep structured launch evidence"); + }; + assert_eq!(app.as_ref(), Some(&app_ref())); + assert_eq!(windows, &vec![window_ref()]); + assert!(posture.held); + assert_eq!(pending.trigger_action_id, *action_id); +} + +#[async_trait] +impl WindowProvider for FakePlatform { + async fn list_windows(&self, _app: Option<&AppRef>) -> Result, NativeError> { + Ok(vec![window_ref()]) + } + + async fn rehydrate( + &self, + _id: &WindowId, + _app: Option<&AppRef>, + ) -> Result { + Ok(window_ref()) + } + + async fn resolve(&self, window: &WindowRef) -> Result { + if window.id != window_ref().id || window.app.id != app_ref().id { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "fake window mismatch", + )); + } + Ok(resolved_window()) + } +} + +#[async_trait] +impl ObservationProvider for FakePlatform { + async fn settle( + &self, + _target: &mut FakeTargetState, + dirty: &DirtyState, + _deadline: Instant, + ) -> Result { + self.record("settle"); + if self.settle_pending.load(Ordering::SeqCst) { + let mut pending = dirty.pending_evidence(); + pending.observed_signals.push(SettlementSignal::AxAction); + return Ok(SettlementAttempt::Pending(pending)); + } + let mut signals = dirty.observed_signals.clone(); + signals.insert(SettlementSignal::DispatchComplete); + signals.extend( + dirty + .profile + .required_terminal_signals + .iter() + .copied() + .filter(|signal| { + *signal != SettlementSignal::VerificationReadbackComplete + || _target.verification_readback_complete + }), + ); + if !dirty.profile.required_terminal_signals.is_subset(&signals) { + return Ok(SettlementAttempt::Pending(PendingSettlementEvidence { + state: PendingSettlementState::Pending, + trigger_action_id: dirty.action_id.clone(), + profile: dirty.profile.name.clone(), + elapsed_ms: 1, + observed_signals: signals.iter().copied().collect(), + missing_signals: dirty + .profile + .required_terminal_signals + .difference(&signals) + .copied() + .collect(), + })); + } + Ok(SettlementAttempt::Settled(SettlementEvidence { + state: SettledState::Settled, + trigger_action_id: Some(dirty.action_id.clone()), + profile: dirty.profile.name.clone(), + elapsed_ms: 1, + observed_signals: signals.into_iter().collect(), + terminal_signal: "fake_terminal".to_owned(), + quiet_window_ms: dirty.profile.quiet_window_ms, + resumed_from_prior_call: dirty.resumed_from_prior_call, + })) + } + + async fn observe( + &self, + _target: &mut FakeTargetState, + window: &ResolvedWindow, + _request: ObserveRequest, + ) -> Result { + let sequence = self.observation_count.fetch_add(1, Ordering::SeqCst); + self.record("observe"); + let observation_suffix = sequence + 1; + let element_id = id("element-1", ElementId::parse); + let mut observed_window = window.clone(); + if self.refresh_geometry_during_observe.load(Ordering::SeqCst) { + observed_window.geometry.bounds.x += 10.0; + observed_window.geometry.revision = id("geometry-refreshed", GeometryRevision::parse); + } + Ok(NativeObservationUpdate { + window: observed_window.clone(), + surfaces: vec![SurfaceRecord { + id: id(&format!("surface-{observation_suffix}"), SurfaceId::parse), + kind: SurfaceKind::Window, + owner_window: observed_window.public.clone(), + image_url: format!("file:///tmp/surface-{observation_suffix}.png"), + approximate_bytes: 16, + raster_size: Size { + width: 640, + height: 480, + }, + window_bounds: Some(observed_window.geometry.bounds), + capture_revision: id( + &format!("capture-{observation_suffix}"), + CaptureRevision::parse, + ), + transform: SurfaceToWindowTransform { + scale_x: 1.0, + scale_y: 1.0, + offset_x: 0.0, + offset_y: 0.0, + }, + freshness: CaptureFreshness::Fresh, + owner: SurfaceOwner::Target(observed_window.stamp()), + }], + accessibility: Some(NativeAccessibilityUpdate { + normalized_tree: format!( + "window id=root\n button id=element-1 value={observation_suffix}" + ), + elements: vec![NativeAccessibilityElement { + id: element_id.clone(), + native: NativeElementHandle::new("native-element-1").unwrap(), + owner: observed_window.stamp(), + role: Some( + self.observation_role + .lock() + .unwrap() + .clone() + .unwrap_or_else(|| "button".to_owned()), + ), + label: Some("Fixture button".to_owned()), + value: Some(observation_suffix.to_string()), + bounds: Some(Rect { + x: 10.0, + y: 10.0, + width: 100.0, + height: 30.0, + }), + actions: vec!["AXPress".to_owned()], + menu_id: None, + }], + focused_element: Some(element_id), + selected_text: None, + selected_elements: Vec::new(), + document_text: None, + }), + menu: NativeMenuObservation::Unchanged, + captured_at_unix_ms: observation_suffix as u64, + warnings: Vec::new(), + artifacts: Vec::new(), + }) + } +} + +#[async_trait] +impl SemanticActionProvider for FakePlatform { + type PreparedAction = FakePreparedSemantic; + + async fn prepare( + &self, + _target: &mut FakeTargetState, + _scope: &mut InteractionScope, + action: &ResolvedAction, + ) -> Result { + self.record("semantic_prepare"); + if self.semantic_prepare_fail.load(Ordering::SeqCst) { + return Err(NativeError::unsupported( + "fake semantic shape refused during prepare", + )); + } + match action { + ResolvedAction::ElementClick { .. } => Ok(FakePreparedSemantic::Click), + ResolvedAction::SetValue { .. } => Ok(FakePreparedSemantic::SetValue), + ResolvedAction::Secondary { element, action } + if action == "AXPress" + && element.role.as_deref().is_some_and(|role| { + matches!( + role, + "AXMenu" + | "AXMenuItem" + | "AXMenuBar" + | "AXMenuBarItem" + | "AXPopUpButton" + | "AXMenuButton" + ) + }) => + { + Err(NativeError::unsupported( + "recipe_unproven: fake menu-managed secondary action", + )) + } + ResolvedAction::Secondary { .. } => Ok(FakePreparedSemantic::Secondary), + _ => Err(NativeError::unsupported("not used by this fixture")), + } + } + + async fn dispatch( + &self, + target: &mut FakeTargetState, + _scope: &mut InteractionScope, + action: Self::PreparedAction, + ) -> Result { + target.semantic_dispatches += 1; + self.dispatch_count.fetch_add(1, Ordering::SeqCst); + self.record("dispatch"); + let effect_readback = matches!(action, FakePreparedSemantic::SetValue); + if effect_readback { + target.verification_readback_complete = true; + } + if self.semantic_verification_fail.load(Ordering::SeqCst) { + return Err(NativeError::new( + ErrorCode::VerificationFailed, + ErrorPhase::Verify, + true, + "fake exact semantic readback mismatch", + )); + } + let mut dispatch = NativeDispatch::dispatch_verified(); + if effect_readback { + dispatch.verification = VerificationLevel::EffectVerified; + } + Ok(dispatch) + } +} + +#[async_trait] +impl PointerActionProvider for FakePlatform { + async fn click( + &self, + _scope: &mut InteractionScope, + _point: ResolvedPoint, + _click: ClickSpec, + ) -> Result { + self.dispatch_count.fetch_add(1, Ordering::SeqCst); + self.record("dispatch"); + if self.block_dispatch.load(Ordering::SeqCst) { + self.dispatch_entered.notify_waiters(); + std::future::pending::<()>().await; + } + if self.dispatch_fail.load(Ordering::SeqCst) { + return Err(NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "fake dispatch failure", + )); + } + Ok(NativeDispatch::dispatch_verified()) + } + + async fn drag( + &self, + _scope: &mut InteractionScope, + _drag: ResolvedDrag, + ) -> Result { + Err(NativeError::unsupported("not used by this fixture")) + } + + async fn scroll( + &self, + _scope: &mut InteractionScope, + _scroll: ResolvedScroll, + ) -> Result { + Err(NativeError::unsupported("not used by this fixture")) + } +} + +#[async_trait] +impl KeyboardActionProvider for FakePlatform { + async fn press_key( + &self, + _scope: &mut InteractionScope, + _focus: &ResolvedFocus, + _stroke: KeyStroke, + ) -> Result { + Err(NativeError::unsupported("not used by this fixture")) + } + + async fn type_text( + &self, + _scope: &mut InteractionScope, + _focus: &ResolvedFocus, + _text: &str, + ) -> Result { + Err(NativeError::unsupported("not used by this fixture")) + } +} + +#[async_trait] +impl InteractionProvider for FakePlatform { + type NativeScopePlan = (); + + async fn preflight( + &self, + _target: &mut FakeTargetState, + _focus: &mut FakeFocus, + action_id: &ActionId, + window: &ResolvedWindow, + route: Route, + _action: &ResolvedAction, + deadline: MutationDeadline, + requirements: ScopeRequirements, + ) -> Result, NativeError> { + self.record("preflight"); + self.preflight_action_ids + .lock() + .unwrap() + .push(action_id.clone()); + Ok(ScopePlan::new( + action_id.clone(), + window.clone(), + route, + deadline, + requirements, + (), + )) + } + + async fn acquire_scope( + &self, + _target: &mut FakeTargetState, + _focus: &mut FakeFocus, + plan: ScopePlan, + logical_cursor: TargetCursorHandle, + ) -> Result { + self.record("scope_acquired"); + self.acquired_action_ids + .lock() + .unwrap() + .push(plan.action_id.clone()); + let decision = |required| { + if required { + LeaseDecision::Acquired + } else { + LeaseDecision::NotApplicable + } + }; + let acquisition = ScopeLeaseAcquisition { + posture_witness: LeaseDecision::Acquired, + accessibility: decision(plan.requirements.accessibility), + containment: decision(plan.requirements.containment), + menu_dismissal: decision(plan.requirements.menu_dismissal), + target_belief: decision(plan.requirements.target_belief), + }; + let teardown = |decision| match decision { + LeaseDecision::Acquired => LeaseTeardownStatus::Released, + LeaseDecision::NotApplicable => LeaseTeardownStatus::NotApplicable, + }; + let mut teardown_leases = ScopeLeaseTeardown { + posture_witness: teardown(acquisition.posture_witness), + accessibility: teardown(acquisition.accessibility), + containment: teardown(acquisition.containment), + menu_dismissal: teardown(acquisition.menu_dismissal), + target_belief: teardown(acquisition.target_belief), + }; + let failure_count = self.cleanup_failure_count.load(Ordering::SeqCst); + if failure_count > 0 { + teardown_leases.target_belief = LeaseTeardownStatus::Failed; + } + Ok(plan.into_scope( + acquisition, + logical_cursor, + NativeEvidence::default(), + Box::new(FakeScopeCleanup { + ordering: Arc::clone(&self.ordering), + cleanup_count: Arc::clone(&self.cleanup_count), + failure_count, + posture_violated: self.cleanup_posture_violated.load(Ordering::SeqCst), + leases: teardown_leases, + }), + )) + } +} + +#[async_trait] +impl PlatformDriver for FakePlatform { + type TargetState = FakeTargetState; + type TargetFocusCoordinator = FakeFocus; + type Lifecycle = Self; + type Windows = Self; + type Observation = Self; + type Semantic = Self; + type Pointer = Self; + type Keyboard = Self; + type Interaction = Self; + type Invalidations = EmptyInvalidations; + + async fn create_target_state( + &self, + _window: &ResolvedWindow, + ) -> Result<(Self::TargetState, Self::TargetFocusCoordinator), NativeError> { + self.target_creations.fetch_add(1, Ordering::SeqCst); + Ok(( + FakeTargetState::default(), + FakeFocus { + shutdowns: Arc::clone(&self.shutdowns), + }, + )) + } + + fn lifecycle(&self) -> &Self::Lifecycle { + self + } + + fn windows(&self) -> &Self::Windows { + self + } + + fn observation(&self) -> &Self::Observation { + self + } + + fn semantic(&self) -> &Self::Semantic { + self + } + + fn pointer(&self) -> &Self::Pointer { + self + } + + fn keyboard(&self) -> &Self::Keyboard { + self + } + + fn interaction(&self) -> &Self::Interaction { + self + } + + fn capability_cells(&self, os_version: &str) -> Vec { + vec![ + CapabilityCell { + key: CapabilityKey { + platform: PlatformName::Macos, + os_version: os_version.to_owned(), + action: ActionKind::Click, + addressing: AddressingMode::CapturedPoint, + framework: Framework::AppKit, + window_state: WindowStateKind::Visible, + }, + decision: RouteDecision::Supported { + route: Route::TargetedPointer, + }, + }, + CapabilityCell { + key: CapabilityKey { + platform: PlatformName::Macos, + os_version: os_version.to_owned(), + action: ActionKind::Click, + addressing: AddressingMode::Element, + framework: Framework::AppKit, + window_state: WindowStateKind::Visible, + }, + decision: RouteDecision::Supported { + route: Route::Semantic, + }, + }, + CapabilityCell { + key: CapabilityKey { + platform: PlatformName::Macos, + os_version: os_version.to_owned(), + action: ActionKind::PerformSecondaryAction, + addressing: AddressingMode::Element, + framework: Framework::AppKit, + window_state: WindowStateKind::Visible, + }, + decision: RouteDecision::Supported { + route: Route::Semantic, + }, + }, + CapabilityCell { + key: CapabilityKey { + platform: PlatformName::Macos, + os_version: os_version.to_owned(), + action: ActionKind::SetValue, + addressing: AddressingMode::Element, + framework: Framework::AppKit, + window_state: WindowStateKind::Visible, + }, + decision: RouteDecision::Supported { + route: Route::Semantic, + }, + }, + ] + } + + fn subscribe_invalidations(&self) -> Self::Invalidations { + EmptyInvalidations + } +} + +#[tokio::test] +async fn controller_preserves_target_state_and_consumes_at_dispatch_boundary() { + let platform = Arc::new(FakePlatform::default()); + let controller = DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os"); + let manifest = controller.get_capabilities().await.unwrap(); + assert_eq!(manifest.cells.len(), 4); + assert!(manifest.cells.iter().any(|cell| matches!( + cell.decision, + RouteDecision::Supported { + route: Route::TargetedPointer + } + ))); + + let client_one = id("client-one", ClientId::parse); + let first = controller + .get_window_state( + &client_one, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::DiffIfAvailable, + }, + ) + .await + .unwrap(); + assert!(matches!( + first.accessibility.as_ref().unwrap().tree_update, + AxTreeUpdate::Full { .. } + )); + + let second = controller + .get_window_state( + &client_one, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::DiffIfAvailable, + }, + ) + .await + .unwrap(); + let AxTreeUpdate::Diff { + base_revision, + revision, + .. + } = &second.accessibility.as_ref().unwrap().tree_update + else { + panic!("second observation must be a diff"); + }; + assert_ne!(base_revision, revision); + assert_eq!(platform.target_creations.load(Ordering::SeqCst), 1); + + let surface_id = second.surfaces[0].id.clone(); + let command = ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Point { + observation_id: second.observation_id.clone(), + surface_id, + point: Point { x: 20.0, y: 20.0 }, + }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }; + let receipt = controller + .click(&client_one, command.clone()) + .await + .unwrap(); + assert_eq!(receipt.consumed_observation_id, second.observation_id); + assert_eq!(receipt.route, Route::TargetedPointer); + assert_eq!(receipt.settlement.state, SettledState::Settled); + assert_eq!( + platform.preflight_action_ids.lock().unwrap().as_slice(), + [receipt.action_id.clone()] + ); + assert_eq!( + platform.acquired_action_ids.lock().unwrap().as_slice(), + [receipt.action_id.clone()] + ); + let scope_evidence = receipt + .native_evidence + .interaction_scope + .as_ref() + .expect("receipt carries typed scope evidence"); + assert_eq!( + scope_evidence.acquisition.posture_witness, + LeaseDecision::Acquired + ); + assert_eq!( + scope_evidence + .teardown + .as_ref() + .expect("completed scope carries teardown evidence") + .target_belief, + LeaseTeardownStatus::Released + ); + assert_eq!( + receipt.native_evidence.fields.get("fake_cleanup"), + Some(&serde_json::json!("complete")) + ); + + let mut cross_window = command.clone(); + cross_window.window.id = id("window-2", WindowId::parse); + let cross_error = controller + .click(&client_one, cross_window) + .await + .unwrap_err(); + assert_eq!(cross_error.code, ErrorCode::WindowIdentityChanged); + assert_eq!(platform.dispatch_count.load(Ordering::SeqCst), 1); + + let stale = controller.click(&client_one, command).await.unwrap_err(); + assert_eq!(stale.code, ErrorCode::ObservationStale); + assert_eq!(platform.dispatch_count.load(Ordering::SeqCst), 1); + assert_eq!( + platform.ordering.lock().unwrap().as_slice(), + [ + "observe", + "observe", + "preflight", + "scope_acquired", + "dispatch", + "settle", + "scope_released" + ] + ); + + let resynchronized = controller + .get_window_state( + &client_one, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::Full, + }, + ) + .await + .unwrap(); + assert!(matches!( + resynchronized.accessibility.as_ref().unwrap().tree_update, + AxTreeUpdate::Full { .. } + )); + platform.settle_pending.store(true, Ordering::SeqCst); + let pending_command = ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Point { + observation_id: resynchronized.observation_id.clone(), + surface_id: resynchronized.surfaces[0].id.clone(), + point: Point { x: 20.0, y: 20.0 }, + }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }; + let pending_error = controller + .click(&client_one, pending_command) + .await + .unwrap_err(); + assert_eq!(pending_error.code, ErrorCode::UiNotSettled); + assert!(pending_error.pending_settlement.is_some()); + assert!(matches!( + pending_error.partial_evidence.as_deref(), + Some(PartialEvidence::Action { + dispatch: Some(PartialNativeDispatch { .. }), + pending_settlement: Some(_), + .. + }) + )); + platform.settle_pending.store(false, Ordering::SeqCst); + controller + .get_window_state( + &client_one, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::DiffIfAvailable, + }, + ) + .await + .unwrap(); + + let client_two = id("client-two", ClientId::parse); + controller + .get_window_state( + &client_two, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::DiffIfAvailable, + }, + ) + .await + .unwrap(); + let first_key = TargetKey::from_window(client_one.clone(), &resolved_window()); + let second_key = TargetKey::from_window(client_two.clone(), &resolved_window()); + let first_target = controller.targets.get(&first_key).await.unwrap(); + let second_target = controller.targets.get(&second_key).await.unwrap(); + assert!(Arc::ptr_eq( + &first_target.mutation_lock, + &second_target.mutation_lock + )); + + assert_eq!(controller.close_connection(&client_one).await.unwrap(), 1); + assert_eq!(controller.targets.len().await, 1); + assert_eq!(platform.shutdowns.load(Ordering::SeqCst), 1); + assert_eq!( + controller + .targets + .handle_invalidation( + &platform, + TargetInvalidation::ProcessExited { + process: resolved_window().process, + }, + ) + .await + .unwrap(), + 1 + ); + assert_eq!(controller.targets.len().await, 0); + assert_eq!(platform.shutdowns.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn semantic_element_dispatch_uses_the_locked_target_and_interaction_scope() { + let platform = Arc::new(FakePlatform::default()); + let controller = DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os"); + let client = id("semantic-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: false, + ax_tree_mode: AxTreeMode::Full, + }, + ) + .await + .unwrap(); + let element = observed + .accessibility + .as_ref() + .unwrap() + .elements + .first() + .unwrap() + .element_ref + .clone(); + let receipt = controller + .click( + &client, + ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Element { element }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }, + ) + .await + .unwrap(); + + assert_eq!(receipt.route, Route::Semantic); + assert_eq!(platform.dispatch_count.load(Ordering::SeqCst), 1); + let key = TargetKey::from_window(client.clone(), &resolved_window()); + let target = controller.targets.get(&key).await.unwrap(); + assert_eq!(target.state.lock().await.platform.semantic_dispatches, 1); + assert_eq!( + platform.ordering.lock().unwrap().as_slice(), + [ + "observe", + "preflight", + "scope_acquired", + "semantic_prepare", + "dispatch", + "settle", + "scope_released" + ] + ); +} + +#[tokio::test] +async fn semantic_prepare_refusal_preserves_the_observation_and_never_enters_dispatch() { + let platform = Arc::new(FakePlatform::default()); + platform.semantic_prepare_fail.store(true, Ordering::SeqCst); + let controller = DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os"); + let client = id("semantic-prepare-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: false, + ax_tree_mode: AxTreeMode::Full, + }, + ) + .await + .unwrap(); + let element = observed + .accessibility + .as_ref() + .unwrap() + .elements + .first() + .unwrap() + .element_ref + .clone(); + let command = ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Element { element }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }; + + let refused = controller + .click(&client, command.clone()) + .await + .unwrap_err(); + assert_eq!(refused.code, ErrorCode::UnsupportedInBackground); + assert_eq!(platform.dispatch_count.load(Ordering::SeqCst), 0); + let key = TargetKey::from_window(client.clone(), &resolved_window()); + let target = controller.targets.get(&key).await.unwrap(); + assert_eq!(target.state.lock().await.platform.semantic_dispatches, 0); + + platform + .semantic_prepare_fail + .store(false, Ordering::SeqCst); + controller.click(&client, command).await.unwrap(); + assert_eq!(platform.dispatch_count.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn menu_managed_secondary_axpress_refuses_in_prepare_without_consuming_observation() { + for role in [ + "AXMenu", + "AXMenuItem", + "AXMenuBar", + "AXMenuBarItem", + "AXPopUpButton", + "AXMenuButton", + ] { + let platform = Arc::new(FakePlatform::default()); + *platform.observation_role.lock().unwrap() = Some(role.to_owned()); + let controller = + DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os"); + let client = ClientId::parse(format!("menu-secondary-{role}")).unwrap(); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: false, + ax_tree_mode: AxTreeMode::Full, + }, + ) + .await + .unwrap(); + let element = observed + .accessibility + .as_ref() + .unwrap() + .elements + .first() + .unwrap() + .element_ref + .clone(); + + let error = controller + .perform_secondary_action( + &client, + SecondaryActionCommand { + window: window_ref(), + request: SecondaryActionRequest { + element, + action: "AXPress".to_owned(), + }, + }, + ) + .await + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::UnsupportedInBackground, "{role}"); + assert_eq!(platform.dispatch_count.load(Ordering::SeqCst), 0, "{role}"); + let key = TargetKey::from_window(client, &resolved_window()); + let target = controller.targets.get(&key).await.unwrap(); + let mut state = target.state.lock().await; + let current_window = state.window.clone(); + state + .observations + .current(&observed.observation_id, ¤t_window) + .unwrap_or_else(|error| panic!("{role} observation was consumed: {error}")); + } +} + +#[tokio::test] +async fn semantic_readback_mismatch_settles_and_remains_the_primary_failure() { + let platform = Arc::new(FakePlatform::default()); + platform + .semantic_verification_fail + .store(true, Ordering::SeqCst); + let controller = DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os"); + let client = id("semantic-readback-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: false, + ax_tree_mode: AxTreeMode::Full, + }, + ) + .await + .unwrap(); + let element = observed + .accessibility + .as_ref() + .unwrap() + .elements + .first() + .unwrap() + .element_ref + .clone(); + + let error = controller + .set_value( + &client, + SetValueCommand { + window: window_ref(), + request: SetValueRequest { + element, + value: "expected exact value".to_owned(), + }, + }, + ) + .await + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::VerificationFailed); + assert!(!error + .related_failures + .iter() + .any(|failure| failure.code == ErrorCode::UiNotSettled)); + let Some(PartialEvidence::Action { + pending_settlement, .. + }) = error.partial_evidence.as_deref() + else { + panic!("post-dispatch verification failure must retain partial action evidence"); + }; + assert!(pending_settlement.is_none()); + assert_eq!(platform.dispatch_count.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn coherent_observation_accepts_refreshed_geometry_without_relaxing_identity() { + let platform = Arc::new(FakePlatform::default()); + platform + .refresh_geometry_during_observe + .store(true, Ordering::SeqCst); + let controller = DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os"); + let client = id("geometry-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::DiffIfAvailable, + }, + ) + .await + .unwrap(); + + let key = TargetKey::from_window(client, &resolved_window()); + let target = controller.targets.get(&key).await.unwrap(); + let mut state = target.state.lock().await; + assert_eq!( + state.window.geometry.revision, + id("geometry-refreshed", GeometryRevision::parse) + ); + assert_eq!(state.window.geometry.bounds.x, 110.0); + let current_window = state.window.clone(); + state + .observations + .current(&observed.observation_id, ¤t_window) + .unwrap(); +} + +#[tokio::test] +async fn target_registry_tears_down_idle_and_superseded_generations_exactly() { + let platform = FakePlatform::default(); + let registry = TargetControllerRegistry::new( + Arc::new(ProcessMutationLockRegistry::default()), + Duration::ZERO, + ); + let client = id("registry-client", ClientId::parse); + let generation_one = resolved_window(); + registry + .get_or_create( + &platform, + TargetKey::from_window(client.clone(), &generation_one), + generation_one.clone(), + ) + .await + .unwrap(); + assert_eq!(registry.expire_idle(&platform).await.unwrap(), 1); + assert!(registry.is_empty().await); + + registry + .get_or_create( + &platform, + TargetKey::from_window(client.clone(), &generation_one), + generation_one.clone(), + ) + .await + .unwrap(); + let mut generation_two = generation_one; + generation_two.generation = WindowGeneration(2); + registry + .get_or_create( + &platform, + TargetKey::from_window(client, &generation_two), + generation_two, + ) + .await + .unwrap(); + assert_eq!(registry.len().await, 1); + assert_eq!(platform.shutdowns.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn native_event_source_lag_tears_down_every_cached_target() { + let platform = FakePlatform::default(); + let registry = TargetControllerRegistry::new( + Arc::new(ProcessMutationLockRegistry::default()), + Duration::from_secs(60), + ); + let first = resolved_window(); + registry + .get_or_create( + &platform, + TargetKey::from_window(id("lag-client-one", ClientId::parse), &first), + first.clone(), + ) + .await + .unwrap(); + registry + .get_or_create( + &platform, + TargetKey::from_window(id("lag-client-two", ClientId::parse), &first), + first, + ) + .await + .unwrap(); + + assert_eq!( + registry + .handle_invalidation(&platform, TargetInvalidation::NativeStateResyncRequired) + .await + .unwrap(), + 2 + ); + assert!(registry.is_empty().await); + assert_eq!(platform.shutdowns.load(Ordering::SeqCst), 2); +} + +#[test] +fn interaction_scope_release_is_idempotent_and_accumulates_teardown_evidence() { + let cleanup_count = Arc::new(AtomicUsize::new(0)); + let ordering = Arc::new(Mutex::new(Vec::new())); + let acquisition = ScopeLeaseAcquisition { + posture_witness: LeaseDecision::Acquired, + accessibility: LeaseDecision::Acquired, + containment: LeaseDecision::Acquired, + menu_dismissal: LeaseDecision::NotApplicable, + target_belief: LeaseDecision::Acquired, + }; + let mut scope = ScopePlan::new( + id("idempotent-action", ActionId::parse), + resolved_window(), + Route::TargetedPointer, + test_mutation_deadline(), + ScopeRequirements::for_route(Route::TargetedPointer), + (), + ) + .into_scope( + acquisition, + TargetCursorHandle::default(), + NativeEvidence::default(), + Box::new(FakeScopeCleanup { + ordering, + cleanup_count: Arc::clone(&cleanup_count), + failure_count: 2, + posture_violated: true, + leases: ScopeLeaseTeardown { + posture_witness: LeaseTeardownStatus::Released, + accessibility: LeaseTeardownStatus::Released, + containment: LeaseTeardownStatus::Released, + menu_dismissal: LeaseTeardownStatus::NotApplicable, + target_belief: LeaseTeardownStatus::Failed, + }, + }), + ); + + let first = scope.release(); + let second = scope.release(); + + assert_eq!(first, second); + assert_eq!(first.failures.len(), 2); + assert_eq!(cleanup_count.load(Ordering::SeqCst), 1); + assert!(!scope.posture.held); + assert!(scope.posture.restored_after_violation); + assert_eq!( + scope.native_evidence.fields.get("fake_cleanup"), + Some(&serde_json::json!("complete")) + ); + assert_eq!( + scope + .native_evidence + .interaction_scope + .as_ref() + .and_then(|evidence| evidence.teardown.as_ref()) + .map(|teardown| teardown.target_belief), + Some(LeaseTeardownStatus::Failed) + ); + drop(scope); + assert_eq!(cleanup_count.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn posture_violation_remains_primary_and_keeps_every_cleanup_failure() { + let platform = Arc::new(FakePlatform::default()); + platform.dispatch_fail.store(true, Ordering::SeqCst); + platform.settle_pending.store(true, Ordering::SeqCst); + platform.cleanup_failure_count.store(2, Ordering::SeqCst); + platform + .cleanup_posture_violated + .store(true, Ordering::SeqCst); + let controller = DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os"); + let client = id("failure-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::DiffIfAvailable, + }, + ) + .await + .unwrap(); + let error = controller + .click( + &client, + ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Point { + observation_id: observed.observation_id.clone(), + surface_id: observed.surfaces[0].id.clone(), + point: Point { x: 20.0, y: 20.0 }, + }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }, + ) + .await + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::PostureViolated); + assert_eq!(error.related_failures.len(), 4); + for expected in [ + ErrorCode::DispatchFailed, + ErrorCode::UiNotSettled, + ErrorCode::VerificationFailed, + ErrorCode::Internal, + ] { + assert!(error + .related_failures + .iter() + .any(|failure| failure.code == expected)); + } + let Some(PartialEvidence::Action { + dispatch: None, + posture, + native_evidence, + pending_settlement: Some(_), + .. + }) = error.partial_evidence.as_deref() + else { + panic!("post-dispatch failure must keep scope and settlement evidence"); + }; + assert!(!posture.held); + assert!(posture.restored_after_violation); + assert_eq!( + native_evidence + .interaction_scope + .as_ref() + .and_then(|evidence| evidence.teardown.as_ref()) + .map(|teardown| teardown.target_belief), + Some(LeaseTeardownStatus::Failed) + ); + assert_eq!(platform.cleanup_count.load(Ordering::SeqCst), 1); + let missing = controller + .targets + .get(&TargetKey::from_window(client, &resolved_window())) + .await + .err() + .expect("failed scope teardown removes the target"); + assert_eq!(missing.code, ErrorCode::ObservationStale); + assert_eq!(platform.shutdowns.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn cancellation_inside_provider_leaves_observation_consumed_and_target_dirty() { + let platform = Arc::new(FakePlatform::default()); + let controller = Arc::new(DriverController::new( + Arc::clone(&platform), + PlatformName::Macos, + "test-os", + )); + let client = id("cancel-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::DiffIfAvailable, + }, + ) + .await + .unwrap(); + let command = ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Point { + observation_id: observed.observation_id.clone(), + surface_id: observed.surfaces[0].id.clone(), + point: Point { x: 20.0, y: 20.0 }, + }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }; + + platform.block_dispatch.store(true, Ordering::SeqCst); + let dispatch_entered = platform.dispatch_entered.notified(); + let action = tokio::spawn({ + let controller = Arc::clone(&controller); + let client = client.clone(); + let command = command.clone(); + async move { controller.click(&client, command).await } + }); + dispatch_entered.await; + action.abort(); + assert!(action.await.unwrap_err().is_cancelled()); + assert_eq!(platform.cleanup_count.load(Ordering::SeqCst), 1); + platform.block_dispatch.store(false, Ordering::SeqCst); + + let target = controller + .targets + .get(&TargetKey::from_window(client.clone(), &resolved_window())) + .await + .unwrap(); + let mut state = target.state.lock().await; + assert!(state.settlement.settled_evidence().is_none()); + let stale = state + .observations + .current(&observed.observation_id, &resolved_window()) + .unwrap_err(); + assert_eq!(stale.code, ErrorCode::ObservationStale); + drop(state); + + let dirty = controller.click(&client, command).await.unwrap_err(); + assert_eq!(dirty.code, ErrorCode::UiNotSettled); +} + +#[tokio::test] +async fn cancellation_cleanup_failure_poison_is_rebuilt_by_the_next_observation() { + let platform = Arc::new(FakePlatform::default()); + platform.cleanup_failure_count.store(1, Ordering::SeqCst); + let controller = Arc::new(DriverController::new( + Arc::clone(&platform), + PlatformName::Macos, + "test-os", + )); + let client = id("cancel-cleanup-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::DiffIfAvailable, + }, + ) + .await + .unwrap(); + let command = ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Point { + observation_id: observed.observation_id, + surface_id: observed.surfaces[0].id.clone(), + point: Point { x: 20.0, y: 20.0 }, + }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }; + + platform.block_dispatch.store(true, Ordering::SeqCst); + let dispatch_entered = platform.dispatch_entered.notified(); + let action = tokio::spawn({ + let controller = Arc::clone(&controller); + let client = client.clone(); + async move { controller.click(&client, command).await } + }); + dispatch_entered.await; + action.abort(); + assert!(action.await.unwrap_err().is_cancelled()); + assert_eq!(platform.cleanup_count.load(Ordering::SeqCst), 1); + + let poisoned = controller + .targets + .get(&TargetKey::from_window(client.clone(), &resolved_window())) + .await + .err() + .expect("cancellation cleanup failure poisons the target"); + assert_eq!(poisoned.code, ErrorCode::WindowIdentityChanged); + + platform.block_dispatch.store(false, Ordering::SeqCst); + platform.cleanup_failure_count.store(0, Ordering::SeqCst); + controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::Full, + }, + ) + .await + .unwrap(); + assert_eq!(platform.target_creations.load(Ordering::SeqCst), 2); + assert_eq!(platform.shutdowns.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn wedged_dispatch_deadline_releases_scope_without_reusing_observation() { + let platform = Arc::new(FakePlatform::default()); + let controller = DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os") + .with_mutation_timeouts(Duration::from_millis(40), Duration::from_millis(100)); + let client = id("deadline-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::DiffIfAvailable, + }, + ) + .await + .unwrap(); + let command = ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Point { + observation_id: observed.observation_id.clone(), + surface_id: observed.surfaces[0].id.clone(), + point: Point { x: 20.0, y: 20.0 }, + }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }; + + platform.block_dispatch.store(true, Ordering::SeqCst); + let error = controller + .click(&client, command.clone()) + .await + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::DispatchFailed); + assert_eq!(error.phase, ErrorPhase::Dispatch); + assert!(!error.retryable); + let Some(PartialEvidence::Action { + dispatch: None, + pending_settlement: Some(pending), + native_evidence, + .. + }) = error.partial_evidence.as_deref() + else { + panic!("dispatch deadline must keep incomplete-dispatch and teardown evidence"); + }; + assert!(!pending + .observed_signals + .contains(&SettlementSignal::DispatchComplete)); + assert_eq!( + native_evidence.fields.get("fake_cleanup"), + Some(&serde_json::json!("complete")) + ); + assert_eq!(platform.cleanup_count.load(Ordering::SeqCst), 1); + let ordering = platform.ordering.lock().unwrap(); + let dispatch_index = ordering + .iter() + .position(|event| *event == "dispatch") + .unwrap(); + let release_index = ordering + .iter() + .position(|event| *event == "scope_released") + .unwrap(); + assert!(dispatch_index < release_index); + drop(ordering); + + let target = controller + .targets + .get(&TargetKey::from_window(client.clone(), &resolved_window())) + .await + .unwrap(); + let mut state = target.state.lock().await; + assert!(state.settlement.settled_evidence().is_none()); + let stale = state + .observations + .current(&observed.observation_id, &resolved_window()) + .unwrap_err(); + assert_eq!(stale.code, ErrorCode::ObservationStale); + drop(state); + platform.block_dispatch.store(false, Ordering::SeqCst); + let dirty = controller.click(&client, command).await.unwrap_err(); + assert_eq!(dirty.code, ErrorCode::UiNotSettled); +} + +#[tokio::test] +async fn deadline_cleanup_failure_keeps_evidence_and_removes_poisoned_target() { + let platform = Arc::new(FakePlatform::default()); + platform.cleanup_failure_count.store(1, Ordering::SeqCst); + let controller = DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os") + .with_mutation_timeouts(Duration::from_millis(40), Duration::from_millis(100)); + let client = id("deadline-cleanup-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::DiffIfAvailable, + }, + ) + .await + .unwrap(); + platform.block_dispatch.store(true, Ordering::SeqCst); + let error = controller + .click( + &client, + ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Point { + observation_id: observed.observation_id, + surface_id: observed.surfaces[0].id.clone(), + point: Point { x: 20.0, y: 20.0 }, + }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }, + ) + .await + .unwrap_err(); + + assert_eq!(error.code, ErrorCode::DispatchFailed); + assert!(error + .related_failures + .iter() + .any(|failure| failure.code == ErrorCode::VerificationFailed)); + let Some(PartialEvidence::Action { + dispatch: None, + pending_settlement: Some(pending), + native_evidence, + .. + }) = error.partial_evidence.as_deref() + else { + panic!("cleanup failure must retain dispatch, settlement, and scope evidence"); + }; + assert!(!pending + .observed_signals + .contains(&SettlementSignal::DispatchComplete)); + assert_eq!( + native_evidence.fields.get("fake_cleanup"), + Some(&serde_json::json!("complete")) + ); + assert_eq!( + native_evidence + .interaction_scope + .as_ref() + .and_then(|evidence| evidence.teardown.as_ref()) + .map(|teardown| teardown.target_belief), + Some(LeaseTeardownStatus::Failed) + ); + assert_eq!(platform.cleanup_count.load(Ordering::SeqCst), 1); + let missing = controller + .targets + .get(&TargetKey::from_window(client, &resolved_window())) + .await + .err() + .expect("deadline cleanup failure removes the poisoned target"); + assert_eq!(missing.code, ErrorCode::ObservationStale); + assert_eq!(platform.shutdowns.load(Ordering::SeqCst), 1); +} + +#[test] +fn pending_settlement_progress_survives_for_the_next_attempt() { + let action_id = id("pending-action", ActionId::parse); + let profile = SettlementProfile::requiring( + "exact_value", + [ + SettlementSignal::AxValueChanged, + SettlementSignal::VerificationReadbackComplete, + ], + ); + let mut state = SettlementState::default(); + state.mark_dirty(action_id.clone(), profile).unwrap(); + let dirty = state.begin(false).unwrap(); + let pending = PendingSettlementEvidence { + state: PendingSettlementState::Pending, + trigger_action_id: action_id, + profile: "exact_value".to_owned(), + elapsed_ms: 20, + observed_signals: vec![ + SettlementSignal::DispatchComplete, + SettlementSignal::AxValueChanged, + ], + missing_signals: vec![SettlementSignal::VerificationReadbackComplete], + }; + assert!(!dirty + .observed_signals + .contains(&SettlementSignal::AxValueChanged)); + state.preserve_pending(&pending).unwrap(); + let resumed = state.begin(true).unwrap(); + assert!(resumed + .observed_signals + .contains(&SettlementSignal::AxValueChanged)); + assert!(resumed.resumed_from_prior_call); +} + +#[test] +fn menu_state_only_publishes_stable_lifecycle_states() { + let mut menu = MenuControllerState::default(); + let action = id("action-1", ActionId::parse); + let owner = resolved_window().stamp(); + let menu_id = menu.begin_open(action.clone(), window_ref(), owner.clone()); + let transitional = menu.observation().unwrap_err(); + assert_eq!(transitional.code, ErrorCode::MenuStateStale); + menu.record_open( + &menu_id, + &menu_id, + &action, + &owner, + NativeMenuIdentity { + process: NativeProcessHandle::new("menu-process").unwrap(), + window: NativeWindowHandle::new("menu-window").unwrap(), + generation: WindowGeneration(1), + }, + Vec::new(), + None, + ) + .unwrap(); + assert!(matches!( + menu.observation().unwrap(), + MenuState::Open { + opened_by_action_id, + .. + } if opened_by_action_id == action + )); + menu.begin_dismiss(id("action-2", ActionId::parse)).unwrap(); + menu.close(); + assert!(matches!( + menu.observation().unwrap(), + MenuState::Closed { .. } + )); +} + +#[allow(dead_code)] +fn _settlement_type_assertion(_: BTreeSet) {} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs index 61471b3eec..02dab5b908 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/apps/mod.rs @@ -332,7 +332,7 @@ pub(crate) fn locate_by_name(name: &str) -> Option { /// Read `CFBundleIdentifier` from an `.app` bundle's `Info.plist`. /// Falls back to shelling out to `plutil` (already used elsewhere in /// this file) to avoid pulling in a plist crate just for this. -fn bundle_id_for_app_path(app_path: &str) -> Option { +pub(crate) fn bundle_id_for_app_path(app_path: &str) -> Option { let plist = format!("{app_path}/Contents/Info.plist"); let out = Command::new("plutil") .args(["-extract", "CFBundleIdentifier", "raw", "-o", "-", &plist]) @@ -391,6 +391,13 @@ pub fn list_all_apps() -> Vec { all } +/// Enumerate installed application bundles without consulting running +/// processes. The v2 lifecycle provider combines this with native +/// `NSWorkspace.runningApplications` data and does not shell to System Events. +pub fn list_installed_apps() -> Vec { + scan_installed_apps() +} + fn scan_installed_apps() -> Vec { let dirs = [ "/Applications", diff --git a/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs b/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs index ec6bdf8e00..756295178e 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/apps/nsworkspace.rs @@ -28,25 +28,100 @@ //! hand-roll the binding in [`apple_event`] via `extern_methods!`. use std::ptr::NonNull; -use std::sync::Arc; +use std::sync::{Arc, OnceLock}; use std::time::Duration; use block2::RcBlock; use objc2::rc::Retained; use objc2_app_kit::{ - NSRunningApplication, NSWorkspace, NSWorkspaceOpenConfiguration, + NSApplicationActivationPolicy, NSRunningApplication, NSWorkspace, NSWorkspaceApplicationKey, + NSWorkspaceDidActivateApplicationNotification, NSWorkspaceDidDeactivateApplicationNotification, + NSWorkspaceDidLaunchApplicationNotification, NSWorkspaceDidTerminateApplicationNotification, + NSWorkspaceOpenConfiguration, }; use objc2_foundation::{ - NSAppleEventDescriptor, NSArray, NSDictionary, NSError, NSString, NSURL, + NSAppleEventDescriptor, NSArray, NSDictionary, NSError, NSNotification, NSNotificationName, + NSOperationQueue, NSString, NSURL, }; +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkspaceEventKind { + Launched, + Terminated, + Activated, + Deactivated, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct WorkspaceEvent { + pub kind: WorkspaceEventKind, + pub pid: i32, + pub bundle_id: Option, + pub process_generation: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RunningApplicationInfo { + pub pid: i32, + pub name: Option, + pub bundle_id: Option, + pub executable_path: Option, + pub process_generation: Option, + pub active: bool, + pub hidden: bool, + pub finished_launching: bool, + pub regular: bool, +} + +/// Process-wide NSWorkspace notification fanout. AppKit owns the observer +/// registrations for the daemon lifetime; individual callers own bounded +/// broadcast receivers and cannot leak suppression or callback state. +pub struct WorkspaceEventHub { + sender: tokio::sync::broadcast::Sender, + observer_queue: Retained, +} + +impl WorkspaceEventHub { + pub fn shared() -> Arc { + static HUB: OnceLock> = OnceLock::new(); + HUB.get_or_init(|| { + let (sender, _) = tokio::sync::broadcast::channel(256); + // All four lifecycle notifications share one serial queue. Besides + // preserving native order, this gives bounded witnesses a real + // barrier: an operation enqueued after native teardown runs only + // after every previously delivered workspace callback. + let observer_queue = unsafe { NSOperationQueue::new() }; + unsafe { observer_queue.setMaxConcurrentOperationCount(1) }; + let hub = Arc::new(Self { + sender, + observer_queue, + }); + install_workspace_observers(&hub); + hub + }) + .clone() + } + + pub fn subscribe(&self) -> tokio::sync::broadcast::Receiver { + self.sender.subscribe() + } + + /// Wait until every workspace callback queued before this call has run. + /// A timeout is not treated as success by posture witnesses. + pub fn barrier(&self, timeout: Duration) -> bool { + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let block = RcBlock::new(move || { + let _ = sender.send(()); + }); + unsafe { self.observer_queue.addBarrierBlock(&block) }; + receiver.recv_timeout(timeout).is_ok() + } +} + /// FourCharCode helper — packs a 4-byte ASCII tag into a `u32` the same way /// Apple's CoreServices headers do (`kCoreEventClass = 'aevt'` etc). const fn fourcc(s: &[u8; 4]) -> u32 { - ((s[0] as u32) << 24) - | ((s[1] as u32) << 16) - | ((s[2] as u32) << 8) - | (s[3] as u32) + ((s[0] as u32) << 24) | ((s[1] as u32) << 16) | ((s[2] as u32) << 8) | (s[3] as u32) } const K_CORE_EVENT_CLASS: u32 = fourcc(b"aevt"); // kCoreEventClass @@ -87,8 +162,8 @@ pub enum LaunchError { Cocoa(String), #[error("NSWorkspace launch returned no NSRunningApplication and no NSError")] NoApp, - #[error("NSWorkspace launch did not complete within {:?}", COMPLETION_TIMEOUT)] - Timeout, + #[error("NSWorkspace launch did not complete within {0:?}")] + Timeout(Duration), #[error("invalid url: {0}")] BadUrl(String), } @@ -129,14 +204,35 @@ pub fn open_application( ?app_url, "calling openApplicationAtURL"); unsafe { - ws.openApplicationAtURL_configuration_completionHandler( - &url, - &config, - Some(&block), - ); + ws.openApplicationAtURL_configuration_completionHandler(&url, &config, Some(&block)); } - wait_for_completion(rx) + wait_for_completion(rx, COMPLETION_TIMEOUT) +} + +/// Launch with a caller-owned completion deadline and return only the pid. +/// Keeping the Cocoa object inside the callback avoids sending AppKit's +/// non-Send `NSRunningApplication` across the worker channel. +pub fn open_application_pid_with_timeout( + app_url: &str, + cfg: &OpenConfig, + timeout: Duration, + on_completion_pid: F, +) -> Result { + let ws = unsafe { NSWorkspace::sharedWorkspace() }; + let url = resolve_application_url(&ws, app_url)?; + let config = build_configuration(cfg); + let (tx, rx) = std::sync::mpsc::sync_channel::(1); + let tx = Arc::new(std::sync::Mutex::new(Some(tx))); + let block = make_pid_completion_block(tx, Arc::new(on_completion_pid)); + unsafe { + ws.openApplicationAtURL_configuration_completionHandler(&url, &config, Some(&block)); + } + match rx.recv_timeout(timeout) { + Ok(result) => result, + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(LaunchError::Timeout(timeout)), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err(LaunchError::NoApp), + } } /// Launch the application bundle at `app_url` and hand it `urls` via @@ -179,7 +275,7 @@ pub fn open_urls_with_application( ); } - wait_for_completion(rx) + wait_for_completion(rx, COMPLETION_TIMEOUT) } // ── Internal helpers ───────────────────────────────────────────────────────── @@ -291,6 +387,38 @@ fn file_or_app_url(s: &str) -> Result, LaunchError> { } type CompletionResult = Result, LaunchError>; +type PidCompletionResult = Result; + +fn make_pid_completion_block( + tx: Arc>>>, + on_completion_pid: Arc, +) -> RcBlock { + RcBlock::new( + move |app_ptr: *mut NSRunningApplication, err_ptr: *mut NSError| { + let result = unsafe { + if !err_ptr.is_null() { + let description = (&*err_ptr).localizedDescription(); + Err(LaunchError::Cocoa(description.to_string())) + } else if app_ptr.is_null() { + Err(LaunchError::NoApp) + } else { + Ok((&*app_ptr).processIdentifier()) + } + }; + if let Ok(pid) = result.as_ref() { + // Narrow detached wildcard containment before waking the + // awaiting task. The callback can outlive timeout/cancellation; + // its deadline-owned registry entry does not depend on this + // stack frame remaining alive. + on_completion_pid(*pid); + } + let sender = tx.lock().ok().and_then(|mut guard| guard.take()); + if let Some(sender) = sender { + let _ = sender.send(result); + } + }, + ) +} /// Build the `(NSRunningApplication?, NSError?) -> Void` completion block. /// @@ -330,22 +458,130 @@ fn make_completion_block( /// the completion handler never fires (wedged LaunchServices). fn wait_for_completion( rx: std::sync::mpsc::Receiver, + timeout: Duration, ) -> Result, LaunchError> { - match rx.recv_timeout(COMPLETION_TIMEOUT) { + match rx.recv_timeout(timeout) { Ok(r) => r, - Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(LaunchError::Timeout), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(LaunchError::Timeout(timeout)), Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err(LaunchError::NoApp), } } +/// Enumerate regular GUI applications without shelling out to System Events. +pub fn running_applications() -> Vec { + use objc2::msg_send_id; + + let workspace = unsafe { NSWorkspace::sharedWorkspace() }; + let applications: Retained> = + unsafe { msg_send_id![&*workspace, runningApplications] }; + (0..applications.count()) + .map(|index| unsafe { applications.objectAtIndex(index) }) + .map(|application| running_application_info(&application)) + .filter(|application| application.regular) + .collect() +} + +pub fn running_application(pid: i32) -> Option { + let application = + unsafe { NSRunningApplication::runningApplicationWithProcessIdentifier(pid) }?; + Some(running_application_info(&application)) +} + +fn running_application_info(application: &NSRunningApplication) -> RunningApplicationInfo { + let pid = unsafe { application.processIdentifier() }; + let name = unsafe { application.localizedName() }.map(|value| value.to_string()); + let bundle_id = unsafe { application.bundleIdentifier() }.map(|value| value.to_string()); + let executable_path = unsafe { application.executableURL() } + .and_then(|url| unsafe { url.path() }) + .map(|path| path.to_string()); + let process_generation = unsafe { application.launchDate() } + .map(|date| unsafe { date.timeIntervalSince1970() }.to_bits()); + RunningApplicationInfo { + pid, + name, + bundle_id, + executable_path, + process_generation, + active: unsafe { application.isActive() }, + hidden: unsafe { application.isHidden() }, + finished_launching: unsafe { application.isFinishedLaunching() }, + regular: unsafe { application.activationPolicy() } + == NSApplicationActivationPolicy::Regular, + } +} + +fn install_workspace_observers(hub: &Arc) { + install_workspace_observer( + hub, + unsafe { NSWorkspaceDidLaunchApplicationNotification }, + WorkspaceEventKind::Launched, + ); + install_workspace_observer( + hub, + unsafe { NSWorkspaceDidTerminateApplicationNotification }, + WorkspaceEventKind::Terminated, + ); + install_workspace_observer( + hub, + unsafe { NSWorkspaceDidActivateApplicationNotification }, + WorkspaceEventKind::Activated, + ); + install_workspace_observer( + hub, + unsafe { NSWorkspaceDidDeactivateApplicationNotification }, + WorkspaceEventKind::Deactivated, + ); +} + +fn install_workspace_observer( + hub: &Arc, + name: &'static NSNotificationName, + kind: WorkspaceEventKind, +) { + use objc2::{msg_send, runtime::AnyObject}; + + let workspace = unsafe { NSWorkspace::sharedWorkspace() }; + let center = unsafe { workspace.notificationCenter() }; + let queue = hub.observer_queue.clone(); + let hub = Arc::clone(hub); + let block = RcBlock::new(move |note_ptr: NonNull| { + let note = unsafe { note_ptr.as_ref() }; + let Some(info) = (unsafe { note.userInfo() }) else { + return; + }; + let app_ptr: *mut AnyObject = + unsafe { msg_send![&*info, objectForKey: NSWorkspaceApplicationKey] }; + if app_ptr.is_null() { + return; + } + // The notification owns the NSRunningApplication for the callback + // span, including termination notifications where the process has + // already disappeared from the global running-app registry. Read the + // launch generation from that exact object so app-exit invalidation + // cannot accidentally target a newer process that reused the pid. + let application = unsafe { &*app_ptr.cast::() }; + let application = running_application_info(application); + let _ = hub.sender.send(WorkspaceEvent { + kind, + pid: application.pid, + bundle_id: application.bundle_id, + process_generation: application.process_generation, + }); + }); + let token = unsafe { + center.addObserverForName_object_queue_usingBlock(Some(name), None, Some(&queue), &block) + }; + std::mem::forget(token); +} + /// Hand-rolled binding for /// `-[NSAppleEventDescriptor initWithEventClass:eventID:targetDescriptor:returnID:transactionID:]` /// (and the associated `OpenApplication` event constructor) — the selector /// is not exposed by `objc2-foundation 0.2.2`. mod apple_event { use super::{ - NSAppleEventDescriptor, NSString, Retained, K_AE_OPEN_APPLICATION, - K_ANY_TRANSACTION_ID, K_AUTO_GENERATE_RETURN_ID, K_CORE_EVENT_CLASS, + NSAppleEventDescriptor, NSString, Retained, K_AE_OPEN_APPLICATION, K_ANY_TRANSACTION_ID, + K_AUTO_GENERATE_RETURN_ID, K_CORE_EVENT_CLASS, }; use objc2::msg_send_id; use objc2::rc::Allocated; @@ -354,13 +590,11 @@ mod apple_event { /// Build an `aevt/oapp` AppleEvent addressed to the bundle id `bid`. pub fn open_application_event(bid: &str) -> Retained { let target_string = NSString::from_str(bid); - let target: Retained = unsafe { - NSAppleEventDescriptor::descriptorWithBundleIdentifier(&target_string) - }; + let target: Retained = + unsafe { NSAppleEventDescriptor::descriptorWithBundleIdentifier(&target_string) }; unsafe { - let alloc: Allocated = - NSAppleEventDescriptor::alloc(); + let alloc: Allocated = NSAppleEventDescriptor::alloc(); // initWithEventClass:eventID:targetDescriptor:returnID:transactionID: let event: Retained = msg_send_id![ alloc, @@ -374,4 +608,3 @@ mod apple_event { } } } - diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs index b4e4c8902d..d031651006 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/bindings.rs @@ -3,11 +3,17 @@ //! We call the C-level AX API directly rather than using a crate wrapper, //! because most available crates are incomplete or unmaintained. -#![allow(non_upper_case_globals, non_camel_case_types, non_snake_case, dead_code)] +#![allow( + non_upper_case_globals, + non_camel_case_types, + non_snake_case, + dead_code +)] use core_foundation::{ array::CFArrayRef, base::{CFRelease, CFRetain, CFTypeID, CFTypeRef}, + runloop::CFRunLoopSourceRef, string::CFStringRef, }; use std::os::raw::{c_int, c_void}; @@ -18,6 +24,18 @@ use std::os::raw::{c_int, c_void}; pub struct __AXUIElement(c_void); pub type AXUIElementRef = *mut __AXUIElement; +// ── AXObserver opaque type ────────────────────────────────────────────────── + +#[repr(C)] +pub struct __AXObserver(c_void); +pub type AXObserverRef = *mut __AXObserver; +pub type AXObserverCallback = unsafe extern "C" fn( + observer: AXObserverRef, + element: AXUIElementRef, + notification: CFStringRef, + refcon: *mut c_void, +); + // ── AXError ────────────────────────────────────────────────────────────────── pub type AXError = c_int; @@ -54,26 +72,44 @@ extern "C" { element: AXUIElementRef, names: *mut CFArrayRef, ) -> AXError; - pub fn AXUIElementCopyActionNames( - element: AXUIElementRef, - names: *mut CFArrayRef, - ) -> AXError; - pub fn AXUIElementPerformAction( - element: AXUIElementRef, - action: CFStringRef, - ) -> AXError; + pub fn AXUIElementCopyActionNames(element: AXUIElementRef, names: *mut CFArrayRef) -> AXError; + pub fn AXUIElementPerformAction(element: AXUIElementRef, action: CFStringRef) -> AXError; pub fn AXUIElementSetAttributeValue( element: AXUIElementRef, attribute: CFStringRef, value: CFTypeRef, ) -> AXError; + pub fn AXUIElementIsAttributeSettable( + element: AXUIElementRef, + attribute: CFStringRef, + settable: *mut u8, + ) -> AXError; pub fn AXUIElementGetTypeID() -> CFTypeID; + pub fn AXObserverCreate( + application: i32, + callback: AXObserverCallback, + observer: *mut AXObserverRef, + ) -> AXError; + pub fn AXObserverAddNotification( + observer: AXObserverRef, + element: AXUIElementRef, + notification: CFStringRef, + refcon: *mut c_void, + ) -> AXError; + pub fn AXObserverRemoveNotification( + observer: AXObserverRef, + element: AXUIElementRef, + notification: CFStringRef, + ) -> AXError; + pub fn AXObserverGetRunLoopSource(observer: AXObserverRef) -> CFRunLoopSourceRef; pub fn AXIsProcessTrusted() -> bool; /// `AXIsProcessTrustedWithOptions(options)` — when called with /// `{kAXTrustedCheckOptionPrompt: true}` raises the system Accessibility /// prompt if the process isn't already trusted. Returns the post-prompt /// trust state (may still be false if the user dismissed the prompt). - pub fn AXIsProcessTrustedWithOptions(options: core_foundation::dictionary::CFDictionaryRef) -> bool; + pub fn AXIsProcessTrustedWithOptions( + options: core_foundation::dictionary::CFDictionaryRef, + ) -> bool; /// Private SPI: maps an AX window element to its CGWindowID. /// Stable since macOS 10.9; used by yabai, Hammerspoon, Accessibility Inspector. @@ -83,33 +119,50 @@ extern "C" { // ── AXValue functions ──────────────────────────────────────────────────────── #[link(name = "ApplicationServices", kind = "framework")] extern "C" { + pub fn AXValueCreate(the_type: AXValueType, value_ptr: *const c_void) -> AXValueRef; + pub fn AXValueGetTypeID() -> CFTypeID; pub fn AXValueGetType(value: AXValueRef) -> AXValueType; - pub fn AXValueGetValue(value: AXValueRef, the_type: AXValueType, value_ptr: *mut c_void) -> bool; + pub fn AXValueGetValue( + value: AXValueRef, + the_type: AXValueType, + value_ptr: *mut c_void, + ) -> bool; } // ── Helper functions ────────────────────────────────────────────────────────── -use core_foundation::{ - array::CFArray, - base::TCFType, - string::CFString as CFStr, -}; +use core_foundation::{array::CFArray, base::TCFType, string::CFString as CFStr}; /// Copy a string attribute from an AX element. Returns `None` on any error. pub unsafe fn copy_string_attr(element: AXUIElementRef, attr_name: &str) -> Option { + copy_string_attr_exact(element, attr_name).ok().flatten() +} + +/// Copy a string attribute without collapsing query/type failures into a +/// truthful missing value. +pub unsafe fn copy_string_attr_exact( + element: AXUIElementRef, + attr_name: &str, +) -> Result, AXError> { let attr = CFStr::new(attr_name); let mut value: CFTypeRef = std::ptr::null(); let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value); - if err != kAXErrorSuccess || value.is_null() { - return None; + if err == kAXErrorNoValue || err == kAXErrorAttributeUnsupported { + return Ok(None); + } + if err != kAXErrorSuccess { + return Err(err); + } + if value.is_null() { + return Ok(None); } let cf_string_type_id = CFStr::type_id(); if core_foundation::base::CFGetTypeID(value) != cf_string_type_id { CFRelease(value); - return None; + return Err(kAXErrorFailure); } let s = CFStr::wrap_under_create_rule(value as _); - Some(s.to_string()) + Ok(Some(s.to_string())) } /// Copy a numeric attribute from an AX element as an `f64`. Returns `None` on @@ -133,21 +186,190 @@ pub unsafe fn copy_number_attr(element: AXUIElementRef, attr_name: &str) -> Opti n.to_f64() } +/// Copy a boolean attribute from an AX element. Returns `None` when the +/// attribute is missing, unsupported, or not a CFBoolean. +pub unsafe fn copy_bool_attr(element: AXUIElementRef, attr_name: &str) -> Option { + use core_foundation::boolean::CFBoolean; + + let attr = CFStr::new(attr_name); + let mut value: CFTypeRef = std::ptr::null(); + let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value); + if err != kAXErrorSuccess || value.is_null() { + return None; + } + if core_foundation::base::CFGetTypeID(value) != CFBoolean::type_id() { + CFRelease(value); + return None; + } + let boolean = CFBoolean::wrap_under_create_rule(value as _); + Some(bool::from(boolean)) +} + /// Get the action names for an AX element. pub unsafe fn copy_action_names(element: AXUIElementRef) -> Vec { + copy_action_names_exact(element).unwrap_or_default() +} + +/// Get the exact current action-name list without collapsing an AX query +/// failure into a truthful empty list. +pub unsafe fn copy_action_names_exact(element: AXUIElementRef) -> Result, AXError> { let mut names: CFArrayRef = std::ptr::null_mut(); let err = AXUIElementCopyActionNames(element, &mut names); - if err != kAXErrorSuccess || names.is_null() { - return vec![]; + if err != kAXErrorSuccess { + return Err(err); + } + if names.is_null() { + return Err(kAXErrorFailure); } // Use CFArray (the typed wrapper) to satisfy FromVoid bound. let arr = CFArray::::wrap_under_create_rule(names); - (0..arr.len()) + Ok((0..arr.len()) .filter_map(|i| { let cf = arr.get(i)?; Some(cf.to_string()) }) - .collect() + .collect()) +} + +/// Copy an arbitrary AX attribute value under the create rule. The caller +/// owns any non-null result and must release it. +pub unsafe fn copy_attr_value( + element: AXUIElementRef, + attr_name: &str, +) -> Result, AXError> { + let attr = CFStr::new(attr_name); + let mut value: CFTypeRef = std::ptr::null(); + let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value); + if err == kAXErrorNoValue || err == kAXErrorAttributeUnsupported { + return Ok(None); + } + if err != kAXErrorSuccess { + return Err(err); + } + if value.is_null() { + Ok(None) + } else { + Ok(Some(value)) + } +} + +/// Core Foundation's `CFRange` layout, used by `AXSelectedTextRange`. +/// Locations and lengths are UTF-16 code-unit offsets for AX text elements. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[repr(C)] +pub struct AxCfRange { + pub location: isize, + pub length: isize, +} + +impl AxCfRange { + pub fn from_utf16(location: usize, length: usize) -> Option { + Some(Self { + location: isize::try_from(location).ok()?, + length: isize::try_from(length).ok()?, + }) + } +} + +/// Return whether AX says an attribute is writable. Errors and indeterminate +/// answers are not promoted to `false`: callers need the distinction to avoid +/// guessing a route from an incomplete preflight. +pub unsafe fn is_attribute_settable( + element: AXUIElementRef, + attr_name: &str, +) -> Result { + let attr = CFStr::new(attr_name); + let mut settable = 0_u8; + let err = AXUIElementIsAttributeSettable(element, attr.as_concrete_TypeRef(), &mut settable); + if err == kAXErrorSuccess { + Ok(settable != 0) + } else { + Err(err) + } +} + +/// Copy an AX element-valued attribute. The returned reference follows the +/// create rule and must be released by the caller. +pub unsafe fn copy_element_attr( + element: AXUIElementRef, + attr_name: &str, +) -> Result, AXError> { + let attr = CFStr::new(attr_name); + let mut value: CFTypeRef = std::ptr::null(); + let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value); + if err == kAXErrorNoValue || err == kAXErrorAttributeUnsupported { + return Ok(None); + } + if err != kAXErrorSuccess { + return Err(err); + } + if value.is_null() { + return Ok(None); + } + if core_foundation::base::CFGetTypeID(value) != AXUIElementGetTypeID() { + CFRelease(value); + return Err(kAXErrorFailure); + } + Ok(Some(value as AXUIElementRef)) +} + +/// Copy a `kAXValueCFRangeType` attribute. +pub unsafe fn copy_cf_range_attr( + element: AXUIElementRef, + attr_name: &str, +) -> Result, AXError> { + let attr = CFStr::new(attr_name); + let mut value: CFTypeRef = std::ptr::null(); + let err = AXUIElementCopyAttributeValue(element, attr.as_concrete_TypeRef(), &mut value); + if err == kAXErrorNoValue || err == kAXErrorAttributeUnsupported { + return Ok(None); + } + if err != kAXErrorSuccess { + return Err(err); + } + if value.is_null() { + return Ok(None); + } + if core_foundation::base::CFGetTypeID(value) != AXValueGetTypeID() { + CFRelease(value); + return Err(kAXErrorFailure); + } + let value = value as AXValueRef; + if AXValueGetType(value) != kAXValueCFRangeType { + CFRelease(value as CFTypeRef); + return Err(kAXErrorFailure); + } + let mut range = AxCfRange { + location: 0, + length: 0, + }; + let copied = AXValueGetValue( + value, + kAXValueCFRangeType, + &mut range as *mut _ as *mut c_void, + ); + CFRelease(value as CFTypeRef); + if copied { + Ok(Some(range)) + } else { + Err(kAXErrorFailure) + } +} + +/// Set a `kAXValueCFRangeType` attribute. +pub unsafe fn set_cf_range_attr( + element: AXUIElementRef, + attr_name: &str, + range: AxCfRange, +) -> AXError { + let value = AXValueCreate(kAXValueCFRangeType, &range as *const _ as *const c_void); + if value.is_null() { + return kAXErrorFailure; + } + let attr = CFStr::new(attr_name); + let err = AXUIElementSetAttributeValue(element, attr.as_concrete_TypeRef(), value as CFTypeRef); + CFRelease(value as CFTypeRef); + err } /// Read the on-screen center of an AX element (AXPosition + AXSize → center). @@ -162,7 +384,10 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 return None; } #[repr(C)] - struct CGPoint { x: f64, y: f64 } + struct CGPoint { + x: f64, + y: f64, + } let mut pos = CGPoint { x: 0.0, y: 0.0 }; let ok = AXValueGetValue( pos_ref as AXValueRef, @@ -170,7 +395,9 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 &mut pos as *mut _ as *mut std::ffi::c_void, ); CFRelease(pos_ref); - if !ok { return None; } + if !ok { + return None; + } // AXSize → CGSize let sz_attr = CFStr::new("AXSize"); @@ -180,7 +407,10 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 return None; } #[repr(C)] - struct CGSize { w: f64, h: f64 } + struct CGSize { + w: f64, + h: f64, + } let mut sz = CGSize { w: 0.0, h: 0.0 }; let ok2 = AXValueGetValue( sz_ref as AXValueRef, @@ -188,7 +418,9 @@ pub unsafe fn element_screen_center(element: AXUIElementRef) -> Option<(f64, f64 &mut sz as *mut _ as *mut std::ffi::c_void, ); CFRelease(sz_ref); - if !ok2 || sz.w < 1.0 || sz.h < 1.0 { return None; } + if !ok2 || sz.w < 1.0 || sz.h < 1.0 { + return None; + } Some((pos.x + sz.w / 2.0, pos.y + sz.h / 2.0)) } @@ -204,7 +436,10 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { return None; } #[repr(C)] - struct CGPoint { x: f64, y: f64 } + struct CGPoint { + x: f64, + y: f64, + } let mut pos = CGPoint { x: 0.0, y: 0.0 }; let ok = AXValueGetValue( pos_ref as AXValueRef, @@ -212,7 +447,9 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { &mut pos as *mut _ as *mut std::ffi::c_void, ); CFRelease(pos_ref); - if !ok { return None; } + if !ok { + return None; + } // AXSize → CGSize let sz_attr = CFStr::new("AXSize"); @@ -222,7 +459,10 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { return None; } #[repr(C)] - struct CGSize { w: f64, h: f64 } + struct CGSize { + w: f64, + h: f64, + } let mut sz = CGSize { w: 0.0, h: 0.0 }; let ok2 = AXValueGetValue( sz_ref as AXValueRef, @@ -230,7 +470,9 @@ pub unsafe fn element_screen_rect(element: AXUIElementRef) -> Option<[f64; 4]> { &mut sz as *mut _ as *mut std::ffi::c_void, ); CFRelease(sz_ref); - if !ok2 || sz.w < 1.0 || sz.h < 1.0 { return None; } + if !ok2 || sz.w < 1.0 || sz.h < 1.0 { + return None; + } Some([pos.x, pos.y, sz.w, sz.h]) } @@ -352,7 +594,11 @@ pub unsafe fn enable_chromium_accessibility(app_element: AXUIElementRef) -> bool pub unsafe fn ax_get_window_id(element: AXUIElementRef) -> Option { let mut wid: u32 = 0; let err = _AXUIElementGetWindow(element, &mut wid); - if err == kAXErrorSuccess && wid != 0 { Some(wid) } else { None } + if err == kAXErrorSuccess && wid != 0 { + Some(wid) + } else { + None + } } /// Read the `AXWindows` attribute of an application element. diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/enablement.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/enablement.rs new file mode 100644 index 0000000000..899862d050 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/enablement.rs @@ -0,0 +1,288 @@ +//! Prior-state-preserving scoped AX enablement for Chromium accessibility. + +use std::sync::Arc; + +use core_foundation::{ + base::{CFGetTypeID, CFRelease, CFTypeRef, TCFType}, + boolean::CFBoolean, + string::CFString, +}; +use cua_driver_core::api::errors::{ErrorCode, ErrorPhase, NativeError}; + +use super::bindings::{ + kAXErrorAttributeUnsupported, kAXErrorNoValue, kAXErrorSuccess, AXUIElementCopyAttributeValue, + AXUIElementCreateApplication, AXUIElementSetAttributeValue, +}; + +const ATTRIBUTES: [&str; 2] = ["AXManualAccessibility", "AXEnhancedUserInterface"]; + +pub(crate) trait AxBooleanAccess: Send + Sync { + fn read(&self, pid: i32, attribute: &'static str) -> Result, NativeError>; + fn write( + &self, + pid: i32, + attribute: &'static str, + value: bool, + phase: ErrorPhase, + ) -> Result<(), NativeError>; +} + +#[derive(Default)] +struct SystemAxBooleanAccess; + +impl AxBooleanAccess for SystemAxBooleanAccess { + fn read(&self, pid: i32, attribute: &'static str) -> Result, NativeError> { + unsafe { + let application = AXUIElementCreateApplication(pid); + if application.is_null() { + return Err(ax_error( + ErrorPhase::Preflight, + true, + "AX application element is unavailable during enablement preflight", + pid, + attribute, + None, + )); + } + let name = CFString::new(attribute); + let mut value: CFTypeRef = std::ptr::null(); + let status = + AXUIElementCopyAttributeValue(application, name.as_concrete_TypeRef(), &mut value); + CFRelease(application as CFTypeRef); + if status == kAXErrorAttributeUnsupported || status == kAXErrorNoValue { + return Ok(None); + } + if status != kAXErrorSuccess || value.is_null() { + return Err(ax_error( + ErrorPhase::Preflight, + true, + "AX enablement state could not be read exactly", + pid, + attribute, + Some(status), + )); + } + if CFGetTypeID(value) != CFBoolean::type_id() { + CFRelease(value); + return Err(ax_error( + ErrorPhase::Preflight, + false, + "AX enablement attribute was not a boolean", + pid, + attribute, + None, + )); + } + let value = CFBoolean::wrap_under_create_rule(value.cast_mut().cast()); + Ok(Some(bool::from(value))) + } + } + + fn write( + &self, + pid: i32, + attribute: &'static str, + value: bool, + phase: ErrorPhase, + ) -> Result<(), NativeError> { + unsafe { + let application = AXUIElementCreateApplication(pid); + if application.is_null() { + return Err(ax_error( + phase, + true, + "AX application element is unavailable during enablement write", + pid, + attribute, + None, + )); + } + let name = CFString::new(attribute); + let value = if value { + CFBoolean::true_value() + } else { + CFBoolean::false_value() + }; + let status = AXUIElementSetAttributeValue( + application, + name.as_concrete_TypeRef(), + value.as_CFTypeRef(), + ); + CFRelease(application as CFTypeRef); + if status == kAXErrorSuccess { + Ok(()) + } else { + Err(ax_error( + phase, + true, + "AX enablement state write failed", + pid, + attribute, + Some(status), + )) + } + } + } +} + +pub struct AxEnablementLease { + pid: i32, + attribute: &'static str, + prior: bool, + changed: bool, + access: Arc, + released: bool, +} + +impl AxEnablementLease { + pub fn acquire(pid: i32) -> Result { + Self::acquire_with(pid, Arc::new(SystemAxBooleanAccess)) + } + + fn acquire_with(pid: i32, access: Arc) -> Result { + for attribute in ATTRIBUTES { + let Some(prior) = access.read(pid, attribute)? else { + continue; + }; + let changed = !prior; + if changed { + access.write(pid, attribute, true, ErrorPhase::Preflight)?; + } + return Ok(Self { + pid, + attribute, + prior, + changed, + access, + released: false, + }); + } + Err(NativeError::unsupported( + "accessibility_enablement_unavailable: neither reversible Chromium AX enablement attribute is readable", + ) + .with_detail("pid", pid)) + } + + pub fn attribute(&self) -> &'static str { + self.attribute + } + + pub fn prior(&self) -> bool { + self.prior + } + + pub fn changed(&self) -> bool { + self.changed + } + + pub fn release(&mut self) -> Result<(), NativeError> { + if self.released { + return Ok(()); + } + if self.changed { + self.access + .write(self.pid, self.attribute, self.prior, ErrorPhase::Verify)?; + } + self.released = true; + Ok(()) + } +} + +impl Drop for AxEnablementLease { + fn drop(&mut self) { + if let Err(error) = self.release() { + tracing::error!(error = %error, "failed to restore prior AX enablement state"); + } + } +} + +fn ax_error( + phase: ErrorPhase, + retryable: bool, + message: &'static str, + pid: i32, + attribute: &'static str, + status: Option, +) -> NativeError { + let mut error = NativeError::new(ErrorCode::Internal, phase, retryable, message) + .with_detail("pid", pid) + .with_detail("attribute", attribute); + if let Some(status) = status { + error = error.with_detail("ax_status", status); + } + error +} + +#[cfg(test)] +mod tests { + use std::{ + collections::HashMap, + sync::{Arc, Mutex}, + }; + + use super::*; + + #[derive(Default)] + struct FakeAccess { + values: Mutex>>, + writes: Mutex>, + } + + impl AxBooleanAccess for FakeAccess { + fn read(&self, _pid: i32, attribute: &'static str) -> Result, NativeError> { + Ok(self + .values + .lock() + .unwrap() + .get(attribute) + .copied() + .flatten()) + } + + fn write( + &self, + _pid: i32, + attribute: &'static str, + value: bool, + _phase: ErrorPhase, + ) -> Result<(), NativeError> { + self.writes.lock().unwrap().push((attribute, value)); + Ok(()) + } + } + + #[test] + fn enablement_restores_the_exact_prior_attribute_value() { + let access = Arc::new(FakeAccess::default()); + access + .values + .lock() + .unwrap() + .insert("AXManualAccessibility", Some(false)); + let mut lease = AxEnablementLease::acquire_with(42, access.clone()).unwrap(); + assert!(lease.changed()); + assert!(!lease.prior()); + lease.release().unwrap(); + assert_eq!( + *access.writes.lock().unwrap(), + vec![ + ("AXManualAccessibility", true), + ("AXManualAccessibility", false) + ] + ); + } + + #[test] + fn already_enabled_state_is_observed_without_a_write_or_clear() { + let access = Arc::new(FakeAccess::default()); + access + .values + .lock() + .unwrap() + .insert("AXManualAccessibility", Some(true)); + let mut lease = AxEnablementLease::acquire_with(42, access.clone()).unwrap(); + assert!(!lease.changed()); + lease.release().unwrap(); + assert!(access.writes.lock().unwrap().is_empty()); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs index 3d4c476763..7a7dda02f9 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/ax/mod.rs @@ -17,6 +17,7 @@ //! - 2-space indent per depth level pub mod bindings; +pub mod enablement; pub mod tree; pub mod cache; diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/mod.rs new file mode 100644 index 0000000000..2460fbbd9a --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/mod.rs @@ -0,0 +1,112 @@ +//! Exact, background-safe macOS semantic action routes. + +mod scroll; +mod semantic; + +pub use semantic::MacSemanticActions; + +use cua_driver_core::api::{ + capabilities::{ + ActionKind, AddressingMode, CapabilityCell, CapabilityKey, Framework, PlatformName, + RouteDecision, WindowStateKind, + }, + contracts::Route, +}; + +pub(crate) fn semantic_capability_cells(os_version: &str) -> Vec { + let frameworks = [ + Framework::Unknown, + Framework::AppKit, + Framework::Chromium, + Framework::WebKit, + ]; + let actions = [ + ActionKind::Click, + ActionKind::Scroll, + ActionKind::SetValue, + ActionKind::SelectText, + ActionKind::PerformSecondaryAction, + ]; + let states = [ + WindowStateKind::Visible, + WindowStateKind::Occluded, + WindowStateKind::Minimized, + WindowStateKind::OffSpace, + WindowStateKind::Unknown, + ]; + let mut cells = Vec::with_capacity(frameworks.len() * actions.len() * states.len()); + for framework in frameworks { + for action in &actions { + for state in &states { + let decision = match state { + WindowStateKind::Visible | WindowStateKind::Occluded => { + RouteDecision::Supported { + route: Route::Semantic, + } + } + WindowStateKind::Minimized => RouteDecision::Unsupported { + reason: "macOS semantic background actions refuse minimized windows" + .to_owned(), + }, + WindowStateKind::OffSpace => RouteDecision::Unsupported { + reason: "macOS semantic background actions refuse off-Space windows" + .to_owned(), + }, + WindowStateKind::Unknown => RouteDecision::Unsupported { + reason: + "macOS semantic background actions require an exact visibility state" + .to_owned(), + }, + }; + cells.push(CapabilityCell { + key: CapabilityKey { + platform: PlatformName::Macos, + os_version: os_version.to_owned(), + action: action.clone(), + addressing: AddressingMode::Element, + framework: framework.clone(), + window_state: state.clone(), + }, + decision, + }); + } + } + } + cells +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn capability_cells_are_semantic_element_only_and_state_explicit() { + let cells = semantic_capability_cells("fixture"); + assert_eq!(cells.len(), 100); + assert!(cells.iter().all(|cell| { + cell.key.addressing == AddressingMode::Element + && !matches!( + cell.key.action, + ActionKind::Drag | ActionKind::PressKey | ActionKind::TypeText + ) + })); + assert_eq!( + cells + .iter() + .filter(|cell| matches!(cell.decision, RouteDecision::Supported { .. })) + .count(), + 40 + ); + assert!(cells + .iter() + .filter(|cell| matches!( + cell.key.window_state, + WindowStateKind::Minimized | WindowStateKind::OffSpace | WindowStateKind::Unknown + )) + .all(|cell| matches!(cell.decision, RouteDecision::Unsupported { .. }))); + assert!(!cells.iter().any(|cell| matches!( + cell.key.framework, + Framework::Electron | Framework::Catalyst + ))); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/scroll.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/scroll.rs new file mode 100644 index 0000000000..9ed454ad04 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/scroll.rs @@ -0,0 +1,139 @@ +use cua_driver_core::api::contracts::ScrollDirection; + +use super::semantic::LiveAxElement; +use crate::driver::observation::RegisteredElementSnapshot; + +pub(super) const AX_PRESS: &str = "AXPress"; + +pub(super) enum PageRoute { + Direct { + element: LiveAxElement, + action: &'static str, + }, + ScrollbarPageChild { + element: LiveAxElement, + action: &'static str, + }, +} + +pub(super) fn direct_page_action( + direction: ScrollDirection, + actions: &[String], + orientation: Option<&str>, +) -> Option<&'static str> { + let directional_action = match direction { + ScrollDirection::Up => "AXPageUp", + ScrollDirection::Down => "AXPageDown", + ScrollDirection::Left => "AXPageLeft", + ScrollDirection::Right => "AXPageRight", + }; + if actions + .iter() + .any(|candidate| candidate == directional_action) + { + return Some(directional_action); + } + let (required_orientation, relative_action) = match direction { + ScrollDirection::Up => ("AXVerticalOrientation", "AXDecrementPage"), + ScrollDirection::Down => ("AXVerticalOrientation", "AXIncrementPage"), + ScrollDirection::Left => ("AXHorizontalOrientation", "AXDecrementPage"), + ScrollDirection::Right => ("AXHorizontalOrientation", "AXIncrementPage"), + }; + (orientation == Some(required_orientation) + && actions.iter().any(|candidate| candidate == relative_action)) + .then_some(relative_action) +} + +pub(super) fn page_child_snapshot( + root: &RegisteredElementSnapshot, + elements: &[RegisteredElementSnapshot], + direction: ScrollDirection, +) -> Option { + let (orientation, subrole) = match direction { + ScrollDirection::Up => ("AXVerticalOrientation", "AXDecrementPage"), + ScrollDirection::Down => ("AXVerticalOrientation", "AXIncrementPage"), + ScrollDirection::Left => ("AXHorizontalOrientation", "AXDecrementPage"), + ScrollDirection::Right => ("AXHorizontalOrientation", "AXIncrementPage"), + }; + + let mut matches = elements.iter().filter(|candidate| { + candidate.subrole.as_deref() == Some(subrole) + && candidate.actions.iter().any(|action| action == AX_PRESS) + && candidate.owner == root.owner + && candidate.parent.as_ref().is_some_and(|parent| { + elements.iter().any(|scrollbar| { + scrollbar.element.same_identity(parent) + && scrollbar.role == "AXScrollBar" + && scrollbar.orientation.as_deref() == Some(orientation) + && is_descendant_or_same(scrollbar, root, elements) + }) + }) + }); + let only = matches.next()?.clone(); + matches.next().is_none().then_some(only) +} + +fn is_descendant_or_same( + candidate: &RegisteredElementSnapshot, + root: &RegisteredElementSnapshot, + elements: &[RegisteredElementSnapshot], +) -> bool { + if candidate.element.same_identity(&root.element) { + return true; + } + let mut current = candidate.parent.clone(); + for _ in 0..elements.len().min(64) { + let Some(parent) = current else { + return false; + }; + if parent.same_identity(&root.element) { + return true; + } + current = elements + .iter() + .find(|element| element.element.same_identity(&parent)) + .and_then(|element| element.parent.clone()); + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn direct_page_actions_are_exact_and_case_sensitive() { + assert_eq!( + direct_page_action(ScrollDirection::Down, &["AXPageDown".to_owned()], None,), + Some("AXPageDown") + ); + assert_eq!( + direct_page_action(ScrollDirection::Down, &["axpagedown".to_owned()], None,), + None + ); + assert_eq!( + direct_page_action( + ScrollDirection::Down, + &["AXIncrement".to_owned()], + Some("AXVerticalOrientation"), + ), + None + ); + assert_eq!( + direct_page_action( + ScrollDirection::Right, + &["AXIncrementPage".to_owned()], + Some("AXHorizontalOrientation"), + ), + Some("AXIncrementPage") + ); + assert_eq!( + direct_page_action( + ScrollDirection::Right, + &["AXIncrementPage".to_owned()], + Some("AXVerticalOrientation"), + ), + None + ); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/semantic.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/semantic.rs new file mode 100644 index 0000000000..013a66504b --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/semantic.rs @@ -0,0 +1,1150 @@ +use std::collections::BTreeMap; + +use async_trait::async_trait; +use core_foundation::base::{CFRelease, CFTypeRef}; +use cua_driver_core::api::{ + contracts::{MouseButton, Route, ScrollDirection, SelectionType, VerificationLevel}, + errors::{ErrorCode, ErrorPhase, NativeError}, + interaction::{InteractionScope, LeaseDecision, NativeEvidence}, + observation::ResolvedElement, + platform::{ClickSpec, ElementScrollSpec, NativeDispatch, ResolvedAction, SelectionSpec}, + settlement::SettlementSignal, +}; +use serde_json::Value; + +use crate::{ + ax::bindings::{ + self, copy_action_names_exact, copy_attr_value, copy_ax_windows, copy_children, + copy_element_attr, copy_string_attr, copy_string_attr_exact, is_attribute_settable, + kAXErrorSuccess, AXUIElementCreateApplication, AXUIElementRef, AxCfRange, + }, + driver::{ + observation::{RegisteredElementSnapshot, RetainedAxElement, RetainedCfValue}, + target::MacTargetState, + windows::MacWindowRegistry, + }, +}; + +use super::scroll::{direct_page_action, page_child_snapshot, PageRoute, AX_PRESS}; + +const AX_SHOW_MENU: &str = "AXShowMenu"; +const MAX_OWNER_DEPTH: usize = 64; + +#[derive(Clone)] +pub struct MacSemanticActions { + windows: MacWindowRegistry, +} + +/// Retained, side-effect-free semantic recipe produced before core consumes +/// the observation. Its internals stay platform-owned and cannot be rebuilt +/// from loose metadata at the dispatch boundary. +pub struct MacPreparedSemanticAction(PreparedKind); + +enum PreparedKind { + AxAction { + element: LiveAxElement, + action: String, + primitive: &'static str, + }, + PageScroll { + route: PageRoute, + direction: ScrollDirection, + pages: u16, + }, + SetValue { + element: LiveAxElement, + value: String, + }, + SelectText { + element: LiveAxElement, + range: AxCfRange, + expected_text: String, + }, +} + +impl MacSemanticActions { + pub fn new(windows: MacWindowRegistry) -> Self { + Self { windows } + } + + async fn prepare( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + action: &ResolvedAction, + ) -> Result { + let prepared = match action { + ResolvedAction::ElementClick { element, spec, .. } => { + preflight_scope(target, scope, element)?; + if spec.button == MouseButton::Right { + return Err(menu_recipe_unproven( + "explicit AXShowMenu/right-click publication is unavailable", + )); + } + let live = self.refetch_exact(target, element).await?; + let action = + click_action(&live.snapshot.role, live.snapshot.subrole.as_deref(), spec)?; + require_exact_action(&live.snapshot.actions, action)?; + PreparedKind::AxAction { + element: live, + action: action.to_owned(), + primitive: "macos_ax_press", + } + } + ResolvedAction::ElementScroll { element, spec } => { + preflight_scope(target, scope, element)?; + if spec.pages == 0 { + return Err(NativeError::invalid("scroll pages must be nonzero")); + } + PreparedKind::PageScroll { + route: self.prepare_scroll_route(target, element, spec).await?, + direction: spec.direction, + pages: spec.pages, + } + } + ResolvedAction::SetValue { element, value } => { + preflight_scope(target, scope, element)?; + let live = self.refetch_exact(target, element).await?; + if live.snapshot.value_settable != Some(true) + || live.snapshot.string_value.is_none() + { + return Err(NativeError::unsupported( + "macOS AXValue is not both settable and a CFString", + )); + } + PreparedKind::SetValue { + element: live, + value: value.clone(), + } + } + ResolvedAction::SelectText { element, selection } => { + preflight_scope(target, scope, element)?; + let live = self.refetch_exact(target, element).await?; + if live.snapshot.selected_text_range_settable != Some(true) + || live.snapshot.selected_text_range.is_none() + { + return Err(NativeError::unsupported( + "macOS element has no exact writable AXSelectedTextRange route; text-marker-only selection is refused", + )); + } + let document = live.snapshot.string_value.as_deref().ok_or_else(|| { + NativeError::unsupported( + "macOS text selection requires an exact CFString AXValue snapshot", + ) + })?; + let range = resolve_selection_range(document, selection)?; + let expected_text = match selection.selection_type { + SelectionType::Text => selection.text.clone(), + SelectionType::CursorBefore | SelectionType::CursorAfter => String::new(), + }; + PreparedKind::SelectText { + element: live, + range, + expected_text, + } + } + ResolvedAction::Secondary { element, action } => { + preflight_scope(target, scope, element)?; + if action == AX_SHOW_MENU { + return Err(menu_recipe_unproven( + "explicit AXShowMenu secondary action publication is unavailable", + )); + } + let live = self.refetch_exact(target, element).await?; + if menu_managed_role(&live.snapshot.role, live.snapshot.subrole.as_deref()) { + return Err(menu_recipe_unproven( + "secondary actions on menu-managed roles require proved menu provenance/suppression", + )); + } + require_exact_action(&live.snapshot.actions, action)?; + PreparedKind::AxAction { + element: live, + action: action.clone(), + primitive: "macos_ax_secondary_action", + } + } + ResolvedAction::DeltaScroll(_) => { + return Err(NativeError::unsupported( + "platform has no exact semantic delta-scroll route", + )) + } + ResolvedAction::TypeText { .. } => { + return Err(NativeError::unsupported( + "platform has no semantic insertion-preserving text route", + )) + } + ResolvedAction::PointClick { .. } + | ResolvedAction::Drag(_) + | ResolvedAction::PressKey { .. } => { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "non-semantic action reached macOS semantic prepare", + )) + } + }; + Ok(MacPreparedSemanticAction(prepared)) + } + + async fn prepare_scroll_route( + &self, + target: &mut MacTargetState, + element: &ResolvedElement, + scroll: &ElementScrollSpec, + ) -> Result { + let requested = self.refetch_exact(target, element).await?; + let snapshots = target.elements.registered_snapshots(); + let candidates = retained_ancestor_chain(&requested.snapshot, &snapshots)?; + let mut requested_live = Some(requested); + + for (index, candidate) in candidates.into_iter().enumerate() { + let live = if index == 0 { + requested_live + .take() + .expect("first scroll candidate owns the requested retained element") + } else { + self.refetch_registered_exact(target, candidate).await? + }; + if let Some(action) = direct_page_action( + scroll.direction, + &live.snapshot.actions, + live.snapshot.orientation.as_deref(), + ) { + return Ok(PageRoute::Direct { + element: live, + action, + }); + } + if is_scroll_container_role(&live.snapshot.role) { + if let Some(page_child) = + page_child_snapshot(&live.snapshot, &snapshots, scroll.direction) + { + let page_child = self.refetch_registered_exact(target, page_child).await?; + require_exact_action(&page_child.snapshot.actions, AX_PRESS)?; + return Ok(PageRoute::ScrollbarPageChild { + element: page_child, + action: AX_PRESS, + }); + } + } + } + Err(unsupported_scroll(scroll.direction)) + } + + async fn dispatch( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + action: MacPreparedSemanticAction, + ) -> Result { + match action.0 { + PreparedKind::AxAction { + element, + action, + primitive, + } => { + perform_exact(element.as_ptr(), &action)?; + target.signals.record(SettlementSignal::AxAction); + Ok(dispatch( + VerificationLevel::DispatchVerified, + primitive, + [("ax_action", Value::String(action))], + )) + } + PreparedKind::PageScroll { + route, + direction, + pages, + } => { + let (native, action, route_name) = match &route { + PageRoute::Direct { element, action } => { + (element.as_ptr(), *action, "nearest_direct_page_action") + } + PageRoute::ScrollbarPageChild { element, action } => ( + element.as_ptr(), + *action, + "nearest_container_scrollbar_page_child", + ), + }; + scope.native_evidence.fields.insert( + "semantic_scroll_requested_pages".to_owned(), + Value::from(pages), + ); + scope.native_evidence.fields.insert( + "semantic_scroll_completed_pages".to_owned(), + Value::from(0_u16), + ); + scope.native_evidence.fields.insert( + "semantic_scroll_direction".to_owned(), + Value::String(format!("{direction:?}").to_lowercase()), + ); + scope.native_evidence.fields.insert( + "semantic_scroll_route".to_owned(), + Value::String(route_name.to_owned()), + ); + let mut completed_pages = 0_u16; + for _ in 0..pages { + let result = unsafe { bindings::perform_action(native, action) }; + if result != kAXErrorSuccess { + return Err(NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "exact macOS AX page-scroll action failed after a partial dispatch", + ) + .with_detail("ax_error", result) + .with_detail("completed_pages", completed_pages) + .with_detail("requested_pages", pages) + .with_detail("route", route_name)); + } + completed_pages += 1; + scope.native_evidence.fields.insert( + "semantic_scroll_completed_pages".to_owned(), + Value::from(completed_pages), + ); + target.signals.record(SettlementSignal::AxAction); + } + Ok(dispatch( + VerificationLevel::DispatchVerified, + "macos_ax_page_scroll", + [ + ("route", Value::String(route_name.to_owned())), + ("completed_pages", Value::from(completed_pages)), + ], + )) + } + PreparedKind::SetValue { element, value } => { + let result = + unsafe { bindings::set_string_attr(element.as_ptr(), "AXValue", &value) }; + if result != kAXErrorSuccess { + return Err(ax_dispatch_error("AXValue write", result)); + } + let readback = unsafe { copy_string_attr_exact(element.as_ptr(), "AXValue") }; + target + .signals + .record(SettlementSignal::VerificationReadbackComplete); + let readback = readback.map_err(|error| { + ax_verification_error("typed CFString AXValue readback", error) + })?; + if !exact_typed_cfstring_readback_matches(readback.as_deref(), &value) { + return Err(verification_error( + "macOS typed CFString AXValue readback did not exactly match the requested string", + )); + } + Ok(dispatch( + VerificationLevel::EffectVerified, + "macos_ax_string_value", + [("exact_typed_readback", Value::Bool(true))], + )) + } + PreparedKind::SelectText { + element, + range, + expected_text, + } => { + let result = unsafe { + bindings::set_cf_range_attr(element.as_ptr(), "AXSelectedTextRange", range) + }; + if result != kAXErrorSuccess { + return Err(ax_dispatch_error("AXSelectedTextRange write", result)); + } + let range_readback = unsafe { + bindings::copy_cf_range_attr(element.as_ptr(), "AXSelectedTextRange") + }; + let selected_text = + unsafe { copy_string_attr_exact(element.as_ptr(), "AXSelectedText") }; + target + .signals + .record(SettlementSignal::VerificationReadbackComplete); + let range_readback = range_readback.map_err(|error| { + ax_verification_error("typed AXSelectedTextRange readback", error) + })?; + let selected_text = selected_text.map_err(|error| { + ax_verification_error("typed AXSelectedText readback", error) + })?; + if !exact_typed_selection_readback_matches( + range_readback, + selected_text.as_deref(), + range, + &expected_text, + ) { + return Err(verification_error( + "macOS typed text selection range/text readback did not exactly match the requested selection", + )); + } + Ok(dispatch( + VerificationLevel::EffectVerified, + "macos_ax_utf16_selection", + [("exact_typed_readback", Value::Bool(true))], + )) + } + } + } + + async fn refetch_exact( + &self, + target: &mut MacTargetState, + element: &ResolvedElement, + ) -> Result { + if target.invalidated() { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "macOS target invalidated before semantic element refetch", + )); + } + if target.window != element.window.stamp() { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "resolved element target stamp no longer matches the locked macOS target", + )); + } + let snapshot = target + .elements + .registered(&element.native, &element.element_id) + .ok_or_else(|| { + NativeError::stale( + ErrorCode::ElementStale, + "resolved element has no exact retained macOS AX identity", + ) + })?; + if snapshot.owner != element.owner + || element.role.as_deref() != Some(snapshot.role.as_str()) + || !same_actions(&snapshot.actions, &element.actions) + { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "resolved element metadata disagrees with its retained macOS AX snapshot", + )); + } + self.refetch_registered_exact(target, snapshot).await + } + + async fn refetch_registered_exact( + &self, + target: &mut MacTargetState, + snapshot: RegisteredElementSnapshot, + ) -> Result { + let (pid, owner_window_id) = if snapshot.owner == target.window { + let facts = self.windows.facts_for_stamp(&target.window).await?; + (facts.pid, facts.cg_window_id) + } else { + let facts = self + .windows + .facts_for_related_stamp(&snapshot.owner, &target.window) + .await?; + (facts.pid, facts.cg_window_id) + }; + if owner_window_id != snapshot.owner_window_id { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "retained AX element owner window no longer matches its exact observation owner", + )); + } + + let refetched = if let Some(expected_parent) = &snapshot.parent { + let current_parent = + unsafe { copy_element_attr(snapshot.element.as_ptr(), "AXParent") } + .map_err(|error| ax_stale_error("AXParent refetch", error))? + .ok_or_else(|| { + NativeError::stale( + ErrorCode::ElementStale, + "retained AX element lost its observed parent", + ) + })?; + let current_parent = unsafe { RetainedAxElement::from_owned(current_parent) }; + if !current_parent.same_identity(expected_parent) { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "retained AX element parent identity changed", + )); + } + exact_child_of(¤t_parent, &snapshot.element)? + } else { + let window = exact_ax_window(pid, owner_window_id)?; + if !window.same_identity(&snapshot.element) { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "retained root AX identity no longer matches the exact owner window", + )); + } + window + }; + if !refetched.same_identity(&snapshot.element) { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "CFEqual rejected the retained/refetched AX element identity", + )); + } + if current_owner_window_id(&refetched)? != owner_window_id { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "live AX parent chain no longer reaches the observed owner window", + )); + } + + let (live_role, live_role_proven) = + match unsafe { copy_string_attr_exact(refetched.as_ptr(), "AXRole") } { + Ok(Some(role)) => (Some(role), true), + _ => (None, false), + }; + let live_subrole = unsafe { copy_string_attr(refetched.as_ptr(), "AXSubrole") }; + let live_orientation = unsafe { copy_string_attr(refetched.as_ptr(), "AXOrientation") }; + let (live_value_proof, live_value_query_proven) = + match unsafe { copy_attr_value(refetched.as_ptr(), "AXValue") } { + Ok(Some(value)) => (Some(unsafe { RetainedCfValue::from_owned(value) }), true), + Ok(None) => (None, true), + Err(_) => (None, false), + }; + let live_value = live_value_proof + .as_ref() + .and_then(RetainedCfValue::as_string); + let live_value_settable = live_value_proof + .as_ref() + .and_then(|_| unsafe { is_attribute_settable(refetched.as_ptr(), "AXValue").ok() }); + let live_range = live_value.as_ref().and_then(|_| unsafe { + bindings::copy_cf_range_attr(refetched.as_ptr(), "AXSelectedTextRange") + .ok() + .flatten() + }); + let live_range_settable = live_range.as_ref().and_then(|_| unsafe { + is_attribute_settable(refetched.as_ptr(), "AXSelectedTextRange").ok() + }); + let (live_actions, live_actions_proven) = + match unsafe { copy_action_names_exact(refetched.as_ptr()) } { + Ok(actions) => (actions, true), + Err(_) => (Vec::new(), false), + }; + if !snapshot.role_proven + || !live_role_proven + || live_role.as_deref() != Some(snapshot.role.as_str()) + || live_subrole != snapshot.subrole + || live_orientation != snapshot.orientation + || !snapshot.value_query_proven + || !live_value_query_proven + || !same_values(&live_value_proof, &snapshot.value_proof) + || live_value != snapshot.string_value + || live_value_settable != snapshot.value_settable + || live_range != snapshot.selected_text_range + || live_range_settable != snapshot.selected_text_range_settable + || !snapshot.actions_proven + || !live_actions_proven + || !same_actions(&live_actions, &snapshot.actions) + { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "live macOS AX role/actions/value/settable/parent/selection proof changed", + )); + } + Ok(LiveAxElement { + native: refetched, + snapshot, + }) + } +} + +#[derive(Clone)] +pub(super) struct LiveAxElement { + native: RetainedAxElement, + pub(super) snapshot: RegisteredElementSnapshot, +} + +impl LiveAxElement { + pub(super) fn as_ptr(&self) -> AXUIElementRef { + self.native.as_ptr() + } +} + +fn preflight_scope( + target: &MacTargetState, + scope: &InteractionScope, + element: &ResolvedElement, +) -> Result<(), NativeError> { + if scope.route != Route::Semantic + || scope.owner != target.window + || scope.window.stamp() != target.window + || element.window.stamp() != target.window + || scope.leases.posture_witness != LeaseDecision::Acquired + { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "semantic AX action entered without the exact locked interaction scope", + )); + } + Ok(()) +} + +fn click_action( + role: &str, + subrole: Option<&str>, + click: &ClickSpec, +) -> Result<&'static str, NativeError> { + if click.click_count != 1 || !click.modifiers.is_empty() { + return Err(NativeError::unsupported( + "semantic macOS click supports exactly one click with no modifiers", + )); + } + match click.button { + MouseButton::Left if menu_managed_role(role, subrole) => Err(menu_recipe_unproven( + "AXPress on this menu-managed role requires proved menu provenance/suppression", + )), + MouseButton::Left => Ok(AX_PRESS), + MouseButton::Right => Err(menu_recipe_unproven( + "explicit AXShowMenu/right-click publication is unavailable", + )), + MouseButton::Middle => Err(NativeError::unsupported( + "semantic macOS click has no exact middle-button route", + )), + } +} + +fn menu_managed_role(role: &str, subrole: Option<&str>) -> bool { + matches!( + role, + "AXMenu" | "AXMenuItem" | "AXMenuBar" | "AXMenuBarItem" | "AXPopUpButton" | "AXMenuButton" + ) || matches!( + subrole, + Some( + "AXMenu" + | "AXMenuItem" + | "AXMenuBar" + | "AXMenuBarItem" + | "AXPopUpButton" + | "AXMenuButton" + ) + ) +} + +fn is_scroll_container_role(role: &str) -> bool { + matches!( + role, + "AXScrollArea" + | "AXWebArea" + | "AXList" + | "AXTable" + | "AXOutline" + | "AXBrowser" + | "AXTextArea" + ) +} + +fn menu_recipe_unproven(reason: &str) -> NativeError { + NativeError::unsupported(format!("recipe_unproven: {reason}")) + .with_detail("recipe_status", "recipe_unproven") +} + +fn require_exact_action(actions: &[String], action: &str) -> Result<(), NativeError> { + if actions.iter().any(|candidate| candidate == action) { + Ok(()) + } else { + Err(NativeError::unsupported(format!( + "live macOS AX element does not expose exact action {action}" + ))) + } +} + +fn perform_exact(element: AXUIElementRef, action: &str) -> Result<(), NativeError> { + let result = unsafe { bindings::perform_action(element, action) }; + if result == kAXErrorSuccess { + Ok(()) + } else { + Err(ax_dispatch_error("exact AX action", result)) + } +} + +fn resolve_selection_range( + document: &str, + selection: &SelectionSpec, +) -> Result { + let matches: Vec<_> = document + .char_indices() + .filter_map(|(start, _)| { + let tail = &document[start..]; + if !tail.starts_with(&selection.text) { + return None; + } + let end = start + selection.text.len(); + let prefix_matches = selection + .prefix + .as_ref() + .is_none_or(|prefix| document[..start].ends_with(prefix)); + let suffix_matches = selection + .suffix + .as_ref() + .is_none_or(|suffix| document[end..].starts_with(suffix)); + (prefix_matches && suffix_matches).then_some((start, end)) + }) + .collect(); + let (start, end) = match matches.as_slice() { + [] => { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "requested text/context does not occur in the exact AXValue snapshot", + )) + } + [only] => *only, + _ => { + return Err(NativeError::new( + ErrorCode::InvalidRequest, + ErrorPhase::Preflight, + false, + "requested text/context is ambiguous in the exact AXValue snapshot", + )) + } + }; + let utf16_start = document[..start].encode_utf16().count(); + let utf16_end = document[..end].encode_utf16().count(); + let (location, length) = match selection.selection_type { + SelectionType::Text => (utf16_start, utf16_end - utf16_start), + SelectionType::CursorBefore => (utf16_start, 0), + SelectionType::CursorAfter => (utf16_end, 0), + }; + AxCfRange::from_utf16(location, length).ok_or_else(|| { + NativeError::unsupported("requested UTF-16 text range exceeds macOS CFRange limits") + }) +} + +fn retained_ancestor_chain( + requested: &RegisteredElementSnapshot, + elements: &[RegisteredElementSnapshot], +) -> Result, NativeError> { + let mut chain = vec![requested.clone()]; + let mut parent = requested.parent.clone(); + for _ in 0..MAX_OWNER_DEPTH { + let Some(parent_identity) = parent else { + return Ok(chain); + }; + if chain + .iter() + .any(|candidate| candidate.element.same_identity(&parent_identity)) + { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "retained AX scroll ancestor chain contains an identity cycle", + )); + } + let mut matches = elements + .iter() + .filter(|candidate| candidate.element.same_identity(&parent_identity)); + let Some(next) = matches.next().cloned() else { + // The observed owner window may have an AXApplication parent that + // is intentionally outside the window-scoped registry. + return Ok(chain); + }; + if matches.next().is_some() || next.owner != requested.owner { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "retained AX scroll ancestor identity was not unique in the exact owner", + )); + } + parent = next.parent.clone(); + chain.push(next); + } + if parent.is_some() { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "retained AX scroll ancestor chain exceeded its bounded identity proof", + )); + } + Ok(chain) +} + +fn exact_child_of( + parent: &RetainedAxElement, + expected: &RetainedAxElement, +) -> Result { + let children: Vec<_> = unsafe { copy_children(parent.as_ptr()) } + .into_iter() + .map(|child| unsafe { RetainedAxElement::from_owned(child) }) + .collect(); + let mut matches = children + .iter() + .filter(|child| child.same_identity(expected)); + let matched = matches.next().cloned(); + if matches.next().is_some() { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "AX parent returned duplicate CFEqual child identities", + )); + } + matched.ok_or_else(|| { + NativeError::stale( + ErrorCode::ElementStale, + "retained AX element is no longer an exact child of its observed parent", + ) + }) +} + +fn exact_ax_window(pid: i32, window_id: u32) -> Result { + let application = unsafe { AXUIElementCreateApplication(pid) }; + if application.is_null() { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "macOS AX application root disappeared during exact refetch", + )); + } + let windows: Vec<_> = unsafe { copy_ax_windows(application) } + .into_iter() + .map(|window| unsafe { RetainedAxElement::from_owned(window) }) + .collect(); + unsafe { CFRelease(application as CFTypeRef) }; + let mut matches = windows.iter().filter(|window| { + (unsafe { bindings::ax_get_window_id(window.as_ptr()) }) == Some(window_id) + }); + let matched = matches.next().cloned(); + if matches.next().is_some() { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "macOS AX returned multiple windows for one exact WindowServer id", + )); + } + matched.ok_or_else(|| { + NativeError::stale( + ErrorCode::ElementStale, + "exact macOS AX owner window disappeared during refetch", + ) + }) +} + +fn current_owner_window_id(element: &RetainedAxElement) -> Result { + let mut current = element.clone(); + for _ in 0..MAX_OWNER_DEPTH { + if let Some(window_id) = unsafe { bindings::ax_get_window_id(current.as_ptr()) } { + return Ok(window_id); + } + let parent = unsafe { copy_element_attr(current.as_ptr(), "AXParent") } + .map_err(|error| ax_stale_error("AX owner parent-chain refetch", error))? + .ok_or_else(|| { + NativeError::stale( + ErrorCode::ElementStale, + "live AX parent chain ended before an exact owner window", + ) + })?; + let parent = unsafe { RetainedAxElement::from_owned(parent) }; + if parent.same_identity(¤t) { + return Err(NativeError::stale( + ErrorCode::ElementStale, + "live AX parent chain contains an identity cycle", + )); + } + current = parent; + } + Err(NativeError::stale( + ErrorCode::ElementStale, + "live AX parent chain exceeded the bounded owner-depth proof", + )) +} + +fn same_actions(left: &[String], right: &[String]) -> bool { + let mut left = left.to_vec(); + let mut right = right.to_vec(); + left.sort(); + left.dedup(); + right.sort(); + right.dedup(); + left == right +} + +fn same_values(left: &Option, right: &Option) -> bool { + match (left, right) { + (None, None) => true, + (Some(left), Some(right)) => left.same_value(right), + _ => false, + } +} + +/// Exact typed CFString comparison. Rust string equality is byte exact for +/// these decoded scalar values; this deliberately performs no Unicode +/// normalization or case folding. +fn exact_typed_cfstring_readback_matches(readback: Option<&str>, expected: &str) -> bool { + readback == Some(expected) +} + +/// Exact typed CFRange plus CFString comparison, with no normalization. +fn exact_typed_selection_readback_matches( + range_readback: Option, + text_readback: Option<&str>, + expected_range: AxCfRange, + expected_text: &str, +) -> bool { + range_readback == Some(expected_range) && text_readback == Some(expected_text) +} + +fn unsupported_scroll(direction: ScrollDirection) -> NativeError { + NativeError::unsupported(format!( + "macOS element exposes neither an exact {direction:?} page action nor one unique exact scrollbar page child" + )) +} + +fn dispatch( + verification: VerificationLevel, + primitive: &str, + fields: [(&str, Value); N], +) -> NativeDispatch { + let mut evidence = + BTreeMap::from([("primitive".to_owned(), Value::String(primitive.to_owned()))]); + evidence.extend( + fields + .into_iter() + .map(|(key, value)| (key.to_owned(), value)), + ); + NativeDispatch { + verification, + evidence: NativeEvidence { + fields: evidence, + interaction_scope: None, + }, + warnings: Vec::new(), + // Menu provenance comes only from the canonical menu lifecycle. An AX + // dispatch result alone never proves which menu appeared. + menu: None, + } +} + +fn ax_stale_error(operation: &str, error: i32) -> NativeError { + NativeError::stale( + ErrorCode::ElementStale, + format!("{operation} failed while proving exact macOS AX identity"), + ) + .with_detail("ax_error", error) +} + +fn ax_dispatch_error(operation: &str, error: i32) -> NativeError { + NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + format!("{operation} failed"), + ) + .with_detail("ax_error", error) +} + +fn ax_verification_error(operation: &str, error: i32) -> NativeError { + verification_error(format!("{operation} failed")).with_detail("ax_error", error) +} + +fn verification_error(message: impl Into) -> NativeError { + NativeError::new( + ErrorCode::VerificationFailed, + ErrorPhase::Verify, + true, + message, + ) +} + +#[async_trait] +impl cua_driver_core::api::platform::SemanticActionProvider for MacSemanticActions { + type PreparedAction = MacPreparedSemanticAction; + + async fn prepare( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + action: &ResolvedAction, + ) -> Result { + MacSemanticActions::prepare(self, target, scope, action).await + } + + async fn dispatch( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + action: Self::PreparedAction, + ) -> Result { + MacSemanticActions::dispatch(self, target, scope, action).await + } +} + +#[cfg(test)] +mod tests { + use super::*; + use cua_driver_core::api::{contracts::Modifier, platform::ClickSpec}; + + fn selection( + text: &str, + prefix: Option<&str>, + suffix: Option<&str>, + selection_type: SelectionType, + ) -> SelectionSpec { + SelectionSpec { + text: text.to_owned(), + prefix: prefix.map(str::to_owned), + suffix: suffix.map(str::to_owned), + selection_type, + } + } + + #[test] + fn semantic_click_shapes_and_menu_managed_roles_refuse_before_dispatch() { + assert_eq!( + click_action( + "AXButton", + None, + &ClickSpec { + button: MouseButton::Left, + click_count: 1, + modifiers: vec![], + }, + ) + .unwrap(), + AX_PRESS + ); + let right = click_action( + "AXButton", + None, + &ClickSpec { + button: MouseButton::Right, + click_count: 1, + modifiers: vec![], + }, + ) + .unwrap_err(); + assert_eq!(right.code, ErrorCode::UnsupportedInBackground); + assert_eq!(right.details["recipe_status"], "recipe_unproven"); + for (role, subrole) in [ + ("AXPopUpButton", None), + ("AXMenuButton", None), + ("AXMenu", None), + ("AXMenuItem", None), + ("AXMenuBar", None), + ("AXMenuBarItem", None), + ("AXButton", Some("AXMenuButton")), + ] { + let error = click_action( + role, + subrole, + &ClickSpec { + button: MouseButton::Left, + click_count: 1, + modifiers: vec![], + }, + ) + .unwrap_err(); + assert_eq!(error.details["recipe_status"], "recipe_unproven"); + } + for click in [ + ClickSpec { + button: MouseButton::Middle, + click_count: 1, + modifiers: vec![], + }, + ClickSpec { + button: MouseButton::Left, + click_count: 2, + modifiers: vec![], + }, + ClickSpec { + button: MouseButton::Left, + click_count: 1, + modifiers: vec![Modifier::Shift], + }, + ] { + assert_eq!( + click_action("AXButton", None, &click).unwrap_err().code, + ErrorCode::UnsupportedInBackground + ); + } + } + + #[test] + fn secondary_action_names_are_exact_and_case_sensitive() { + let actions = vec!["AXConfirm".to_owned()]; + assert!(require_exact_action(&actions, "AXConfirm").is_ok()); + assert_eq!( + require_exact_action(&actions, "axconfirm") + .unwrap_err() + .code, + ErrorCode::UnsupportedInBackground + ); + } + + #[test] + fn selection_requires_one_contextual_match_and_uses_utf16_offsets() { + let document = "😀 alpha target omega / target"; + let exact = selection( + "target", + Some("alpha "), + Some(" omega"), + SelectionType::Text, + ); + assert_eq!( + resolve_selection_range(document, &exact).unwrap(), + AxCfRange { + location: 9, + length: 6 + } + ); + let before = selection( + "target", + Some("alpha "), + Some(" omega"), + SelectionType::CursorBefore, + ); + assert_eq!( + resolve_selection_range(document, &before).unwrap(), + AxCfRange { + location: 9, + length: 0 + } + ); + let after = selection( + "target", + Some("alpha "), + Some(" omega"), + SelectionType::CursorAfter, + ); + assert_eq!( + resolve_selection_range(document, &after).unwrap(), + AxCfRange { + location: 15, + length: 0 + } + ); + assert_eq!( + resolve_selection_range( + document, + &selection("target", None, None, SelectionType::Text), + ) + .unwrap_err() + .code, + ErrorCode::InvalidRequest + ); + } + + #[test] + fn overlapping_matches_are_not_silently_collapsed() { + assert_eq!( + resolve_selection_range("aaa", &selection("aa", None, None, SelectionType::Text),) + .unwrap_err() + .code, + ErrorCode::InvalidRequest + ); + } + + #[test] + fn typed_readback_is_exact_not_unicode_normalized_or_range_only() { + assert!(exact_typed_cfstring_readback_matches(Some("é"), "é")); + assert!(!exact_typed_cfstring_readback_matches( + Some("e\u{301}"), + "é" + )); + let expected = AxCfRange { + location: 2, + length: 1, + }; + assert!(exact_typed_selection_readback_matches( + Some(expected), + Some("x"), + expected, + "x" + )); + assert!(!exact_typed_selection_readback_matches( + Some(expected), + Some("y"), + expected, + "x" + )); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/focus.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/focus.rs new file mode 100644 index 0000000000..7d2e5c6f61 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/focus.rs @@ -0,0 +1,512 @@ +//! Pure macOS interaction recipes and target-only SLPS make-key leases. + +use std::sync::{Arc, Mutex}; + +use cua_driver_core::api::{ + capabilities::Framework, + contracts::{ActionId, MouseButton, Route}, + errors::{ErrorCode, ErrorPhase, NativeError}, + interaction::ScopeRequirements, + platform::ResolvedAction, +}; + +use crate::input::slps_make_key::{self, SlpsMakeKeyState}; + +use super::target::{MacActiveBelief, MacFocusState}; + +const PROVEN_OS_VERSION: &str = "26.5.1"; +const PROVEN_ARCHITECTURE: &str = "arm64"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct HostRecipeContext { + pub os_version: String, + pub architecture: String, +} + +impl HostRecipeContext { + pub fn current() -> Self { + let os_version = std::process::Command::new("/usr/bin/sw_vers") + .arg("-productVersion") + .output() + .ok() + .filter(|output| output.status.success()) + .and_then(|output| String::from_utf8(output.stdout).ok()) + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) + .unwrap_or_else(|| "unknown".to_owned()); + let architecture = match std::env::consts::ARCH { + "aarch64" => "arm64", + value => value, + } + .to_owned(); + Self { + os_version, + architecture, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AccessibilityRecipe { + NotApplicable, + ChromiumPriorStatePreserving, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MenuSuppressionRecipe { + NotApplicable, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TargetBeliefRecipe { + NotApplicable, + ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct MacScopeRecipe { + pub accessibility: AccessibilityRecipe, + pub menu: MenuSuppressionRecipe, + pub target_belief: TargetBeliefRecipe, +} + +impl MacScopeRecipe { + pub fn evidence_name(self) -> &'static str { + match self.target_belief { + TargetBeliefRecipe::NotApplicable => "semantic_no_target_belief", + TargetBeliefRecipe::ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64 => { + "chromium_point_click_slps_make_key_macos_26_5_1_arm64" + } + } + } +} + +pub fn select_scope_recipe( + host: &HostRecipeContext, + framework: &Framework, + route: Route, + action: &ResolvedAction, + requirements: &ScopeRequirements, +) -> Result { + if requirements.menu_dismissal { + return Err(recipe_unproven( + host, + framework, + route, + action, + "exact per-pid and per-menu event-tap predicate is not proved", + )); + } + + let accessibility = if requirements.accessibility && *framework == Framework::Chromium { + AccessibilityRecipe::ChromiumPriorStatePreserving + } else { + AccessibilityRecipe::NotApplicable + }; + + let target_belief = match route { + Route::Semantic if !requirements.target_belief => TargetBeliefRecipe::NotApplicable, + Route::TargetedPointer + if requirements.target_belief + && host.os_version == PROVEN_OS_VERSION + && host.architecture == PROVEN_ARCHITECTURE + && *framework == Framework::Chromium + && is_exact_proven_point_click(action) => + { + TargetBeliefRecipe::ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64 + } + _ => { + return Err(recipe_unproven( + host, + framework, + route, + action, + "no exact target-belief recipe is proved for this host/framework/action cell", + )); + } + }; + + Ok(MacScopeRecipe { + accessibility, + menu: MenuSuppressionRecipe::NotApplicable, + target_belief, + }) +} + +fn is_exact_proven_point_click(action: &ResolvedAction) -> bool { + matches!( + action, + ResolvedAction::PointClick { spec, .. } + if spec.button == MouseButton::Left + && spec.click_count == 1 + && spec.modifiers.is_empty() + ) +} + +fn action_shape(action: &ResolvedAction) -> &'static str { + match action { + ResolvedAction::ElementClick { .. } => "element_click", + ResolvedAction::PointClick { .. } => "point_click", + ResolvedAction::Drag(_) => "drag", + ResolvedAction::ElementScroll { .. } => "element_scroll", + ResolvedAction::DeltaScroll(_) => "delta_scroll", + ResolvedAction::PressKey { .. } => "press_key", + ResolvedAction::TypeText { .. } => "type_text", + ResolvedAction::SetValue { .. } => "set_value", + ResolvedAction::SelectText { .. } => "select_text", + ResolvedAction::Secondary { .. } => "secondary", + } +} + +fn recipe_unproven( + host: &HostRecipeContext, + framework: &Framework, + route: Route, + action: &ResolvedAction, + reason: &'static str, +) -> NativeError { + NativeError::unsupported(format!("recipe_unproven: {reason}")) + .with_detail("recipe_status", "recipe_unproven") + .with_detail("os_version", host.os_version.clone()) + .with_detail("architecture", host.architecture.clone()) + .with_detail("framework", format!("{framework:?}")) + .with_detail("route", format!("{route:?}")) + .with_detail("action", action_shape(action)) +} + +pub(crate) trait TargetBeliefPoster: Send + Sync { + fn post(&self, pid: i32, window_id: u32, state: SlpsMakeKeyState) -> Result<(), NativeError>; +} + +#[derive(Default)] +pub(crate) struct SystemTargetBeliefPoster; + +impl TargetBeliefPoster for SystemTargetBeliefPoster { + fn post(&self, pid: i32, window_id: u32, state: SlpsMakeKeyState) -> Result<(), NativeError> { + slps_make_key::post_target_only(pid, window_id, &[state]).map_err(|error| { + NativeError::new( + ErrorCode::UnsupportedInBackground, + ErrorPhase::Preflight, + false, + format!("target-only SLPS make-key record failed: {error}"), + ) + .with_detail("pid", pid) + .with_detail("cg_window_id", window_id) + .with_detail("slps_record_state", state.evidence_name()) + }) + } +} + +pub(crate) struct TargetBeliefLease { + action_id: ActionId, + pid: i32, + window_id: u32, + state: Arc>, + poster: Arc, + released: bool, +} + +impl TargetBeliefLease { + pub fn acquire( + action_id: ActionId, + pid: i32, + window_id: u32, + state: Arc>, + ) -> Result { + Self::acquire_with( + action_id, + pid, + window_id, + state, + Arc::new(SystemTargetBeliefPoster), + ) + } + + fn acquire_with( + action_id: ActionId, + pid: i32, + window_id: u32, + state: Arc>, + poster: Arc, + ) -> Result { + { + let mut state_guard = state.lock().expect("macOS focus coordinator poisoned"); + if state_guard.shutdown { + return Err(NativeError::new( + ErrorCode::WindowIdentityChanged, + ErrorPhase::Preflight, + false, + "target focus coordinator is shut down", + )); + } + if let Some(active) = &state_guard.active_belief { + return Err(NativeError::new( + ErrorCode::TargetBusy, + ErrorPhase::Preflight, + true, + "target already owns an active belief lease", + ) + .with_detail("active_action_id", active.action_id.to_string())); + } + state_guard.active_belief = Some(MacActiveBelief { + action_id: action_id.clone(), + pid, + cg_window_id: window_id, + }); + } + + if let Err(error) = poster.post(pid, window_id, SlpsMakeKeyState::MakeKey) { + state + .lock() + .expect("macOS focus coordinator poisoned") + .active_belief = None; + return Err(error); + } + + Ok(Self { + action_id, + pid, + window_id, + state, + poster, + released: false, + }) + } + + pub fn release(&mut self) -> Result<(), NativeError> { + if self.released { + return Ok(()); + } + self.poster + .post(self.pid, self.window_id, SlpsMakeKeyState::RemoveKey) + .map_err(|error| { + NativeError::new( + error.code, + ErrorPhase::Verify, + error.retryable, + error.message, + ) + .with_detail("action_id", self.action_id.to_string()) + .with_detail("pid", self.pid) + .with_detail("cg_window_id", self.window_id) + })?; + self.state + .lock() + .expect("macOS focus coordinator poisoned") + .active_belief = None; + self.released = true; + Ok(()) + } +} + +impl Drop for TargetBeliefLease { + fn drop(&mut self) { + if let Err(error) = self.release() { + tracing::error!(error = %error, "failed to post target-only SLPS remove-key record"); + } + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use cua_driver_core::api::{ + contracts::{CaptureRevision, GeometryRevision, Point, Rect, SurfaceId, WindowGeneration}, + observation::{ + NativeProcessHandle, NativeWindowHandle, ResolvedPoint, ResolvedWindow, SurfaceOwner, + WindowGeometry, + }, + platform::ClickSpec, + }; + + use super::*; + + struct FakePoster { + signals: Mutex>, + fail: Option, + } + + impl TargetBeliefPoster for FakePoster { + fn post( + &self, + _pid: i32, + _window_id: u32, + signal: SlpsMakeKeyState, + ) -> Result<(), NativeError> { + self.signals.lock().unwrap().push(signal); + if self.fail == Some(signal) { + Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "injected SLPS failure", + )) + } else { + Ok(()) + } + } + } + + fn point_click(button: MouseButton, count: u8) -> ResolvedAction { + let app = cua_driver_core::api::contracts::AppRef { + id: cua_driver_core::api::contracts::AppId::parse("app").unwrap(), + name: None, + pid: Some(1), + running: true, + }; + let public = cua_driver_core::api::contracts::WindowRef { + id: cua_driver_core::api::contracts::WindowId::parse("window").unwrap(), + app, + title: None, + }; + let window = ResolvedWindow { + public, + native: NativeWindowHandle::new("native").unwrap(), + process: NativeProcessHandle::new("process").unwrap(), + framework: Framework::Chromium, + geometry: WindowGeometry { + bounds: Rect { + x: 0.0, + y: 0.0, + width: 10.0, + height: 10.0, + }, + scale_factor: 1.0, + revision: GeometryRevision::parse("geometry").unwrap(), + }, + generation: WindowGeneration(1), + state: cua_driver_core::api::capabilities::WindowStateKind::Visible, + }; + ResolvedAction::PointClick { + point: ResolvedPoint { + window: window.clone(), + surface_id: SurfaceId::parse("surface").unwrap(), + surface_owner: SurfaceOwner::Target(window.stamp()), + capture_revision: CaptureRevision::parse("capture").unwrap(), + surface_point: Point { x: 1.0, y: 1.0 }, + window_point: Point { x: 1.0, y: 1.0 }, + screen_point: Point { x: 1.0, y: 1.0 }, + geometry_revision: GeometryRevision::parse("geometry").unwrap(), + }, + spec: ClickSpec { + button, + click_count: count, + modifiers: Vec::new(), + }, + } + } + + #[test] + fn recipe_selector_accepts_only_the_proven_pointer_cell() { + let host = HostRecipeContext { + os_version: PROVEN_OS_VERSION.to_owned(), + architecture: PROVEN_ARCHITECTURE.to_owned(), + }; + let requirements = ScopeRequirements::for_route(Route::TargetedPointer); + let recipe = select_scope_recipe( + &host, + &Framework::Chromium, + Route::TargetedPointer, + &point_click(MouseButton::Left, 1), + &requirements, + ) + .unwrap(); + assert_eq!( + recipe.target_belief, + TargetBeliefRecipe::ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64 + ); + + for (host, framework, action) in [ + ( + HostRecipeContext { + os_version: "26.5.2".to_owned(), + architecture: "arm64".to_owned(), + }, + Framework::Chromium, + point_click(MouseButton::Left, 1), + ), + ( + HostRecipeContext { + os_version: PROVEN_OS_VERSION.to_owned(), + architecture: "x86_64".to_owned(), + }, + Framework::Chromium, + point_click(MouseButton::Left, 1), + ), + ( + host.clone(), + Framework::Electron, + point_click(MouseButton::Left, 1), + ), + ( + host.clone(), + Framework::Chromium, + point_click(MouseButton::Right, 1), + ), + ] { + let error = select_scope_recipe( + &host, + &framework, + Route::TargetedPointer, + &action, + &requirements, + ) + .unwrap_err(); + assert_eq!(error.code, ErrorCode::UnsupportedInBackground); + assert_eq!(error.details["recipe_status"], "recipe_unproven"); + } + } + + #[test] + fn target_belief_grant_and_revoke_are_target_only_and_stateful() { + let state = Arc::new(Mutex::new(MacFocusState::default())); + let poster = Arc::new(FakePoster { + signals: Mutex::new(Vec::new()), + fail: None, + }); + let mut lease = TargetBeliefLease::acquire_with( + ActionId::parse("action").unwrap(), + 44, + 99, + Arc::clone(&state), + poster.clone(), + ) + .unwrap(); + assert_eq!( + poster.signals.lock().unwrap().as_slice(), + [SlpsMakeKeyState::MakeKey] + ); + assert!(state.lock().unwrap().active_belief.is_some()); + lease.release().unwrap(); + assert_eq!( + poster.signals.lock().unwrap().as_slice(), + [SlpsMakeKeyState::MakeKey, SlpsMakeKeyState::RemoveKey] + ); + assert!(state.lock().unwrap().active_belief.is_none()); + } + + #[test] + fn failed_make_key_record_clears_controller_ownership_without_claiming_a_grant() { + let state = Arc::new(Mutex::new(MacFocusState::default())); + let poster = Arc::new(FakePoster { + signals: Mutex::new(Vec::new()), + fail: Some(SlpsMakeKeyState::MakeKey), + }); + let error = TargetBeliefLease::acquire_with( + ActionId::parse("action").unwrap(), + 44, + 99, + Arc::clone(&state), + poster.clone(), + ) + .err() + .unwrap(); + assert_eq!(error.phase, ErrorPhase::Preflight); + assert_eq!( + poster.signals.lock().unwrap().as_slice(), + [SlpsMakeKeyState::MakeKey] + ); + assert!(state.lock().unwrap().active_belief.is_none()); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/interaction.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/interaction.rs new file mode 100644 index 0000000000..e6555e3777 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/interaction.rs @@ -0,0 +1,1123 @@ +//! Complete per-mutation macOS interaction scope acquisition and teardown. + +use std::{ + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, + time::{Duration, Instant}, +}; + +use async_trait::async_trait; +use cua_driver_core::api::{ + contracts::{ActionId, Route}, + errors::{ErrorCode, ErrorPhase, NativeError}, + interaction::{ + InteractionScope, LeaseDecision, LeaseTeardownStatus, MutationDeadline, NativeEvidence, + PostureResult, ScopeCleanup, ScopeLeaseAcquisition, ScopeLeaseTeardown, ScopePlan, + ScopeRequirements, ScopeTeardownOutcome, TargetCursorHandle, + }, + observation::ResolvedWindow, + platform::{InteractionProvider, ResolvedAction}, +}; + +use crate::{ + apps::nsworkspace::WorkspaceEventHub, + ax::enablement::AxEnablementLease, + focus_steal::{SuppressionLease, SuppressionOutcome}, + input::slps_make_key, +}; + +use super::{ + focus::{ + select_scope_recipe, AccessibilityRecipe, HostRecipeContext, MacScopeRecipe, + MenuSuppressionRecipe, TargetBeliefLease, TargetBeliefRecipe, + }, + menu::{self, MenuSuppressionPlan}, + posture::MacInteractionPostureWitness, + target::{MacFocusState, MacTargetFocusCoordinator, MacTargetState}, + windows::{MacWindowFacts, MacWindowRegistry}, +}; + +const CONTAINMENT_BARRIER_TIMEOUT: Duration = Duration::from_millis(250); + +#[derive(Debug)] +pub struct MacNativeScopePlan { + facts: MacWindowFacts, + host: HostRecipeContext, + recipe: MacScopeRecipe, + menu: MenuSuppressionPlan, +} + +trait LeaseResource: Send { + fn release(&mut self, deadline: Instant) -> LeaseRelease; +} + +#[derive(Default)] +struct LeaseRelease { + evidence: NativeEvidence, + failure: Option, +} + +trait PostureResource: Send { + fn finish(&mut self, deadline: Instant) + -> Result<(PostureResult, NativeEvidence), NativeError>; +} + +struct PostureAcquisition { + prior_frontmost_pid: i32, + resource: Box, +} + +trait MacInteractionHooks: Send + Sync { + fn acquire_posture(&self) -> Result; + + fn acquire_accessibility( + &self, + recipe: AccessibilityRecipe, + pid: i32, + ) -> Result>, NativeError>; + + fn acquire_containment( + &self, + required: bool, + target_pid: i32, + prior_frontmost_pid: i32, + deadline: Instant, + ) -> Result>, NativeError>; + + fn acquire_menu( + &self, + recipe: MenuSuppressionRecipe, + plan: &MenuSuppressionPlan, + ) -> Result>, NativeError>; + + fn acquire_target_belief( + &self, + recipe: TargetBeliefRecipe, + action_id: &ActionId, + pid: i32, + cg_window_id: u32, + focus_state: Arc>, + ) -> Result>, NativeError>; +} + +#[derive(Default)] +struct SystemInteractionHooks; + +impl MacInteractionHooks for SystemInteractionHooks { + fn acquire_posture(&self) -> Result { + let witness = MacInteractionPostureWitness::begin()?; + let prior_frontmost_pid = witness.prior_frontmost_pid(); + Ok(PostureAcquisition { + prior_frontmost_pid, + resource: Box::new(SystemPostureResource(witness)), + }) + } + + fn acquire_accessibility( + &self, + recipe: AccessibilityRecipe, + pid: i32, + ) -> Result>, NativeError> { + match recipe { + AccessibilityRecipe::NotApplicable => Ok(None), + AccessibilityRecipe::ChromiumPriorStatePreserving => { + let lease = AxEnablementLease::acquire(pid)?; + Ok(Some(Box::new(AxLeaseResource(lease)))) + } + } + } + + fn acquire_containment( + &self, + required: bool, + target_pid: i32, + prior_frontmost_pid: i32, + deadline: Instant, + ) -> Result>, NativeError> { + if !required || target_pid == prior_frontmost_pid { + return Ok(None); + } + Ok(Some(Box::new(ContainmentLeaseResource(Some( + crate::focus_steal::begin_suppression_until( + Some(target_pid), + prior_frontmost_pid, + "driver.v2.interaction", + deadline, + ), + ))))) + } + + fn acquire_menu( + &self, + _recipe: MenuSuppressionRecipe, + plan: &MenuSuppressionPlan, + ) -> Result>, NativeError> { + menu::acquire_production(plan).map(|resource| { + resource + .map(|resource| Box::new(MenuResourceAdapter(resource)) as Box) + }) + } + + fn acquire_target_belief( + &self, + recipe: TargetBeliefRecipe, + action_id: &ActionId, + pid: i32, + cg_window_id: u32, + focus_state: Arc>, + ) -> Result>, NativeError> { + match recipe { + TargetBeliefRecipe::NotApplicable => Ok(None), + TargetBeliefRecipe::ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64 => { + Ok(Some(Box::new(BeliefLeaseResource( + TargetBeliefLease::acquire(action_id.clone(), pid, cg_window_id, focus_state)?, + )))) + } + } + } +} + +struct SystemPostureResource(MacInteractionPostureWitness); + +impl PostureResource for SystemPostureResource { + fn finish( + &mut self, + deadline: Instant, + ) -> Result<(PostureResult, NativeEvidence), NativeError> { + Ok(self.0.finish(deadline)) + } +} + +struct AxLeaseResource(AxEnablementLease); + +impl LeaseResource for AxLeaseResource { + fn release(&mut self, _deadline: Instant) -> LeaseRelease { + let mut evidence = NativeEvidence::default(); + evidence + .fields + .insert("ax_attribute".to_owned(), self.0.attribute().into()); + evidence + .fields + .insert("ax_prior_value".to_owned(), self.0.prior().into()); + evidence + .fields + .insert("ax_state_changed".to_owned(), self.0.changed().into()); + let failure = self.0.release().err(); + evidence + .fields + .insert("ax_release_succeeded".to_owned(), failure.is_none().into()); + LeaseRelease { evidence, failure } + } +} + +struct ContainmentLeaseResource(Option); + +impl LeaseResource for ContainmentLeaseResource { + fn release(&mut self, deadline: Instant) -> LeaseRelease { + let close = self + .0 + .take() + .map(|lease| lease.close_with_evidence(remaining(deadline))) + .unwrap_or_default(); + let workspace_drained = WorkspaceEventHub::shared().barrier(remaining(deadline)); + containment_evidence( + close.evidence, + close.callback_queue_drained, + workspace_drained, + ) + } +} + +fn containment_evidence( + outcome: SuppressionOutcome, + containment_drained: bool, + workspace_drained: bool, +) -> LeaseRelease { + let mut evidence = NativeEvidence::default(); + evidence.fields.insert( + "containment_activations".to_owned(), + outcome.activations.into(), + ); + evidence.fields.insert( + "containment_restore_attempts".to_owned(), + outcome.restore_attempts.into(), + ); + evidence.fields.insert( + "containment_restore_failures".to_owned(), + outcome.restore_failures.into(), + ); + evidence.fields.insert( + "containment_callback_queue_drained".to_owned(), + containment_drained.into(), + ); + evidence.fields.insert( + "workspace_callback_queue_drained".to_owned(), + workspace_drained.into(), + ); + let mut failures = Vec::new(); + if !workspace_drained || !containment_drained { + failures.push( + NativeError::new( + ErrorCode::PostureUnverifiable, + ErrorPhase::Verify, + true, + "focus-containment callback queues did not drain before release", + ) + .with_detail("workspace_queue_drained", workspace_drained) + .with_detail("containment_queue_drained", containment_drained), + ); + } + if outcome.activations > 0 || outcome.restore_failures > 0 { + failures.push( + NativeError::new( + ErrorCode::PostureViolated, + ErrorPhase::Verify, + false, + "focus containment observed a foreground activation excursion", + ) + .with_detail("activations", outcome.activations) + .with_detail("restore_failures", outcome.restore_failures), + ); + } + LeaseRelease { + evidence, + failure: NativeError::primary(failures), + } +} + +struct MenuResourceAdapter(Box); + +impl LeaseResource for MenuResourceAdapter { + fn release(&mut self, _deadline: Instant) -> LeaseRelease { + match self.0.release() { + Ok(evidence) => LeaseRelease { + evidence, + failure: None, + }, + Err(error) => LeaseRelease { + evidence: NativeEvidence::default(), + failure: Some(error), + }, + } + } +} + +struct BeliefLeaseResource(TargetBeliefLease); + +impl LeaseResource for BeliefLeaseResource { + fn release(&mut self, _deadline: Instant) -> LeaseRelease { + let mut evidence = NativeEvidence::default(); + let failure = self.0.release().err(); + evidence + .fields + .insert("target_belief_release_attempted".to_owned(), true.into()); + evidence + .fields + .insert("target_belief_revoked".to_owned(), failure.is_none().into()); + LeaseRelease { evidence, failure } + } +} + +fn remaining(deadline: Instant) -> Duration { + deadline + .saturating_duration_since(Instant::now()) + .min(CONTAINMENT_BARRIER_TIMEOUT) +} + +#[derive(Clone)] +pub struct MacInteractionProvider { + windows: MacWindowRegistry, + host: HostRecipeContext, + hooks: Arc, +} + +impl MacInteractionProvider { + pub fn new(windows: MacWindowRegistry) -> Self { + Self { + windows, + host: HostRecipeContext::current(), + hooks: Arc::new(SystemInteractionHooks), + } + } +} + +#[async_trait] +impl InteractionProvider for MacInteractionProvider { + type NativeScopePlan = MacNativeScopePlan; + + async fn preflight( + &self, + target: &mut MacTargetState, + focus: &mut MacTargetFocusCoordinator, + action_id: &ActionId, + window: &ResolvedWindow, + route: Route, + action: &ResolvedAction, + deadline: MutationDeadline, + requirements: ScopeRequirements, + ) -> Result, NativeError> { + ensure_target_live(target, focus, window)?; + let facts = self.windows.facts_for_stamp(&window.stamp()).await?; + ensure_native_facts_match(&facts, target, window)?; + let recipe = + select_scope_recipe(&self.host, &window.framework, route, action, &requirements)?; + if recipe.target_belief != TargetBeliefRecipe::NotApplicable && !slps_make_key::available() + { + return Err(NativeError::unsupported( + "recipe_unproven: target-only SLPS make-key symbols are unavailable on this host", + ) + .with_detail("recipe_status", "recipe_unproven") + .with_detail("os_version", self.host.os_version.clone()) + .with_detail("architecture", self.host.architecture.clone())); + } + let menu = if requirements.menu_dismissal { + return Err(NativeError::unsupported( + "recipe_unproven: required menu suppression has no exact predicate", + ) + .with_detail("recipe_status", "recipe_unproven")); + } else { + MenuSuppressionPlan::NotApplicable + }; + Ok(ScopePlan::new( + action_id.clone(), + window.clone(), + route, + deadline, + requirements, + MacNativeScopePlan { + facts, + host: self.host.clone(), + recipe, + menu, + }, + )) + } + + async fn acquire_scope( + &self, + target: &mut MacTargetState, + focus: &mut MacTargetFocusCoordinator, + plan: ScopePlan, + logical_cursor: TargetCursorHandle, + ) -> Result { + ensure_target_live(target, focus, &plan.window)?; + let live_facts = self.windows.facts_for_stamp(&plan.window.stamp()).await?; + if live_facts != plan.native.facts { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "native window facts changed between interaction preflight and lease acquisition", + )); + } + ensure_native_facts_match(&live_facts, target, &plan.window)?; + + let acquired = acquire_resources( + self.hooks.as_ref(), + &plan, + target.poison_handle(), + focus.state_handle(), + )?; + Ok(plan.into_scope( + acquired.acquisition, + logical_cursor, + acquired.evidence, + Box::new(acquired.cleanup), + )) + } +} + +fn ensure_target_live( + target: &MacTargetState, + focus: &MacTargetFocusCoordinator, + window: &ResolvedWindow, +) -> Result<(), NativeError> { + if target.invalidated() || focus.is_shutdown() || target.window != window.stamp() { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "interaction target no longer matches the live target controller", + )); + } + Ok(()) +} + +fn ensure_native_facts_match( + facts: &MacWindowFacts, + target: &MacTargetState, + window: &ResolvedWindow, +) -> Result<(), NativeError> { + if facts.stamp != target.window || facts.stamp != window.stamp() { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "revalidated native facts do not match the exact target owner", + )); + } + Ok(()) +} + +struct AcquiredResources { + acquisition: ScopeLeaseAcquisition, + evidence: NativeEvidence, + cleanup: MacScopeCleanup, +} + +fn acquire_resources( + hooks: &dyn MacInteractionHooks, + plan: &ScopePlan, + poison: Arc, + focus_state: Arc>, +) -> Result { + let posture = hooks.acquire_posture()?; + let mut cleanup = MacScopeCleanup::new(poison, posture.resource, plan.deadline.teardown); + + let result = (|| { + cleanup.accessibility = + hooks.acquire_accessibility(plan.native.recipe.accessibility, plan.native.facts.pid)?; + cleanup.containment = hooks.acquire_containment( + plan.requirements.containment, + plan.native.facts.pid, + posture.prior_frontmost_pid, + plan.deadline.teardown, + )?; + cleanup.menu = hooks.acquire_menu(plan.native.recipe.menu, &plan.native.menu)?; + cleanup.target_belief = hooks.acquire_target_belief( + plan.native.recipe.target_belief, + &plan.action_id, + plan.native.facts.pid, + plan.native.facts.cg_window_id, + Arc::clone(&focus_state), + )?; + Ok::<(), NativeError>(()) + })(); + + if let Err(primary) = result { + let outcome = cleanup.cleanup(plan.deadline.teardown); + let belief_still_active = focus_state + .lock() + .expect("macOS focus coordinator poisoned") + .active_belief + .is_some(); + if belief_still_active { + cleanup.poison.store(true, Ordering::Release); + } + let mut failures = Vec::with_capacity(1 + outcome.failures.len()); + failures.push(primary); + failures.extend(outcome.failures); + let mut combined = NativeError::primary(failures).expect("acquisition failure is nonempty"); + if belief_still_active { + combined = combined.with_detail("target_poisoned", true); + } + return Err(combined); + } + + let acquisition = ScopeLeaseAcquisition { + posture_witness: LeaseDecision::Acquired, + accessibility: decision(&cleanup.accessibility), + containment: decision(&cleanup.containment), + menu_dismissal: decision(&cleanup.menu), + target_belief: decision(&cleanup.target_belief), + }; + let mut evidence = NativeEvidence::default(); + evidence.fields.insert( + "interaction_recipe".to_owned(), + plan.native.recipe.evidence_name().into(), + ); + evidence.fields.insert( + "os_version".to_owned(), + plan.native.host.os_version.clone().into(), + ); + evidence.fields.insert( + "architecture".to_owned(), + plan.native.host.architecture.clone().into(), + ); + evidence + .fields + .insert("pid".to_owned(), plan.native.facts.pid.into()); + evidence.fields.insert( + "cg_window_id".to_owned(), + plan.native.facts.cg_window_id.into(), + ); + Ok(AcquiredResources { + acquisition, + evidence, + cleanup, + }) +} + +fn decision(resource: &Option>) -> LeaseDecision { + if resource.is_some() { + LeaseDecision::Acquired + } else { + LeaseDecision::NotApplicable + } +} + +struct MacScopeCleanup { + poison: Arc, + teardown_deadline: Instant, + posture: Option>, + accessibility: Option>, + containment: Option>, + menu: Option>, + target_belief: Option>, + outcome: Option, +} + +impl MacScopeCleanup { + fn new( + poison: Arc, + posture: Box, + teardown_deadline: Instant, + ) -> Self { + Self { + poison, + teardown_deadline, + posture: Some(posture), + accessibility: None, + containment: None, + menu: None, + target_belief: None, + outcome: None, + } + } +} + +impl ScopeCleanup for MacScopeCleanup { + fn cleanup(&mut self, deadline: Instant) -> ScopeTeardownOutcome { + if let Some(outcome) = &self.outcome { + return outcome.clone(); + } + + let mut evidence = NativeEvidence::default(); + let mut failures = Vec::new(); + let target_belief = release_lease( + &mut self.target_belief, + deadline, + &mut evidence, + &mut failures, + ); + let menu_dismissal = release_lease(&mut self.menu, deadline, &mut evidence, &mut failures); + let containment = release_lease( + &mut self.containment, + deadline, + &mut evidence, + &mut failures, + ); + let accessibility = release_lease( + &mut self.accessibility, + deadline, + &mut evidence, + &mut failures, + ); + let (posture_witness, posture) = match self.posture.as_mut() { + None => (LeaseTeardownStatus::NotApplicable, PostureResult::default()), + Some(resource) => match resource.finish(deadline) { + Ok((posture, posture_evidence)) => { + evidence.merge(posture_evidence); + let status = LeaseTeardownStatus::Released; + if let Some(error) = posture_failure(&posture) { + failures.push(error); + } + (status, posture) + } + Err(error) => { + failures.push(error); + ( + LeaseTeardownStatus::Failed, + PostureResult { + held: false, + ..PostureResult::default() + }, + ) + } + }, + }; + self.posture = None; + + if !failures.is_empty() { + self.poison.store(true, Ordering::Release); + evidence + .fields + .insert("target_poisoned".to_owned(), true.into()); + } + let outcome = ScopeTeardownOutcome { + posture, + native_evidence: evidence, + leases: ScopeLeaseTeardown { + posture_witness, + accessibility, + containment, + menu_dismissal, + target_belief, + }, + failures, + }; + self.outcome = Some(outcome.clone()); + outcome + } +} + +fn posture_failure(posture: &PostureResult) -> Option { + if posture.held { + return None; + } + let observed_excursion = posture.frontmost_changed + || posture.key_window_changed + || posture.physical_cursor_moved + || posture.restored_after_violation; + let (code, retryable, message) = if observed_excursion { + ( + ErrorCode::PostureViolated, + false, + "interaction changed foreground, focused window, or physical cursor posture", + ) + } else { + ( + ErrorCode::PostureUnverifiable, + true, + "interaction posture witness was incomplete or lagged", + ) + }; + Some( + NativeError::new(code, ErrorPhase::Verify, retryable, message) + .with_detail("posture", serde_json::to_value(posture).unwrap_or_default()), + ) +} + +impl Drop for MacScopeCleanup { + fn drop(&mut self) { + if self.outcome.is_some() { + return; + } + let outcome = self.cleanup(self.teardown_deadline); + for error in outcome.failures { + tracing::error!(error = %error, "macOS interaction cleanup failed during drop"); + } + } +} + +fn release_lease( + resource: &mut Option>, + deadline: Instant, + evidence: &mut NativeEvidence, + failures: &mut Vec, +) -> LeaseTeardownStatus { + let Some(mut resource) = resource.take() else { + return LeaseTeardownStatus::NotApplicable; + }; + let release = resource.release(deadline); + evidence.merge(release.evidence); + if let Some(error) = release.failure { + failures.push(error); + LeaseTeardownStatus::Failed + } else { + LeaseTeardownStatus::Released + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use cua_driver_core::api::{ + capabilities::{Framework, WindowStateKind}, + contracts::{AppId, AppRef, GeometryRevision, Rect, WindowGeneration, WindowId, WindowRef}, + observation::{NativeProcessHandle, NativeWindowHandle, WindowGeometry}, + }; + + use super::*; + + struct LoggedLease { + log: Arc>>, + release: &'static str, + fail: bool, + } + + impl LeaseResource for LoggedLease { + fn release(&mut self, _deadline: Instant) -> LeaseRelease { + self.log.lock().unwrap().push(self.release); + let mut evidence = NativeEvidence::default(); + evidence + .fields + .insert(format!("{}_attempted", self.release), true.into()); + let failure = self.fail.then(|| { + NativeError::new( + ErrorCode::Internal, + ErrorPhase::Verify, + false, + "injected release failure", + ) + }); + LeaseRelease { evidence, failure } + } + } + + struct LoggedPosture { + log: Arc>>, + posture: PostureResult, + } + + impl PostureResource for LoggedPosture { + fn finish( + &mut self, + _deadline: Instant, + ) -> Result<(PostureResult, NativeEvidence), NativeError> { + self.log.lock().unwrap().push("posture-"); + Ok((self.posture.clone(), NativeEvidence::default())) + } + } + + fn logged( + log: &Arc>>, + release: &'static str, + ) -> Box { + Box::new(LoggedLease { + log: Arc::clone(log), + release, + fail: false, + }) + } + + struct LoggingHooks { + log: Arc>>, + containment_deadlines: Arc>>, + fail_at: Option<&'static str>, + } + + impl LoggingHooks { + fn acquire( + &self, + acquire: &'static str, + release: &'static str, + ) -> Result>, NativeError> { + self.log.lock().unwrap().push(acquire); + if self.fail_at == Some(acquire) { + Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "injected acquisition failure", + )) + } else { + Ok(Some(logged(&self.log, release))) + } + } + } + + impl MacInteractionHooks for LoggingHooks { + fn acquire_posture(&self) -> Result { + self.log.lock().unwrap().push("posture+"); + Ok(PostureAcquisition { + prior_frontmost_pid: 7, + resource: Box::new(LoggedPosture { + log: Arc::clone(&self.log), + posture: PostureResult::default(), + }), + }) + } + + fn acquire_accessibility( + &self, + _recipe: AccessibilityRecipe, + _pid: i32, + ) -> Result>, NativeError> { + self.acquire("ax+", "ax-") + } + + fn acquire_containment( + &self, + _required: bool, + _target_pid: i32, + _prior_frontmost_pid: i32, + deadline: Instant, + ) -> Result>, NativeError> { + self.containment_deadlines.lock().unwrap().push(deadline); + self.acquire("containment+", "containment-") + } + + fn acquire_menu( + &self, + _recipe: MenuSuppressionRecipe, + _plan: &MenuSuppressionPlan, + ) -> Result>, NativeError> { + self.acquire("menu+", "menu-") + } + + fn acquire_target_belief( + &self, + _recipe: TargetBeliefRecipe, + _action_id: &ActionId, + _pid: i32, + _cg_window_id: u32, + _focus_state: Arc>, + ) -> Result>, NativeError> { + self.acquire("belief+", "belief-") + } + } + + fn scope_plan() -> ScopePlan { + let app = AppRef { + id: AppId::parse("app").unwrap(), + name: Some("Fixture".to_owned()), + pid: Some(44), + running: true, + }; + let public = WindowRef { + id: WindowId::parse("window").unwrap(), + app, + title: None, + }; + let window = ResolvedWindow { + public, + native: NativeWindowHandle::new("native-window").unwrap(), + process: NativeProcessHandle::new("native-process").unwrap(), + framework: Framework::Chromium, + geometry: WindowGeometry { + bounds: Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }, + scale_factor: 2.0, + revision: GeometryRevision::parse("geometry").unwrap(), + }, + generation: WindowGeneration(1), + state: WindowStateKind::Visible, + }; + let facts = MacWindowFacts { + stamp: window.stamp(), + pid: 44, + process_generation: 1, + cg_window_id: 99, + owner_name: "Fixture".to_owned(), + layer: 0, + bounds: window.geometry.bounds, + scale_factor: Some(2.0), + state: WindowStateKind::Visible, + is_on_screen: true, + on_current_space: Some(true), + space_ids: Some(vec![1]), + minimized: Some(false), + }; + ScopePlan::new( + ActionId::parse("action").unwrap(), + window, + Route::TargetedPointer, + test_deadline(), + ScopeRequirements::for_route(Route::TargetedPointer), + MacNativeScopePlan { + facts, + host: HostRecipeContext { + os_version: "26.5.1".to_owned(), + architecture: "arm64".to_owned(), + }, + recipe: MacScopeRecipe { + accessibility: AccessibilityRecipe::ChromiumPriorStatePreserving, + menu: MenuSuppressionRecipe::NotApplicable, + target_belief: TargetBeliefRecipe::ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64, + }, + menu: MenuSuppressionPlan::NotApplicable, + }, + ) + } + + fn test_deadline() -> MutationDeadline { + let work = Instant::now() + Duration::from_secs(30); + MutationDeadline::new(work, work + Duration::from_secs(1)).unwrap() + } + + #[test] + fn fake_hooks_prove_exact_acquisition_and_reverse_release_order() { + let log = Arc::new(Mutex::new(Vec::new())); + let hooks = LoggingHooks { + log: Arc::clone(&log), + containment_deadlines: Arc::new(Mutex::new(Vec::new())), + fail_at: None, + }; + let poison = Arc::new(AtomicBool::new(false)); + let plan = scope_plan(); + let mut acquired = acquire_resources( + &hooks, + &plan, + poison, + Arc::new(Mutex::new(MacFocusState::default())), + ) + .unwrap(); + assert_eq!( + hooks.containment_deadlines.lock().unwrap().as_slice(), + [plan.deadline.teardown] + ); + let outcome = acquired.cleanup.cleanup(plan.deadline.teardown); + assert!(outcome.failures.is_empty()); + assert_eq!( + *log.lock().unwrap(), + vec![ + "posture+", + "ax+", + "containment+", + "menu+", + "belief+", + "belief-", + "menu-", + "containment-", + "ax-", + "posture-", + ] + ); + } + + #[test] + fn acquisition_failure_unwinds_every_earlier_lease_in_reverse_order() { + for (fail_at, expected) in [ + ("ax+", vec!["posture+", "ax+", "posture-"]), + ( + "containment+", + vec!["posture+", "ax+", "containment+", "ax-", "posture-"], + ), + ( + "menu+", + vec![ + "posture+", + "ax+", + "containment+", + "menu+", + "containment-", + "ax-", + "posture-", + ], + ), + ( + "belief+", + vec![ + "posture+", + "ax+", + "containment+", + "menu+", + "belief+", + "menu-", + "containment-", + "ax-", + "posture-", + ], + ), + ] { + let log = Arc::new(Mutex::new(Vec::new())); + let hooks = LoggingHooks { + log: Arc::clone(&log), + containment_deadlines: Arc::new(Mutex::new(Vec::new())), + fail_at: Some(fail_at), + }; + let error = acquire_resources( + &hooks, + &scope_plan(), + Arc::new(AtomicBool::new(false)), + Arc::new(Mutex::new(MacFocusState::default())), + ) + .err() + .unwrap(); + assert_eq!(error.phase, ErrorPhase::Preflight); + assert_eq!(*log.lock().unwrap(), expected, "failure at {fail_at}"); + } + } + + #[test] + fn cleanup_is_reverse_order_and_restored_violation_poison_is_sticky() { + let log = Arc::new(Mutex::new(Vec::new())); + let poison = Arc::new(AtomicBool::new(false)); + let mut cleanup = MacScopeCleanup::new( + Arc::clone(&poison), + Box::new(LoggedPosture { + log: Arc::clone(&log), + posture: PostureResult { + held: false, + frontmost_changed: true, + key_window_changed: false, + physical_cursor_moved: false, + restored_after_violation: true, + }, + }), + test_deadline().teardown, + ); + cleanup.accessibility = Some(logged(&log, "ax-")); + cleanup.containment = Some(logged(&log, "containment-")); + cleanup.menu = Some(logged(&log, "menu-")); + cleanup.target_belief = Some(logged(&log, "belief-")); + + let outcome = cleanup.cleanup(test_deadline().teardown); + assert_eq!( + *log.lock().unwrap(), + vec!["belief-", "menu-", "containment-", "ax-", "posture-"] + ); + assert!(outcome.posture.restored_after_violation); + assert_eq!(outcome.failures[0].code, ErrorCode::PostureViolated); + assert!(poison.load(Ordering::Acquire)); + } + + #[test] + fn every_release_is_attempted_after_an_earlier_cleanup_failure() { + let log = Arc::new(Mutex::new(Vec::new())); + let poison = Arc::new(AtomicBool::new(false)); + let mut cleanup = MacScopeCleanup::new( + Arc::clone(&poison), + Box::new(LoggedPosture { + log: Arc::clone(&log), + posture: PostureResult::default(), + }), + test_deadline().teardown, + ); + cleanup.target_belief = Some(Box::new(LoggedLease { + log: Arc::clone(&log), + release: "belief-", + fail: true, + })); + cleanup.containment = Some(logged(&log, "containment-")); + cleanup.accessibility = Some(logged(&log, "ax-")); + + let outcome = cleanup.cleanup(test_deadline().teardown); + assert_eq!( + *log.lock().unwrap(), + vec!["belief-", "containment-", "ax-", "posture-"] + ); + assert_eq!(outcome.leases.target_belief, LeaseTeardownStatus::Failed); + assert_eq!(outcome.leases.containment, LeaseTeardownStatus::Released); + assert_eq!(outcome.native_evidence.fields["belief-_attempted"], true); + assert!(poison.load(Ordering::Acquire)); + } + + #[test] + fn containment_barrier_timeout_preserves_evidence_and_is_unverifiable() { + let release = containment_evidence( + SuppressionOutcome { + activations: 0, + restore_attempts: 2, + restore_failures: 0, + }, + false, + true, + ); + assert_eq!(release.evidence.fields["containment_restore_attempts"], 2); + assert_eq!( + release.failure.as_ref().unwrap().code, + ErrorCode::PostureUnverifiable + ); + } + + #[test] + fn containment_excursion_preserves_evidence_and_is_a_violation() { + let release = containment_evidence( + SuppressionOutcome { + activations: 1, + restore_attempts: 1, + restore_failures: 0, + }, + true, + true, + ); + assert_eq!(release.evidence.fields["containment_activations"], 1); + assert_eq!( + release.failure.as_ref().unwrap().code, + ErrorCode::PostureViolated + ); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/lifecycle.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/lifecycle.rs new file mode 100644 index 0000000000..c843ecbcc1 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/lifecycle.rs @@ -0,0 +1,1139 @@ +use std::{ + collections::{BTreeMap, HashMap}, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant}, +}; + +use async_trait::async_trait; +use cua_driver_core::{ + api::{ + capabilities::{CapabilityManifest, PlatformName}, + contracts::{ + ActionId, AppId, AppQuery, AppRef, AppSelector, PermissionState, Readiness, WindowId, + WindowRef, + }, + errors::{ErrorCode, ErrorPhase, NativeError}, + interaction::PostureResult, + platform::{LaunchPostureScope, LifecycleProvider, NativeLaunch, WindowProvider}, + settlement::{ + PendingSettlementEvidence, PendingSettlementState, SettledState, SettlementEvidence, + SettlementSignal, + }, + }, + protocol::V2_PROTOCOL_VERSION, +}; + +use crate::{ + apps::{ + self, + nsworkspace::{self, RunningApplicationInfo, WorkspaceEventHub, WorkspaceEventKind}, + }, + input::{skylight, slps_make_key}, + permissions, windows, +}; + +use super::{posture::MacLaunchPostureWitness, windows::MacWindowRegistry}; + +const LAUNCH_DEADLINE: Duration = Duration::from_secs(12); +const WINDOW_QUIET_WINDOW: Duration = Duration::from_millis(100); + +#[derive(Clone)] +pub struct MacLifecycle { + windows: MacWindowRegistry, + workspace_events: Arc, +} + +impl MacLifecycle { + pub fn new(windows: MacWindowRegistry) -> Self { + Self { + windows, + workspace_events: WorkspaceEventHub::shared(), + } + } +} + +#[async_trait] +impl LifecycleProvider for MacLifecycle { + async fn readiness(&self) -> Result { + tokio::task::spawn_blocking(readiness_snapshot) + .await + .map_err(join_error) + } + + async fn capabilities(&self) -> Result { + let readiness = self.readiness().await?; + Ok(CapabilityManifest { + platform: PlatformName::Macos, + driver_version: env!("CARGO_PKG_VERSION").to_owned(), + protocol_version: format!( + "{}.{}", + V2_PROTOCOL_VERSION.major, V2_PROTOCOL_VERSION.minor + ), + permissions: readiness + .permissions + .into_iter() + .map(|(name, state)| (name, state == PermissionState::Granted)) + .collect(), + cells: Vec::new(), + }) + } + + async fn list_apps(&self, query: AppQuery) -> Result, NativeError> { + tokio::task::spawn_blocking(move || list_apps_native(query)) + .await + .map_err(join_error)? + } + + async fn launch_background( + &self, + selector: AppSelector, + posture_scope: &mut LaunchPostureScope, + ) -> Result { + if !permissions::status::accessibility_granted() { + return Err(NativeError::new( + ErrorCode::PermissionDenied, + ErrorPhase::Preflight, + true, + "Accessibility permission is required for exact background window identity", + )); + } + let resolved = tokio::task::spawn_blocking(move || resolve_launch(selector)) + .await + .map_err(join_error)??; + let preexisting = nsworkspace::running_applications() + .into_iter() + .find(|application| resolved.matches(application)); + + if reusable_existing_launch(&resolved, preexisting.as_ref()) { + if let Some(application) = preexisting.as_ref() { + let started = Instant::now(); + let deadline = started + LAUNCH_DEADLINE; + let mut events = self.workspace_events.subscribe(); + let mut witness = MacLaunchPostureWitness::begin(deadline)?; + witness.set_target(application.pid)?; + let app = app_ref_for_running(application); + let generation = application + .process_generation + .expect("reusable_existing_launch requires an exact process generation"); + let settlement_action_id = posture_scope + .action_id() + .cloned() + .unwrap_or_else(ActionId::new); + posture_scope.record_partial_result(app.clone(), Vec::new()); + posture_scope.pending_settlement = Some(pending_launch_evidence( + &settlement_action_id, + "macos_launch_reuse", + started, + Vec::new(), + Vec::new(), + )); + let result = match self.windows.list_windows(Some(&app)).await { + Ok(baseline_windows) => { + posture_scope.record_partial_result(app.clone(), baseline_windows.clone()); + wait_for_stable_windows( + &self.windows, + LaunchWait { + app: &app, + pid: application.pid, + expected_generation: generation, + baseline_windows: &baseline_windows, + dispatched: false, + settlement_action_id: &settlement_action_id, + profile: "macos_launch_reuse", + started, + deadline, + }, + &mut events, + &mut witness, + posture_scope, + ) + .await + } + Err(error) => Err(error), + }; + let (posture, native_evidence) = witness.finish(); + posture_scope.posture = posture.clone(); + posture_scope.native_evidence = native_evidence; + if result.is_ok() { + posture_scope.pending_settlement = None; + } + let posture_failure = posture_failure(&posture, Some(&app), posture_scope); + return match (result, posture_failure) { + (Ok(stable), None) => Ok(NativeLaunch { + app, + windows: stable.windows, + reused_running_app: true, + posture, + settlement: launch_settlement( + "macos_launch_reuse", + started, + false, + stable.window_list_changed, + stable.windows_empty, + ), + }), + (Ok(_), Some(posture_error)) => Err(posture_error), + (Err(error), None) => Err(error), + (Err(error), Some(posture_error)) => { + Err(NativeError::primary(vec![error, posture_error]) + .expect("launch failures are nonempty")) + } + }; + } + } + + let started = Instant::now(); + let deadline = started + LAUNCH_DEADLINE; + let settlement_action_id = posture_scope + .action_id() + .cloned() + .unwrap_or_else(ActionId::new); + let baseline_windows = if let Some(application) = preexisting.as_ref() { + let app = app_ref_for_running(application); + self.windows.list_windows(Some(&app)).await? + } else { + Vec::new() + }; + let mut events = self.workspace_events.subscribe(); + // All posture streams, exact baselines and containment must be live + // before this call records or performs any launch side effect. + let mut witness = + begin_launch_after_witness(posture_scope, MacLaunchPostureWitness::begin(deadline))?; + posture_scope.pending_settlement = Some(pending_launch_evidence( + &settlement_action_id, + "macos_background_launch", + started, + vec![SettlementSignal::DispatchStarted], + vec![SettlementSignal::DispatchComplete], + )); + + let launch_ref = resolved.app_ref.clone(); + let config = nsworkspace::OpenConfig { + arguments: resolved.arguments.clone(), + apple_event_bundle_id: resolved.bundle_id.clone(), + ..Default::default() + }; + let completion_timeout = deadline.saturating_duration_since(Instant::now()); + let completion_containment = witness.completion_containment(); + let dispatch = tokio::task::spawn_blocking(move || { + nsworkspace::open_application_pid_with_timeout( + &launch_ref, + &config, + completion_timeout, + move |pid| { + let _ = + completion_containment.narrow_to_target(pid, Duration::from_millis(250)); + }, + ) + }) + .await + .map_err(join_error) + .and_then(|result| result.map_err(|error| launch_error(error.to_string()))); + + let attempt = match dispatch { + Err(error) => Err(error), + Ok(pid) => match witness.set_target(pid) { + Err(error) => Err(error), + Ok(()) => { + posture_scope.pending_settlement = Some(pending_launch_evidence( + &settlement_action_id, + "macos_background_launch", + started, + vec![ + SettlementSignal::DispatchStarted, + SettlementSignal::DispatchComplete, + ], + Vec::new(), + )); + match nsworkspace::running_application(pid) { + None => Err(launch_error( + "NSWorkspace returned an application that is not in the running-app registry", + ) + .with_detail("pid", pid)), + Some(application) => match application.process_generation { + None => Err(launch_error( + "NSWorkspace returned an application without an exact process generation", + ) + .with_detail("pid", pid)), + Some(generation) => { + let app = app_ref_for_running(&application); + posture_scope.record_partial_result(app.clone(), Vec::new()); + wait_for_stable_windows( + &self.windows, + LaunchWait { + app: &app, + pid, + expected_generation: generation, + baseline_windows: &baseline_windows, + dispatched: true, + settlement_action_id: &settlement_action_id, + profile: "macos_background_launch", + started, + deadline, + }, + &mut events, + &mut witness, + posture_scope, + ) + .await + .map(|stable| LaunchAttempt { + reused_running_app: preexisting + .as_ref() + .is_some_and(|prior| same_process_identity(prior, &application)), + app, + stable, + }) + } + }, + } + } + }, + }; + let (posture, native_evidence) = witness.finish(); + posture_scope.posture = posture.clone(); + posture_scope.native_evidence = native_evidence; + if attempt.is_ok() { + posture_scope.pending_settlement = None; + } + let posture_failure = + posture_failure(&posture, posture_scope.partial_app.as_ref(), posture_scope); + match (attempt, posture_failure) { + (Ok(attempt), None) => Ok(NativeLaunch { + app: attempt.app, + windows: attempt.stable.windows, + reused_running_app: attempt.reused_running_app, + posture, + settlement: launch_settlement( + "macos_background_launch", + started, + true, + attempt.stable.window_list_changed, + attempt.stable.windows_empty, + ), + }), + (Ok(_), Some(posture_error)) => Err(posture_error), + (Err(error), None) => Err(error), + (Err(error), Some(posture_error)) => { + Err(NativeError::primary(vec![error, posture_error]) + .expect("launch failures are nonempty")) + } + } + } +} + +struct LaunchAttempt { + app: AppRef, + stable: StableLaunchWindows, + reused_running_app: bool, +} + +fn begin_launch_after_witness( + posture_scope: &mut LaunchPostureScope, + witness: Result, +) -> Result { + let witness = witness?; + posture_scope.begin_launch(); + Ok(witness) +} + +struct StableLaunchWindows { + windows: Vec, + window_list_changed: bool, + windows_empty: bool, +} + +struct LaunchWait<'a> { + app: &'a AppRef, + pid: i32, + expected_generation: u64, + baseline_windows: &'a [WindowRef], + dispatched: bool, + settlement_action_id: &'a ActionId, + profile: &'a str, + started: Instant, + deadline: Instant, +} + +async fn wait_for_stable_windows( + registry: &MacWindowRegistry, + wait: LaunchWait<'_>, + events: &mut tokio::sync::broadcast::Receiver, + witness: &mut MacLaunchPostureWitness, + posture_scope: &mut LaunchPostureScope, +) -> Result { + let mut poll = tokio::time::interval(Duration::from_millis(25)); + poll.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay); + let mut stability = LaunchWindowStability::new(Instant::now()); + loop { + if Instant::now() >= wait.deadline { + return Err(launch_error( + "app launch did not reach a stable window set before its deadline", + ) + .with_detail("pid", wait.pid)); + } + tokio::select! { + _ = poll.tick() => {} + event = events.recv() => { + match event { + Ok(event) => { + if let Some(error) = launch_event_failure(wait.pid, &event) { + return Err(error); + } + } + // A lagged lifecycle receiver is not evidence that the + // process survived. The exact registry snapshot below is + // the conservative resynchronization boundary. + Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {} + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + return Err(launch_error("workspace lifecycle event stream closed during launch")); + } + } + } + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(wait.deadline)) => { + return Err(launch_error("app launch deadline elapsed") + .with_detail("pid", wait.pid)); + } + } + witness.drain_events(); + let application = nsworkspace::running_application(wait.pid).ok_or_else(|| { + launch_error("app disappeared before launch settlement completed") + .with_detail("pid", wait.pid) + })?; + if application.process_generation != Some(wait.expected_generation) { + return Err(launch_error( + "process identity changed before launch settlement completed", + ) + .with_detail("pid", wait.pid) + .with_detail("expected_process_generation", wait.expected_generation) + .with_detail( + "current_process_generation", + application.process_generation.unwrap_or_default(), + )); + } + let windows = registry.list_windows(Some(wait.app)).await?; + posture_scope.record_partial_result(wait.app.clone(), windows.clone()); + let observed_signals = wait + .dispatched + .then_some(SettlementSignal::DispatchComplete) + .into_iter() + .collect(); + posture_scope.pending_settlement = Some(pending_launch_evidence( + wait.settlement_action_id, + wait.profile, + wait.started, + observed_signals, + Vec::new(), + )); + if stability.observe(&windows, Instant::now(), application.finished_launching) { + let window_list_changed = window_ids(wait.baseline_windows) != window_ids(&windows); + return Ok(StableLaunchWindows { + windows_empty: windows.is_empty(), + windows, + window_list_changed, + }); + } + } +} + +fn same_process_identity(left: &RunningApplicationInfo, right: &RunningApplicationInfo) -> bool { + left.pid == right.pid + && left.process_generation.is_some() + && left.process_generation == right.process_generation +} + +fn window_ids(windows: &[WindowRef]) -> Vec { + let mut ids: Vec<_> = windows.iter().map(|window| window.id.clone()).collect(); + ids.sort(); + ids +} + +fn pending_launch_evidence( + action_id: &ActionId, + profile: &str, + started: Instant, + observed_signals: Vec, + missing_signals: Vec, +) -> PendingSettlementEvidence { + PendingSettlementEvidence { + state: PendingSettlementState::Pending, + trigger_action_id: action_id.clone(), + profile: profile.to_owned(), + elapsed_ms: started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64, + observed_signals, + missing_signals, + } +} + +fn launch_settlement( + profile: &str, + started: Instant, + dispatched: bool, + window_list_changed: bool, + windows_empty: bool, +) -> SettlementEvidence { + let mut observed_signals = Vec::new(); + if dispatched { + observed_signals.push(SettlementSignal::DispatchComplete); + } + if window_list_changed { + observed_signals.push(SettlementSignal::WindowListChanged); + } + SettlementEvidence { + state: SettledState::Settled, + trigger_action_id: None, + profile: profile.to_owned(), + elapsed_ms: started.elapsed().as_millis().min(u128::from(u64::MAX)) as u64, + observed_signals, + terminal_signal: if windows_empty { + "application_finished_without_windows".to_owned() + } else { + "stable_window_set".to_owned() + }, + quiet_window_ms: WINDOW_QUIET_WINDOW.as_millis() as u64, + resumed_from_prior_call: false, + } +} + +fn posture_failure( + posture: &PostureResult, + app: Option<&AppRef>, + posture_scope: &LaunchPostureScope, +) -> Option { + (!posture.held).then(|| { + let observed_excursion = posture.frontmost_changed + || posture.key_window_changed + || posture.physical_cursor_moved + || posture.restored_after_violation; + let (code, retryable, message) = if observed_excursion { + ( + ErrorCode::PostureViolated, + false, + "app launched but foreground, exact focused-window, or cursor posture changed", + ) + } else { + ( + ErrorCode::PostureUnverifiable, + true, + "app launch posture witness was incomplete, unreadable, or lagged", + ) + }; + NativeError::new(code, ErrorPhase::Verify, retryable, message) + .with_detail("app", serde_json::to_value(app).unwrap_or_default()) + .with_detail( + "windows", + serde_json::to_value(&posture_scope.partial_windows).unwrap_or_default(), + ) + .with_detail("posture", serde_json::to_value(posture).unwrap_or_default()) + }) +} + +#[derive(Debug)] +struct LaunchWindowStability { + prior_ids: Option>, + stable_since: Instant, +} + +impl LaunchWindowStability { + fn new(now: Instant) -> Self { + Self { + prior_ids: None, + stable_since: now, + } + } + + fn observe(&mut self, windows: &[WindowRef], now: Instant, finished: bool) -> bool { + let ids: Vec<_> = windows.iter().map(|window| window.id.clone()).collect(); + if self.prior_ids.as_ref() != Some(&ids) { + self.prior_ids = Some(ids); + self.stable_since = now; + } + finished && now.saturating_duration_since(self.stable_since) >= WINDOW_QUIET_WINDOW + } +} + +fn launch_event_failure(pid: i32, event: &nsworkspace::WorkspaceEvent) -> Option { + (event.pid == pid && event.kind == WorkspaceEventKind::Terminated).then(|| { + launch_error("app terminated before launch settlement completed").with_detail("pid", pid) + }) +} + +#[derive(Debug, Clone)] +struct ResolvedLaunch { + app_ref: String, + bundle_id: Option, + executable_path: Option, + name: Option, + arguments: Vec, +} + +impl ResolvedLaunch { + fn matches(&self, application: &RunningApplicationInfo) -> bool { + if let Some(bundle_id) = &self.bundle_id { + return application.bundle_id.as_ref() == Some(bundle_id); + } + if let Some(executable) = &self.executable_path { + return application + .executable_path + .as_ref() + .is_some_and(|path| Path::new(path) == executable); + } + self.name.as_ref().is_some_and(|name| { + application + .name + .as_ref() + .is_some_and(|candidate| candidate.eq_ignore_ascii_case(name)) + }) + } +} + +fn reusable_existing_launch( + resolved: &ResolvedLaunch, + application: Option<&RunningApplicationInfo>, +) -> bool { + resolved.arguments.is_empty() + && application.is_some_and(|app| app.process_generation.is_some() && resolved.matches(app)) +} + +trait LaunchCatalog { + fn bundle_exists(&self, bundle_id: &str) -> bool; + fn locate_name(&self, name: &str) -> Option<(String, Option)>; + fn bundle_id_for_path(&self, path: &str) -> Option; +} + +struct SystemLaunchCatalog; + +impl LaunchCatalog for SystemLaunchCatalog { + fn bundle_exists(&self, bundle_id: &str) -> bool { + apps::resolve_bundle_id_to_locator(bundle_id).is_some() + } + + fn locate_name(&self, name: &str) -> Option<(String, Option)> { + apps::locate_by_name(name).map(|locator| locator.app_ref_and_bundle_id()) + } + + fn bundle_id_for_path(&self, path: &str) -> Option { + apps::bundle_id_for_app_path(path) + } +} + +fn resolve_launch(selector: AppSelector) -> Result { + resolve_launch_with_catalog(selector, &SystemLaunchCatalog) +} + +fn resolve_launch_with_catalog( + selector: AppSelector, + catalog: &dyn LaunchCatalog, +) -> Result { + match selector { + AppSelector::BundleId { bundle_id } => { + if bundle_id.trim().is_empty() { + return Err(NativeError::invalid("bundle_id cannot be empty")); + } + if !catalog.bundle_exists(&bundle_id) { + return Err(app_not_found("bundle_id", &bundle_id)); + } + Ok(ResolvedLaunch { + app_ref: bundle_id.clone(), + bundle_id: Some(bundle_id), + executable_path: None, + name: None, + arguments: Vec::new(), + }) + } + AppSelector::Name { name } => { + if name.trim().is_empty() { + return Err(NativeError::invalid("app name cannot be empty")); + } + let (app_ref, bundle_id) = catalog + .locate_name(&name) + .ok_or_else(|| app_not_found("name", &name))?; + Ok(ResolvedLaunch { + app_ref, + bundle_id, + executable_path: None, + name: Some(name), + arguments: Vec::new(), + }) + } + AppSelector::Executable { path, arguments } => { + if path.trim().is_empty() { + return Err(NativeError::invalid("executable path cannot be empty")); + } + let executable = + std::fs::canonicalize(&path).map_err(|_| app_not_found("executable", &path))?; + let bundle = enclosing_app_bundle(&executable).ok_or_else(|| { + NativeError::invalid( + "macOS executable selectors must name a .app bundle or an executable inside one", + ) + .with_detail("path", path.clone()) + })?; + let bundle_string = bundle.to_string_lossy().to_string(); + Ok(ResolvedLaunch { + app_ref: bundle_string.clone(), + bundle_id: catalog.bundle_id_for_path(&bundle_string), + executable_path: Some(executable), + name: None, + arguments, + }) + } + } +} + +fn enclosing_app_bundle(path: &Path) -> Option { + if path.extension().and_then(|extension| extension.to_str()) == Some("app") && path.is_dir() { + return Some(path.to_path_buf()); + } + path.ancestors() + .find(|ancestor| { + ancestor + .extension() + .and_then(|extension| extension.to_str()) + == Some("app") + }) + .map(Path::to_path_buf) +} + +fn list_apps_native(query: AppQuery) -> Result, NativeError> { + let running = nsworkspace::running_applications(); + let running_by_bundle: HashMap<_, _> = running + .iter() + .filter_map(|application| { + application + .bundle_id + .as_ref() + .map(|bundle| (bundle.clone(), application)) + }) + .collect(); + let mut apps: Vec = running.iter().map(app_ref_for_running).collect(); + for installed in apps::list_installed_apps() { + if installed + .bundle_id + .as_ref() + .is_some_and(|bundle| running_by_bundle.contains_key(bundle)) + { + continue; + } + let identity = app_id( + installed.bundle_id.as_deref(), + installed.launch_path.as_deref(), + None, + ); + apps.push(AppRef { + id: identity, + name: Some(installed.name), + pid: None, + running: false, + }); + } + let needle = query.name_contains.map(|value| value.to_lowercase()); + apps.retain(|app| { + query.running.is_none_or(|running| app.running == running) + && needle.as_ref().is_none_or(|needle| { + app.name + .as_ref() + .is_some_and(|name| name.to_lowercase().contains(needle)) + }) + }); + apps.sort_by(|left, right| left.name.cmp(&right.name).then(left.pid.cmp(&right.pid))); + Ok(apps) +} + +fn app_ref_for_running(application: &RunningApplicationInfo) -> AppRef { + AppRef { + id: app_id( + application.bundle_id.as_deref(), + application.executable_path.as_deref(), + application + .process_generation + .map(|generation| (application.pid, generation)), + ), + name: application.name.clone(), + pid: u32::try_from(application.pid).ok(), + running: true, + } +} + +fn app_id( + bundle_id: Option<&str>, + executable_path: Option<&str>, + process: Option<(i32, u64)>, +) -> AppId { + let value = process + .map(|(pid, generation)| format!("macos:process:{pid}:{generation:016x}")) + .or_else(|| bundle_id.map(|bundle| format!("macos:bundle:{bundle}"))) + .or_else(|| executable_path.map(|path| format!("macos:executable:{path}"))) + .unwrap_or_else(|| "macos:unknown".to_owned()); + AppId::parse(value).expect("constructed macOS app id is nonempty") +} + +fn readiness_snapshot() -> Readiness { + let accessibility = permissions::status::accessibility_granted(); + let screen_preflight = permissions::status::screen_recording_granted(); + let screen_live = permissions::status::screen_recording_capturable(); + let targeted_events = skylight::is_available(); + let slps_make_key = slps_make_key::available(); + let spaces = windows::space_query_available(); + let mut permissions = BTreeMap::new(); + permissions.insert( + "accessibility".to_owned(), + if accessibility { + PermissionState::Granted + } else { + PermissionState::Denied + }, + ); + permissions.insert( + "screen_recording".to_owned(), + if screen_live { + PermissionState::Granted + } else { + PermissionState::Denied + }, + ); + let mut diagnostics = BTreeMap::new(); + diagnostics.insert( + "screen_recording_preflight".to_owned(), + screen_preflight.into(), + ); + diagnostics.insert( + "screen_recording_live_capture".to_owned(), + screen_live.into(), + ); + diagnostics.insert("targeted_event_symbols".to_owned(), targeted_events.into()); + diagnostics.insert("slps_make_key_symbols".to_owned(), slps_make_key.into()); + diagnostics.insert("space_query_symbols".to_owned(), spaces.into()); + diagnostics.insert("os_version".to_owned(), macos_version().into()); + Readiness { + ready: accessibility && screen_live && targeted_events && slps_make_key && spaces, + permissions, + diagnostics, + } +} + +fn macos_version() -> String { + std::process::Command::new("sw_vers") + .args(["-productVersion"]) + .output() + .ok() + .filter(|output| output.status.success()) + .map(|output| String::from_utf8_lossy(&output.stdout).trim().to_owned()) + .filter(|version| !version.is_empty()) + .unwrap_or_else(|| "unknown".to_owned()) +} + +fn app_not_found(selector: &str, value: &str) -> NativeError { + NativeError::new( + ErrorCode::AppNotFound, + ErrorPhase::Preflight, + false, + format!("no installed macOS app matched {selector} '{value}'"), + ) + .with_detail(selector, value.to_owned()) +} + +fn launch_error(message: impl Into) -> NativeError { + NativeError::new( + ErrorCode::AppLaunchFailed, + ErrorPhase::Dispatch, + true, + message, + ) +} + +fn join_error(error: tokio::task::JoinError) -> NativeError { + NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + true, + format!("macOS lifecycle task failed: {error}"), + ) +} + +#[cfg(test)] +mod tests { + use std::{fs, time::Duration}; + + use cua_driver_core::api::contracts::{AppId, WindowId}; + + use super::*; + + struct FakeCatalog; + + impl LaunchCatalog for FakeCatalog { + fn bundle_exists(&self, bundle_id: &str) -> bool { + bundle_id == "com.example.fixture" + } + + fn locate_name(&self, name: &str) -> Option<(String, Option)> { + (name == "Fixture").then(|| { + ( + "/Applications/Fixture.app".to_owned(), + Some("com.example.fixture".to_owned()), + ) + }) + } + + fn bundle_id_for_path(&self, _path: &str) -> Option { + Some("com.example.executable".to_owned()) + } + } + + fn running_fixture() -> RunningApplicationInfo { + RunningApplicationInfo { + pid: 120, + name: Some("Fixture".to_owned()), + bundle_id: Some("com.example.fixture".to_owned()), + executable_path: Some("/Applications/Fixture.app/Contents/MacOS/fixture".to_owned()), + process_generation: Some(7), + active: false, + hidden: false, + finished_launching: true, + regular: true, + } + } + + fn window(id: &str) -> WindowRef { + WindowRef { + id: WindowId::parse(id).unwrap(), + app: AppRef { + id: AppId::parse("macos:bundle:com.example.fixture").unwrap(), + name: Some("Fixture".to_owned()), + pid: Some(120), + running: true, + }, + title: None, + } + } + + #[test] + fn all_launch_selector_shapes_preserve_native_dispatch_inputs() { + let bundle = resolve_launch_with_catalog( + AppSelector::BundleId { + bundle_id: "com.example.fixture".to_owned(), + }, + &FakeCatalog, + ) + .unwrap(); + assert_eq!(bundle.app_ref, "com.example.fixture"); + assert_eq!(bundle.bundle_id.as_deref(), Some("com.example.fixture")); + + let named = resolve_launch_with_catalog( + AppSelector::Name { + name: "Fixture".to_owned(), + }, + &FakeCatalog, + ) + .unwrap(); + assert_eq!(named.app_ref, "/Applications/Fixture.app"); + assert_eq!(named.name.as_deref(), Some("Fixture")); + + let root = std::env::temp_dir().join(format!( + "cua-driver-plan002-selector-{}", + uuid::Uuid::new_v4() + )); + let bundle_path = root.join("Executable.app"); + fs::create_dir_all(&bundle_path).unwrap(); + let executable = resolve_launch_with_catalog( + AppSelector::Executable { + path: bundle_path.to_string_lossy().into_owned(), + arguments: vec!["--fixture".to_owned()], + }, + &FakeCatalog, + ) + .unwrap(); + assert_eq!(executable.arguments, vec!["--fixture"]); + assert_eq!( + executable.bundle_id.as_deref(), + Some("com.example.executable") + ); + let canonical_bundle = fs::canonicalize(&bundle_path).unwrap(); + assert_eq!( + executable.executable_path.as_deref(), + Some(canonical_bundle.as_path()) + ); + fs::remove_dir_all(root).unwrap(); + } + + #[test] + fn malformed_and_unresolvable_selectors_fail_before_launch() { + for selector in [ + AppSelector::BundleId { + bundle_id: " ".to_owned(), + }, + AppSelector::Name { + name: "".to_owned(), + }, + AppSelector::Executable { + path: " ".to_owned(), + arguments: Vec::new(), + }, + ] { + let error = resolve_launch_with_catalog(selector, &FakeCatalog).unwrap_err(); + assert_eq!(error.code, ErrorCode::InvalidRequest); + assert_eq!(error.phase, ErrorPhase::Validate); + } + + let error = resolve_launch_with_catalog( + AppSelector::BundleId { + bundle_id: "com.example.missing".to_owned(), + }, + &FakeCatalog, + ) + .unwrap_err(); + assert_eq!(error.code, ErrorCode::AppNotFound); + } + + #[test] + fn reuse_is_exact_and_executable_arguments_force_a_launch_dispatch() { + let application = running_fixture(); + let mut resolved = ResolvedLaunch { + app_ref: "com.example.fixture".to_owned(), + bundle_id: Some("com.example.fixture".to_owned()), + executable_path: None, + name: None, + arguments: Vec::new(), + }; + assert!(reusable_existing_launch(&resolved, Some(&application))); + + resolved.arguments.push("--new-document".to_owned()); + assert!(!reusable_existing_launch(&resolved, Some(&application))); + resolved.arguments.clear(); + resolved.bundle_id = Some("com.example.other".to_owned()); + assert!(!reusable_existing_launch(&resolved, Some(&application))); + + let mut generation_unknown = running_fixture(); + generation_unknown.process_generation = None; + resolved.bundle_id = Some("com.example.fixture".to_owned()); + assert!(!reusable_existing_launch( + &resolved, + Some(&generation_unknown) + )); + } + + #[test] + fn reused_running_app_requires_the_returned_exact_process_identity() { + let prior = running_fixture(); + let mut same = prior.clone(); + assert!(same_process_identity(&prior, &same)); + + same.process_generation = Some(8); + assert!(!same_process_identity(&prior, &same)); + same.process_generation = Some(7); + same.pid = 121; + assert!(!same_process_identity(&prior, &same)); + } + + #[test] + fn settlement_reports_only_signals_that_were_observed() { + let started = Instant::now(); + let empty_dispatch = + launch_settlement("macos_background_launch", started, true, false, true); + assert_eq!( + empty_dispatch.observed_signals, + vec![SettlementSignal::DispatchComplete] + ); + + let reused = launch_settlement("macos_launch_reuse", started, false, false, false); + assert!(reused.observed_signals.is_empty()); + assert_eq!( + window_ids(&[window("one"), window("two")]), + window_ids(&[window("two"), window("one")]) + ); + } + + #[test] + fn settlement_quiet_window_restarts_for_each_window_set_change() { + let start = Instant::now(); + let mut tracker = LaunchWindowStability::new(start); + assert!(!tracker.observe(&[], start, true)); + assert!(!tracker.observe(&[window("one")], start + Duration::from_millis(80), true,)); + assert!(!tracker.observe( + &[window("one"), window("two")], + start + Duration::from_millis(160), + true, + )); + assert!(!tracker.observe( + &[window("one"), window("two")], + start + Duration::from_millis(259), + true, + )); + assert!(tracker.observe( + &[window("one"), window("two")], + start + Duration::from_millis(260), + true, + )); + } + + #[test] + fn finished_app_with_no_windows_is_a_stable_success_shape() { + let start = Instant::now(); + let mut tracker = LaunchWindowStability::new(start); + assert!(!tracker.observe(&[], start, true)); + assert!(tracker.observe(&[], start + WINDOW_QUIET_WINDOW, true)); + } + + #[test] + fn only_target_termination_aborts_launch_settlement() { + let unrelated = nsworkspace::WorkspaceEvent { + kind: WorkspaceEventKind::Terminated, + pid: 121, + bundle_id: Some("com.example.other".to_owned()), + process_generation: Some(8), + }; + assert!(launch_event_failure(120, &unrelated).is_none()); + + let target = nsworkspace::WorkspaceEvent { + pid: 120, + ..unrelated + }; + let error = launch_event_failure(120, &target).unwrap(); + assert_eq!(error.code, ErrorCode::AppLaunchFailed); + assert_eq!(error.phase, ErrorPhase::Dispatch); + } + + #[test] + fn launch_posture_distinguishes_unverifiable_witness_from_observed_excursion() { + let scope = LaunchPostureScope::default(); + let unverifiable = posture_failure( + &PostureResult { + held: false, + ..PostureResult::default() + }, + None, + &scope, + ) + .unwrap(); + assert_eq!(unverifiable.code, ErrorCode::PostureUnverifiable); + + let violated = posture_failure( + &PostureResult { + held: false, + frontmost_changed: true, + restored_after_violation: true, + ..PostureResult::default() + }, + None, + &scope, + ) + .unwrap(); + assert_eq!(violated.code, ErrorCode::PostureViolated); + } + + #[test] + fn failed_witness_acquisition_refuses_before_launch_side_effect_is_marked() { + let mut scope = LaunchPostureScope::for_action(ActionId::new()); + let error = begin_launch_after_witness::<()>( + &mut scope, + Err(NativeError::new( + ErrorCode::PostureUnverifiable, + ErrorPhase::Preflight, + true, + "injected witness acquisition failure", + )), + ) + .unwrap_err(); + assert_eq!(error.code, ErrorCode::PostureUnverifiable); + assert!(!scope.side_effect_started()); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/menu.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/menu.rs new file mode 100644 index 0000000000..6f2830cabd --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/menu.rs @@ -0,0 +1,47 @@ +//! Typed seam for future exact per-target menu dismissal suppression. +//! +//! Plan 004 does not have a proved event-tap predicate. Production therefore +//! refuses every required menu lease instead of installing a broader filter. + +use cua_driver_core::api::{contracts::MenuId, errors::NativeError, interaction::NativeEvidence}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum MenuSuppressionPlan { + NotApplicable, + ExactPredicateRequired { pid: i32, menu_id: MenuId }, +} + +pub(crate) trait MenuSuppressionResource: Send { + fn release(&mut self) -> Result; +} + +pub(crate) fn acquire_production( + plan: &MenuSuppressionPlan, +) -> Result>, NativeError> { + match plan { + MenuSuppressionPlan::NotApplicable => Ok(None), + MenuSuppressionPlan::ExactPredicateRequired { pid, menu_id } => { + Err(NativeError::unsupported( + "recipe_unproven: exact per-pid and per-menu event-tap predicate is not proved", + ) + .with_detail("recipe_status", "recipe_unproven") + .with_detail("pid", *pid) + .with_detail("menu_id", menu_id.to_string())) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn production_refuses_required_menu_suppression() { + let plan = MenuSuppressionPlan::ExactPredicateRequired { + pid: 44, + menu_id: MenuId::parse("menu").unwrap(), + }; + let error = acquire_production(&plan).err().unwrap(); + assert_eq!(error.details["recipe_status"], "recipe_unproven"); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/mod.rs new file mode 100644 index 0000000000..511b18d0f6 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/mod.rs @@ -0,0 +1,207 @@ +//! Composition root for the background-only macOS v2 platform driver. + +use async_trait::async_trait; +use cua_driver_core::api::{ + capabilities::CapabilityCell, + contracts::KeyStroke, + errors::NativeError, + interaction::InteractionScope, + observation::{ResolvedDrag, ResolvedFocus, ResolvedPoint, ResolvedScroll, ResolvedWindow}, + platform::{ + ClickSpec, KeyboardActionProvider, NativeDispatch, PlatformDriver, PointerActionProvider, + TargetFocusCoordinator, + }, +}; + +pub mod actions; +pub mod focus; +pub mod interaction; +pub mod lifecycle; +pub mod menu; +pub mod observation; +pub mod posture; +pub mod settlement; +pub mod target; +pub mod windows; + +use actions::{semantic_capability_cells, MacSemanticActions}; +use interaction::MacInteractionProvider; +use lifecycle::MacLifecycle; +use observation::MacObservationProvider; +use target::{ + MacInvalidationHub, MacInvalidationSubscription, MacTargetFocusCoordinator, MacTargetState, +}; +use windows::MacWindowRegistry; + +/// Native v2 provider composition. Plans 003 through 007 replace the explicit +/// unsupported action providers while preserving this lifecycle and identity +/// foundation. +#[derive(Clone)] +pub struct MacDriver { + lifecycle: MacLifecycle, + windows: MacWindowRegistry, + observation: MacObservationProvider, + interaction: MacInteractionProvider, + semantic: MacSemanticActions, + unavailable: UnavailableProvider, + invalidations: MacInvalidationHub, +} + +impl MacDriver { + pub fn new() -> Self { + let invalidations = MacInvalidationHub::default(); + let windows = MacWindowRegistry::new(invalidations.clone()); + let lifecycle = MacLifecycle::new(windows.clone()); + let observation = MacObservationProvider::new(windows.clone()); + let interaction = MacInteractionProvider::new(windows.clone()); + let semantic = MacSemanticActions::new(windows.clone()); + Self { + lifecycle, + windows, + observation, + interaction, + semantic, + unavailable: UnavailableProvider, + invalidations, + } + } +} + +impl Default for MacDriver { + fn default() -> Self { + Self::new() + } +} + +#[derive(Debug, Clone, Copy, Default)] +pub struct UnavailableProvider; + +fn later_plan(capability: &str) -> NativeError { + NativeError::unsupported(format!( + "macOS v2 {capability} is not implemented by the lifecycle/window provider" + )) +} + +#[async_trait] +impl PointerActionProvider for UnavailableProvider { + async fn click( + &self, + _scope: &mut InteractionScope, + _point: ResolvedPoint, + _click: ClickSpec, + ) -> Result { + Err(later_plan("pointer click")) + } + + async fn drag( + &self, + _scope: &mut InteractionScope, + _drag: ResolvedDrag, + ) -> Result { + Err(later_plan("pointer drag")) + } + + async fn scroll( + &self, + _scope: &mut InteractionScope, + _scroll: ResolvedScroll, + ) -> Result { + Err(later_plan("pointer scroll")) + } +} + +#[async_trait] +impl KeyboardActionProvider for UnavailableProvider { + async fn press_key( + &self, + _scope: &mut InteractionScope, + _focus: &ResolvedFocus, + _stroke: KeyStroke, + ) -> Result { + Err(later_plan("keyboard input")) + } + + async fn type_text( + &self, + _scope: &mut InteractionScope, + _focus: &ResolvedFocus, + _text: &str, + ) -> Result { + Err(later_plan("text input")) + } +} + +#[async_trait] +impl PlatformDriver for MacDriver { + type TargetState = MacTargetState; + type TargetFocusCoordinator = MacTargetFocusCoordinator; + type Lifecycle = MacLifecycle; + type Windows = MacWindowRegistry; + type Observation = MacObservationProvider; + type Semantic = MacSemanticActions; + type Pointer = UnavailableProvider; + type Keyboard = UnavailableProvider; + type Interaction = MacInteractionProvider; + type Invalidations = MacInvalidationSubscription; + + async fn create_target_state( + &self, + window: &ResolvedWindow, + ) -> Result<(Self::TargetState, Self::TargetFocusCoordinator), NativeError> { + let facts = self.windows.facts_for_stamp(&window.stamp()).await?; + Ok(( + MacTargetState::new( + window.stamp(), + facts.pid, + facts.cg_window_id, + self.invalidations.clone(), + )?, + MacTargetFocusCoordinator::default(), + )) + } + + async fn destroy_target_state( + &self, + target: &mut Self::TargetState, + focus: &mut Self::TargetFocusCoordinator, + ) -> Result<(), NativeError> { + target.invalidate(); + focus.shutdown().await + } + + fn lifecycle(&self) -> &Self::Lifecycle { + &self.lifecycle + } + + fn windows(&self) -> &Self::Windows { + &self.windows + } + + fn observation(&self) -> &Self::Observation { + &self.observation + } + + fn semantic(&self) -> &Self::Semantic { + &self.semantic + } + + fn pointer(&self) -> &Self::Pointer { + &self.unavailable + } + + fn keyboard(&self) -> &Self::Keyboard { + &self.unavailable + } + + fn interaction(&self) -> &Self::Interaction { + &self.interaction + } + + fn capability_cells(&self, os_version: &str) -> Vec { + semantic_capability_cells(os_version) + } + + fn subscribe_invalidations(&self) -> Self::Invalidations { + self.invalidations.subscribe() + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/observation.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/observation.rs new file mode 100644 index 0000000000..ca27e2d256 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/observation.rs @@ -0,0 +1,1709 @@ +//! Coherent, window-scoped native observation for the macOS v2 driver. + +use std::{ + collections::{HashMap, HashSet}, + fs::OpenOptions, + io::Write, + os::unix::fs::{DirBuilderExt, OpenOptionsExt, PermissionsExt}, + path::{Path, PathBuf}, + sync::Arc, + time::{Duration, Instant, SystemTime, UNIX_EPOCH}, +}; + +use async_trait::async_trait; +use core_foundation::{ + base::{CFEqual, CFGetTypeID, CFRelease, CFRetain, CFTypeRef, TCFType}, + string::CFString, +}; +use cua_driver_core::api::{ + capabilities::WindowStateKind, + contracts::{CaptureRevision, ElementId, Rect, Size, SurfaceId, SurfaceKind, WindowRef}, + errors::{ErrorCode, ErrorPhase, NativeError}, + menu::NativeMenuObservation, + observation::{ + CaptureFreshness, NativeAccessibilityElement, NativeAccessibilityUpdate, + NativeElementHandle, NativeObservationUpdate, ObservationArtifactHandle, ResolvedWindow, + ResolvedWindowStamp, SurfaceOwner, SurfaceRecord, SurfaceToWindowTransform, + }, + platform::{ObservationProvider, ObserveRequest}, + settlement::{DirtyState, SettlementAttempt}, +}; +use screencapturekit::cm::SCFrameStatus; +use tokio::sync::Semaphore; + +use crate::{ + ax::bindings::{ + self, copy_action_names_exact, copy_attr_value, copy_ax_windows, copy_children, + copy_string_attr, copy_string_attr_exact, element_screen_rect, focused_element_of_pid, + AXUIElementCreateApplication, AXUIElementRef, + }, + video_sckit::{capture_window_sample, WindowFrameSample}, +}; + +use super::{ + target::MacTargetState, + windows::{MacRelatedWindowFacts, MacWindowFacts, MacWindowRegistry}, +}; + +const MAX_AX_ELEMENTS: usize = 2_000; +const MAX_AX_DEPTH: usize = 25; +const OBSERVATION_TIMEOUT: Duration = Duration::from_secs(5); +static SCK_CAPTURE_SLOTS: Semaphore = Semaphore::const_new(2); + +#[derive(Clone)] +pub struct MacObservationProvider { + windows: MacWindowRegistry, + artifact_root: Arc, +} + +impl MacObservationProvider { + pub fn new(windows: MacWindowRegistry) -> Self { + Self { + windows, + artifact_root: Arc::new( + std::env::temp_dir() + .join(format!("cua-driver-v2-observations-{}", std::process::id())), + ), + } + } + + /// Produce real FreshFrame settlement evidence without publishing an + /// observation artifact. Callers must pass the action's bounded deadline. + pub(crate) async fn record_fresh_frame_signal( + &self, + target: &mut MacTargetState, + deadline: Instant, + ) -> Result { + let epoch = target.signals.epoch(); + let facts_a = self.windows.facts_for_identity(&target.window).await?; + validate_observable(&facts_a)?; + let pending = capture(&facts_a, SurfaceKind::Window, deadline).await?; + let facts_b = self.windows.facts_for_identity(&facts_a.stamp).await?; + if facts_a != facts_b || !same_stable_identity(&target.window, &facts_b.stamp) { + return Err(NativeError::stale( + ErrorCode::ObservationRaced, + "target changed while producing FreshFrame evidence", + )); + } + let journal = target.signals.clone(); + let freshness = journal + .commit_if_epoch(epoch, || { + target.frames.classify_and_commit( + pending.cg_window_id, + &pending.sample, + pending.scale_factor, + ) + })? + .ok_or_else(|| { + NativeError::stale( + ErrorCode::ObservationRaced, + "target signaled a mutation while producing FreshFrame evidence", + ) + })?; + if !freshness.action_safe() { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "ScreenCaptureKit did not produce action-safe FreshFrame evidence", + )); + } + target + .signals + .record(cua_driver_core::api::settlement::SettlementSignal::FreshFrame); + Ok(freshness) + } + + async fn attempt( + &self, + target: &mut MacTargetState, + window: &ResolvedWindow, + request: ObserveRequest, + deadline: Instant, + ) -> Result { + let epoch = target.signals.epoch(); + let facts_a = self + .windows + .facts_for_identity(&window.stamp()) + .await + .map_err(classify_stamp_error)?; + validate_observable(&facts_a).map_err(AttemptError::Fatal)?; + let pid = facts_a.pid; + let cg_window_id = facts_a.cg_window_id; + let include_text = request.include_text; + let ax_snapshot = tokio::task::spawn_blocking(move || { + capture_ax_snapshot(pid, cg_window_id, include_text) + }) + .await + .map_err(|error| AttemptError::Fatal(join_error("AX capture", error)))? + .map_err(AttemptError::Fatal)?; + target + .replace_observed_elements(ax_snapshot.retained_elements()) + .map_err(AttemptError::Fatal)?; + + let related = related_surfaces(&self.windows, &facts_a, &ax_snapshot.related_windows) + .await + .map_err(classify_stamp_error)?; + let related_a: Vec<_> = related + .iter() + .map(|surface| surface.facts.clone()) + .collect(); + let mut captures = Vec::new(); + if request.include_screenshots { + captures.push( + capture(&facts_a, SurfaceKind::Window, deadline) + .await + .map_err(AttemptError::Fatal)?, + ); + for related_surface in related { + captures.push( + capture_related(related_surface, deadline) + .await + .map_err(AttemptError::Fatal)?, + ); + } + } + + let related_b_snapshot = tokio::task::spawn_blocking(move || { + capture_ax_snapshot(pid, cg_window_id, include_text) + }) + .await + .map_err(|error| AttemptError::Fatal(join_error("related AX revalidation", error)))? + .map_err(AttemptError::Fatal)?; + target + .replace_observed_elements(related_b_snapshot.retained_elements()) + .map_err(AttemptError::Fatal)?; + if !same_related_identities( + &ax_snapshot.related_windows, + &related_b_snapshot.related_windows, + ) { + return Err(AttemptError::Raced); + } + let related_b = self + .windows + .register_related_windows( + &facts_a, + related_candidates(&related_b_snapshot.related_windows), + ) + .await + .map_err(classify_stamp_error)?; + if related_a != related_b { + return Err(AttemptError::Raced); + } + + let facts_b = self + .windows + .facts_for_identity(&facts_a.stamp) + .await + .map_err(classify_stamp_error)?; + if !coherent_window_bracket( + &target.window, + &facts_a, + &facts_b, + epoch, + target.signals.epoch(), + ) { + return Err(AttemptError::Raced); + } + + Ok(AttemptResult { + facts: facts_b, + related: related_a, + ax_snapshot: related_b_snapshot, + captures, + epoch, + }) + } + + fn publish( + &self, + target: &mut MacTargetState, + window: &ResolvedWindow, + attempt: AttemptResult, + ) -> Result { + let epoch = attempt.epoch; + let journal = target.signals.clone(); + journal + .commit_if_epoch(epoch, || self.publish_unbracketed(target, window, attempt)) + .map_err(AttemptError::Fatal)? + .ok_or(AttemptError::Raced) + } + + fn publish_unbracketed( + &self, + target: &mut MacTargetState, + window: &ResolvedWindow, + attempt: AttemptResult, + ) -> Result { + let observed_window = resolved_from_facts(window, &attempt.facts)?; + let accessibility = target.elements.reconcile( + attempt.ax_snapshot, + attempt.facts.bounds, + attempt.facts.cg_window_id, + &attempt.facts.stamp, + &attempt.related, + )?; + let mut surfaces = Vec::with_capacity(attempt.captures.len()); + let mut artifacts = Vec::with_capacity(attempt.captures.len()); + let mut captured_at_unix_ms = now_unix_ms(); + for capture in attempt.captures { + let freshness = target.frames.classify_and_commit( + capture.cg_window_id, + &capture.sample, + capture.scale_factor, + )?; + if !freshness.action_safe() { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "ScreenCaptureKit did not provide action-safe freshness evidence", + ) + .with_detail("cg_window_id", capture.cg_window_id)); + } + captured_at_unix_ms = + captured_at_unix_ms.max(capture.sample.metadata.completion_unix_ms); + let (image_url, artifact) = materialize_png( + self.artifact_root.as_ref(), + capture.cg_window_id, + &capture.sample.png_bytes, + )?; + artifacts.push(artifact); + + let (owner_window, owner) = match capture.related_owner { + Some((owner_window, owner)) => ( + owner_window, + SurfaceOwner::RelatedTransient { + owner, + parent: observed_window.stamp(), + }, + ), + None => ( + observed_window.public.clone(), + SurfaceOwner::Target(observed_window.stamp()), + ), + }; + surfaces.push(SurfaceRecord { + id: SurfaceId::new(), + kind: capture.kind, + owner_window, + image_url, + raster_size: Size { + width: capture.sample.pixel_width, + height: capture.sample.pixel_height, + }, + window_bounds: Some(capture.bounds), + capture_revision: CaptureRevision::new(), + transform: capture.transform, + freshness, + owner, + approximate_bytes: capture.sample.png_bytes.len(), + }); + } + + let mut warnings = Vec::new(); + if accessibility.truncated { + warnings.push(format!( + "AX tree truncated at the bounded {MAX_AX_ELEMENTS}-element limit" + )); + } + let update = NativeObservationUpdate { + window: observed_window.clone(), + surfaces, + accessibility: Some(accessibility.update), + menu: NativeMenuObservation::Unchanged, + captured_at_unix_ms, + warnings, + artifacts, + }; + target.window = observed_window.stamp(); + Ok(update) + } +} + +#[async_trait] +impl ObservationProvider for MacObservationProvider { + async fn settle( + &self, + target: &mut MacTargetState, + dirty: &DirtyState, + deadline: std::time::Instant, + ) -> Result { + self.record_fresh_frame_signal(target, deadline).await?; + Ok(target + .signals + .settle(dirty, &dirty.profile.relevant_signals, deadline) + .await) + } + + async fn observe( + &self, + target: &mut MacTargetState, + window: &ResolvedWindow, + request: ObserveRequest, + ) -> Result { + if target.invalidated() { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "macOS target resources were invalidated", + )); + } + let deadline = Instant::now() + OBSERVATION_TIMEOUT; + for attempt_index in 0..2 { + let result = match self.attempt(target, window, request, deadline).await { + Ok(attempt) => self.publish(target, window, attempt), + Err(error) => Err(error), + }; + match result { + Ok(update) => return Ok(update), + Err(AttemptError::Raced) if attempt_index == 0 => continue, + Err(AttemptError::Raced) => { + return Err(NativeError::new( + ErrorCode::ObservationRaced, + ErrorPhase::Verify, + true, + "window identity or geometry changed twice during coherent observation", + )) + } + Err(AttemptError::Fatal(error)) => return Err(error), + } + } + unreachable!("coherent observation loop has exactly two attempts") + } +} + +enum AttemptError { + Raced, + Fatal(NativeError), +} + +fn classify_stamp_error(error: NativeError) -> AttemptError { + if matches!( + error.code, + ErrorCode::ObservationStale + | ErrorCode::ObservationRaced + | ErrorCode::WindowIdentityChanged + | ErrorCode::WindowNotFound + ) { + AttemptError::Raced + } else { + AttemptError::Fatal(error) + } +} + +struct AttemptResult { + facts: MacWindowFacts, + related: Vec, + ax_snapshot: RawAxSnapshot, + captures: Vec, + epoch: u64, +} + +fn same_stable_identity(left: &ResolvedWindowStamp, right: &ResolvedWindowStamp) -> bool { + left.app_id == right.app_id + && left.window_id == right.window_id + && left.generation == right.generation + && left.native_window == right.native_window + && left.process == right.process +} + +fn coherent_window_bracket( + target: &ResolvedWindowStamp, + facts_a: &MacWindowFacts, + facts_b: &MacWindowFacts, + epoch_a: u64, + epoch_b: u64, +) -> bool { + epoch_a == epoch_b && facts_a == facts_b && same_stable_identity(target, &facts_b.stamp) +} + +fn resolved_from_facts( + prior: &ResolvedWindow, + facts: &MacWindowFacts, +) -> Result { + if !same_stable_identity(&prior.stamp(), &facts.stamp) { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "refreshed macOS facts changed a stable target identity component", + )); + } + let scale_factor = facts + .scale_factor + .ok_or_else(|| NativeError::unsupported("target window has no exact backing scale"))?; + let mut resolved = prior.clone(); + resolved.geometry.bounds = facts.bounds; + resolved.geometry.scale_factor = scale_factor; + resolved.geometry.revision = facts.stamp.geometry_revision.clone(); + resolved.state = facts.state.clone(); + Ok(resolved) +} + +fn validate_observable(facts: &MacWindowFacts) -> Result<(), NativeError> { + let reason = match facts.state { + WindowStateKind::Minimized => Some("minimized"), + WindowStateKind::OffSpace => Some("off-space"), + WindowStateKind::Unknown => Some("visibility is unknown"), + WindowStateKind::Visible | WindowStateKind::Occluded => None, + }; + if let Some(reason) = reason { + return Err(NativeError::unsupported(format!( + "background observation refuses a {reason} macOS window" + )) + .with_detail("cg_window_id", facts.cg_window_id)); + } + if facts.minimized != Some(false) || facts.on_current_space == Some(false) { + return Err(NativeError::unsupported( + "background observation requires an exactly known non-minimized window on the current Space", + )); + } + if !facts.is_on_screen && facts.state != WindowStateKind::Occluded { + return Err(NativeError::unsupported( + "WindowServer does not report an observable target surface", + )); + } + let scale = facts.scale_factor.ok_or_else(|| { + NativeError::unsupported("target window has no exact display backing scale") + })?; + if !scale.is_finite() || scale <= 0.0 { + return Err(NativeError::unsupported( + "target window has an invalid display backing scale", + )); + } + Ok(()) +} + +struct PendingCapture { + cg_window_id: u32, + kind: SurfaceKind, + bounds: Rect, + scale_factor: f64, + related_owner: Option<(WindowRef, ResolvedWindowStamp)>, + transform: SurfaceToWindowTransform, + sample: WindowFrameSample, +} + +async fn capture( + facts: &MacWindowFacts, + kind: SurfaceKind, + deadline: Instant, +) -> Result { + let window_id = facts.cg_window_id; + let scale = facts + .scale_factor + .ok_or_else(|| NativeError::unsupported("window has no exact backing scale"))?; + let sample = capture_sample_until(window_id, scale, deadline).await?; + let transform = validated_surface_transform(&sample, facts.bounds, facts.bounds, scale)?; + Ok(PendingCapture { + cg_window_id: window_id, + kind, + bounds: facts.bounds, + scale_factor: scale, + related_owner: None, + transform, + sample, + }) +} + +#[derive(Clone)] +struct RelatedSurface { + facts: MacRelatedWindowFacts, + kind: SurfaceKind, + target_bounds: Rect, +} + +async fn capture_related( + surface: RelatedSurface, + deadline: Instant, +) -> Result { + let window_id = surface.facts.cg_window_id; + let scale = surface.facts.scale_factor; + let sample = capture_sample_until(window_id, scale, deadline).await?; + let transform = + validated_surface_transform(&sample, surface.facts.bounds, surface.target_bounds, scale)?; + Ok(PendingCapture { + cg_window_id: window_id, + kind: surface.kind, + bounds: surface.facts.bounds, + scale_factor: scale, + related_owner: Some((surface.facts.public, surface.facts.stamp)), + transform, + sample, + }) +} + +async fn capture_sample_until( + window_id: u32, + scale: f64, + deadline: Instant, +) -> Result { + let deadline = tokio::time::Instant::from_std(deadline); + let permit = tokio::time::timeout_at(deadline, SCK_CAPTURE_SLOTS.acquire()) + .await + .map_err(|_| capture_deadline_error(window_id))? + .map_err(|_| capture_deadline_error(window_id))?; + let worker = tokio::task::spawn_blocking(move || { + let _permit = permit; + capture_window_sample(window_id, scale) + }); + tokio::time::timeout_at(deadline, worker) + .await + .map_err(|_| capture_deadline_error(window_id))? + .map_err(|error| join_error("ScreenCaptureKit capture", error))? + .map_err(capture_error) +} + +fn capture_deadline_error(window_id: u32) -> NativeError { + NativeError::new( + ErrorCode::UnsupportedInBackground, + ErrorPhase::Preflight, + true, + "ScreenCaptureKit observation exceeded its bounded deadline", + ) + .with_detail("cg_window_id", window_id) +} + +async fn related_surfaces( + registry: &MacWindowRegistry, + facts: &MacWindowFacts, + related: &[RawRelatedWindow], +) -> Result, NativeError> { + let target_bounds = facts.bounds; + let registered = registry + .register_related_windows(facts, related_candidates(related)) + .await?; + let kinds: HashMap<_, _> = related + .iter() + .map(|related| (related.window_id, related.kind)) + .collect(); + let mut surfaces = Vec::with_capacity(registered.len()); + for related_facts in registered { + if !related_facts.is_on_screen { + return Err(NativeError::unsupported(format!( + "related AX surface {} is not on-screen", + related_facts.cg_window_id + ))); + } + let kind = kinds + .get(&related_facts.cg_window_id) + .copied() + .ok_or_else(|| { + NativeError::stale( + ErrorCode::SurfaceStale, + "registered related surface has no structural AX owner", + ) + })?; + surfaces.push(RelatedSurface { + facts: related_facts, + kind, + target_bounds, + }); + } + Ok(surfaces) +} + +fn validated_surface_transform( + sample: &WindowFrameSample, + source: Rect, + target: Rect, + expected_scale: f64, +) -> Result { + let metadata = &sample.metadata; + let scale = metadata.scale_factor.ok_or_else(missing_frame_metadata)?; + let content_scale = metadata.content_scale.ok_or_else(missing_frame_metadata)?; + let content = metadata.content_rect.ok_or_else(missing_frame_metadata)?; + let sampled = sample.source_frame; + let surface_width = f64::from(sample.pixel_width) / scale; + let surface_height = f64::from(sample.pixel_height) / scale; + let coherent = approximately(sampled.x, source.x) + && approximately(sampled.y, source.y) + && approximately(sampled.width, source.width) + && approximately(sampled.height, source.height) + && (scale - expected_scale).abs() <= 0.05 + && content_scale.is_finite() + && content_scale > 0.0 + && approximately(content.x, 0.0) + && approximately(content.y, 0.0) + && approximately(content.width, surface_width) + && approximately(content.height, surface_height) + && approximately(content.width / content_scale, sampled.width) + && approximately(content.height / content_scale, sampled.height); + if !coherent { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "ScreenCaptureKit source frame, content rectangle and raster geometry disagree", + )); + } + Ok(SurfaceToWindowTransform { + scale_x: 1.0 / (scale * content_scale), + scale_y: 1.0 / (scale * content_scale), + offset_x: source.x - target.x, + offset_y: source.y - target.y, + }) +} + +fn approximately(left: f64, right: f64) -> bool { + left.is_finite() && right.is_finite() && (left - right).abs() <= 0.75 +} + +#[derive(Default)] +pub struct MacFrameHistory { + previous: HashMap, +} + +struct PreviousFrame { + png: Vec, + completion_unix_ms: u64, + display_time: u64, +} + +impl MacFrameHistory { + fn classify_and_commit( + &mut self, + window_id: u32, + sample: &WindowFrameSample, + expected_scale: f64, + ) -> Result { + let metadata = &sample.metadata; + let status = metadata.frame_status.ok_or_else(missing_frame_metadata)?; + let display_time = metadata.display_time.ok_or_else(missing_frame_metadata)?; + let scale = metadata.scale_factor.ok_or_else(missing_frame_metadata)?; + let content_scale = metadata.content_scale.ok_or_else(missing_frame_metadata)?; + let content_rect = metadata.content_rect.ok_or_else(missing_frame_metadata)?; + if metadata.completion_unix_ms == 0 + || !scale.is_finite() + || (scale - expected_scale).abs() > 0.05 + || !content_scale.is_finite() + || content_scale <= 0.0 + || !content_rect.width.is_finite() + || !content_rect.height.is_finite() + || content_rect.width <= 0.0 + || content_rect.height <= 0.0 + { + return Err(missing_frame_metadata()); + } + + let freshness = if status.has_content() { + CaptureFreshness::Fresh + } else if status == SCFrameStatus::Idle { + match self.previous.get(&window_id) { + Some(previous) + if previous.png == sample.png_bytes + && metadata.completion_unix_ms > previous.completion_unix_ms + && display_time >= previous.display_time => + { + CaptureFreshness::ReusedWithFreshCompletion + } + Some(_) => CaptureFreshness::Frozen, + None => CaptureFreshness::Unavailable, + } + } else { + CaptureFreshness::Unavailable + }; + if freshness.action_safe() { + self.previous.insert( + window_id, + PreviousFrame { + png: sample.png_bytes.clone(), + completion_unix_ms: metadata.completion_unix_ms, + display_time, + }, + ); + } + Ok(freshness) + } +} + +fn missing_frame_metadata() -> NativeError { + NativeError::stale( + ErrorCode::SurfaceStale, + "same-sample ScreenCaptureKit freshness metadata is missing or incoherent", + ) +} + +pub struct RetainedAxElement(usize); + +impl RetainedAxElement { + unsafe fn retain(element: AXUIElementRef) -> Self { + CFRetain(element as CFTypeRef); + Self(element as usize) + } + + /// Adopt a reference returned under Core Foundation's create rule. + pub(crate) unsafe fn from_owned(element: AXUIElementRef) -> Self { + Self(element as usize) + } + + pub(crate) fn same_identity(&self, other: &Self) -> bool { + unsafe { CFEqual(self.0 as CFTypeRef, other.0 as CFTypeRef) != 0 } + } + + pub fn as_ptr(&self) -> AXUIElementRef { + self.0 as AXUIElementRef + } +} + +impl Clone for RetainedAxElement { + fn clone(&self) -> Self { + unsafe { CFRetain(self.0 as CFTypeRef) }; + Self(self.0) + } +} + +unsafe impl Send for RetainedAxElement {} +unsafe impl Sync for RetainedAxElement {} + +impl Drop for RetainedAxElement { + fn drop(&mut self) { + unsafe { CFRelease(self.0 as CFTypeRef) }; + } +} + +pub(crate) struct RetainedCfValue(usize); + +impl RetainedCfValue { + pub(crate) unsafe fn from_owned(value: CFTypeRef) -> Self { + Self(value as usize) + } + + pub(crate) fn same_value(&self, other: &Self) -> bool { + unsafe { CFEqual(self.0 as CFTypeRef, other.0 as CFTypeRef) != 0 } + } + + pub(crate) fn as_string(&self) -> Option { + let value = self.0 as CFTypeRef; + if unsafe { CFGetTypeID(value) } != CFString::type_id() { + return None; + } + Some(unsafe { CFString::wrap_under_get_rule(value as _) }.to_string()) + } +} + +impl Clone for RetainedCfValue { + fn clone(&self) -> Self { + unsafe { CFRetain(self.0 as CFTypeRef) }; + Self(self.0) + } +} + +unsafe impl Send for RetainedCfValue {} +unsafe impl Sync for RetainedCfValue {} + +impl Drop for RetainedCfValue { + fn drop(&mut self) { + unsafe { CFRelease(self.0 as CFTypeRef) }; + } +} + +struct RawAxNode { + element: RetainedAxElement, + parent: Option, + depth: usize, + owner_window_id: u32, + role: String, + role_proven: bool, + subrole: Option, + orientation: Option, + label: Option, + value: Option, + value_proof: Option, + value_query_proven: bool, + string_value: Option, + value_settable: Option, + selected_text_range: Option, + selected_text_range_settable: Option, + bounds: Option, + actions: Vec, + actions_proven: bool, + selected: bool, +} + +struct RawAxSnapshot { + nodes: Vec, + focused: Option, + selected_text: Option, + document_text: Option, + related_windows: Vec, + truncated: bool, +} + +impl RawAxSnapshot { + fn retained_elements(&self) -> Vec { + self.nodes.iter().map(|node| node.element.clone()).collect() + } +} + +struct RawRelatedWindow { + window_id: u32, + kind: SurfaceKind, + element: RetainedAxElement, +} + +fn related_candidates(related: &[RawRelatedWindow]) -> Vec<(u32, usize)> { + related + .iter() + .map(|related| (related.window_id, related.element.as_ptr() as usize)) + .collect() +} + +fn same_related_identities(left: &[RawRelatedWindow], right: &[RawRelatedWindow]) -> bool { + left.len() == right.len() + && left.iter().all(|candidate| { + right.iter().any(|current| { + candidate.window_id == current.window_id + && candidate.kind == current.kind + && candidate.element.same_identity(¤t.element) + }) + }) +} + +fn capture_ax_snapshot( + pid: i32, + target_window_id: u32, + include_text: bool, +) -> Result { + unsafe { + let application = AXUIElementCreateApplication(pid); + if application.is_null() { + return Err(ax_error("AX application root is unavailable")); + } + let windows = copy_ax_windows(application); + let target = windows + .iter() + .copied() + .find(|window| bindings::ax_get_window_id(*window) == Some(target_window_id)); + for window in windows + .iter() + .copied() + .filter(|window| Some(*window) != target) + { + CFRelease(window as CFTypeRef); + } + let Some(target) = target else { + CFRelease(application as CFTypeRef); + return Err(ax_error("exact AX target window is unavailable")); + }; + + let mut nodes = Vec::new(); + let mut related_windows = Vec::new(); + let mut visited = HashSet::new(); + let mut truncated = false; + walk_ax( + target, + None, + 0, + target_window_id, + target_window_id, + &mut nodes, + &mut related_windows, + &mut visited, + &mut truncated, + ); + if nodes.iter().any(|node| node.role == "AXMenu") { + CFRelease(target as CFTypeRef); + CFRelease(application as CFTypeRef); + return Err(NativeError::stale( + ErrorCode::MenuStateStale, + "AX menu detected without a canonical MenuId/action provenance; Plan003 refuses to publish point-actionable menu pixels or elements", + )); + } + if nodes.len() == 1 + && nodes[0].role == "AXWindow" + && nodes[0].actions.is_empty() + && nodes[0].value.is_none() + { + CFRelease(target as CFTypeRef); + CFRelease(application as CFTypeRef); + return Err(ax_error( + "target AX tree is not materialized; Plan003 observation is read-only and will not durably enable application accessibility", + )); + } + let focused = match focused_element_of_pid(pid) { + Some(element) if bindings::ax_get_window_id(element) == Some(target_window_id) => { + Some(RetainedAxElement(element as usize)) + } + Some(element) => { + CFRelease(element as CFTypeRef); + None + } + None => None, + }; + let selected_text = focused + .as_ref() + .and_then(|element| copy_string_attr(element.as_ptr(), "AXSelectedText")); + let document_text = include_text.then(|| { + nodes + .iter() + .filter(|node| { + matches!( + node.role.as_str(), + "AXStaticText" | "AXTextArea" | "AXWebArea" | "AXDocument" + ) + }) + .filter_map(|node| node.value.as_deref()) + .collect::>() + .join("\n") + }); + CFRelease(target as CFTypeRef); + CFRelease(application as CFTypeRef); + Ok(RawAxSnapshot { + nodes, + focused, + selected_text, + document_text, + related_windows, + truncated, + }) + } +} + +#[allow(clippy::too_many_arguments)] +unsafe fn walk_ax( + element: AXUIElementRef, + parent: Option, + depth: usize, + target_window_id: u32, + inherited_owner_window_id: u32, + nodes: &mut Vec, + related_windows: &mut Vec, + visited: &mut HashSet, + truncated: &mut bool, +) { + if depth > MAX_AX_DEPTH || nodes.len() >= MAX_AX_ELEMENTS { + *truncated = true; + return; + } + if !visited.insert(element as usize) { + return; + } + let (role, role_proven) = match copy_string_attr_exact(element, "AXRole") { + Ok(Some(role)) => (role, true), + _ => ("AXUnknown".to_owned(), false), + }; + let subrole = copy_string_attr(element, "AXSubrole"); + let orientation = copy_string_attr(element, "AXOrientation"); + let label = copy_string_attr(element, "AXTitle") + .filter(|value| !value.trim().is_empty()) + .or_else(|| copy_string_attr(element, "AXDescription")) + .filter(|value| !value.trim().is_empty()); + let (value_proof, value_query_proven) = match copy_attr_value(element, "AXValue") { + Ok(Some(value)) => (Some(RetainedCfValue::from_owned(value)), true), + Ok(None) => (None, true), + Err(_) => (None, false), + }; + let string_value = value_proof.as_ref().and_then(RetainedCfValue::as_string); + let value = string_value + .clone() + .or_else(|| copy_string_attr(element, "AXPlaceholderValue")); + let value_settable = string_value + .as_ref() + .and_then(|_| bindings::is_attribute_settable(element, "AXValue").ok()); + let selected_text_range = string_value.as_ref().and_then(|_| { + bindings::copy_cf_range_attr(element, "AXSelectedTextRange") + .ok() + .flatten() + }); + let selected_text_range_settable = selected_text_range + .as_ref() + .and_then(|_| bindings::is_attribute_settable(element, "AXSelectedTextRange").ok()); + let bounds = element_screen_rect(element).map(|frame| Rect { + x: frame[0], + y: frame[1], + width: frame[2], + height: frame[3], + }); + let native_window_id = bindings::ax_get_window_id(element); + let owner_window_id = native_window_id.unwrap_or(inherited_owner_window_id); + if let (Some(window_id), Some(kind)) = (native_window_id, transient_kind(&role)) { + if window_id != target_window_id { + related_windows.push(RawRelatedWindow { + window_id, + kind, + element: RetainedAxElement::retain(element), + }); + } + } + let retained = RetainedAxElement::retain(element); + let (actions, actions_proven) = match copy_action_names_exact(element) { + Ok(actions) => (actions, true), + Err(_) => (Vec::new(), false), + }; + nodes.push(RawAxNode { + element: retained.clone(), + parent, + depth, + owner_window_id, + role, + role_proven, + subrole, + orientation, + label, + value, + value_proof, + value_query_proven, + string_value, + value_settable, + selected_text_range, + selected_text_range_settable, + bounds, + actions, + actions_proven, + selected: bindings::copy_bool_attr(element, "AXSelected").unwrap_or(false), + }); + for child in copy_children(element) { + walk_ax( + child, + Some(retained.clone()), + depth + 1, + target_window_id, + owner_window_id, + nodes, + related_windows, + visited, + truncated, + ); + CFRelease(child as CFTypeRef); + if *truncated { + break; + } + } +} + +fn transient_kind(role: &str) -> Option { + match role { + "AXMenu" => Some(SurfaceKind::Menu), + "AXPopover" => Some(SurfaceKind::Popover), + "AXSheet" => Some(SurfaceKind::Sheet), + _ => None, + } +} + +struct RegisteredElement { + id: ElementId, + native: NativeElementHandle, + element: RetainedAxElement, + parent: Option, + owner: ResolvedWindowStamp, + owner_window_id: u32, + role: String, + role_proven: bool, + subrole: Option, + orientation: Option, + value_proof: Option, + value_query_proven: bool, + string_value: Option, + value_settable: Option, + selected_text_range: Option, + selected_text_range_settable: Option, + actions: Vec, + actions_proven: bool, +} + +#[derive(Clone)] +pub(crate) struct RegisteredElementSnapshot { + pub element: RetainedAxElement, + pub parent: Option, + pub owner: ResolvedWindowStamp, + pub owner_window_id: u32, + pub role: String, + pub role_proven: bool, + pub subrole: Option, + pub orientation: Option, + pub value_proof: Option, + pub value_query_proven: bool, + pub string_value: Option, + pub value_settable: Option, + pub selected_text_range: Option, + pub selected_text_range_settable: Option, + pub actions: Vec, + pub actions_proven: bool, +} + +#[derive(Default)] +pub struct MacElementRegistry { + elements: Vec, +} + +pub struct AxPublication { + update: NativeAccessibilityUpdate, + truncated: bool, +} + +impl MacElementRegistry { + fn reconcile( + &mut self, + snapshot: RawAxSnapshot, + window_bounds: Rect, + window_id: u32, + target_owner: &ResolvedWindowStamp, + related_windows: &[MacRelatedWindowFacts], + ) -> Result { + let old = std::mem::take(&mut self.elements); + let mut used = HashSet::new(); + let mut next = Vec::with_capacity(snapshot.nodes.len()); + let mut manifest = Vec::with_capacity(snapshot.nodes.len()); + let mut lines = Vec::with_capacity(snapshot.nodes.len()); + let mut focused_element = None; + let mut selected_elements = Vec::new(); + + for node in snapshot.nodes { + let prior = old.iter().enumerate().find(|(index, candidate)| { + !used.contains(index) && candidate.element.same_identity(&node.element) + }); + let (id, native) = if let Some((index, prior)) = prior { + used.insert(index); + (prior.id.clone(), prior.native.clone()) + } else { + let id = ElementId::new(); + let native = + NativeElementHandle::new(format!("macos-ax:{window_id}:{}", id.as_str()))?; + (id, native) + }; + if snapshot + .focused + .as_ref() + .is_some_and(|focused| focused.same_identity(&node.element)) + { + focused_element = Some(id.clone()); + } + if node.selected { + selected_elements.push(id.clone()); + } + let bounds = node.bounds.map(|bounds| Rect { + x: bounds.x - window_bounds.x, + y: bounds.y - window_bounds.y, + width: bounds.width, + height: bounds.height, + }); + let owner = if node.owner_window_id == window_id { + target_owner.clone() + } else { + related_windows + .iter() + .find(|related| related.cg_window_id == node.owner_window_id) + .map(|related| related.stamp.clone()) + .ok_or_else(|| { + NativeError::stale( + ErrorCode::ElementStale, + "AX element belongs to an unregistered native window", + ) + .with_detail("cg_window_id", node.owner_window_id) + })? + }; + lines.push(render_ax_line(node.depth, &id, &node)); + manifest.push(NativeAccessibilityElement { + id: id.clone(), + native: native.clone(), + owner: owner.clone(), + role: Some(node.role.clone()), + label: node.label, + value: node.value, + bounds, + actions: node.actions.clone(), + menu_id: None, + }); + next.push(RegisteredElement { + id, + native, + element: node.element, + parent: node.parent, + owner, + owner_window_id: node.owner_window_id, + role: node.role, + role_proven: node.role_proven, + subrole: node.subrole, + orientation: node.orientation, + value_proof: node.value_proof, + value_query_proven: node.value_query_proven, + string_value: node.string_value, + value_settable: node.value_settable, + selected_text_range: node.selected_text_range, + selected_text_range_settable: node.selected_text_range_settable, + actions: node.actions, + actions_proven: node.actions_proven, + }); + } + self.elements = next; + Ok(AxPublication { + update: NativeAccessibilityUpdate { + normalized_tree: lines.join("\n"), + elements: manifest, + focused_element, + selected_text: snapshot.selected_text, + selected_elements, + document_text: snapshot.document_text, + }, + truncated: snapshot.truncated, + }) + } + + pub(crate) fn registered( + &self, + native: &NativeElementHandle, + id: &ElementId, + ) -> Option { + self.elements + .iter() + .find(|element| &element.native == native && &element.id == id) + .map(|element| RegisteredElementSnapshot { + element: element.element.clone(), + parent: element.parent.clone(), + owner: element.owner.clone(), + owner_window_id: element.owner_window_id, + role: element.role.clone(), + role_proven: element.role_proven, + subrole: element.subrole.clone(), + orientation: element.orientation.clone(), + value_proof: element.value_proof.clone(), + value_query_proven: element.value_query_proven, + string_value: element.string_value.clone(), + value_settable: element.value_settable, + selected_text_range: element.selected_text_range, + selected_text_range_settable: element.selected_text_range_settable, + actions: element.actions.clone(), + actions_proven: element.actions_proven, + }) + } + + pub(crate) fn registered_snapshots(&self) -> Vec { + self.elements + .iter() + .map(|element| RegisteredElementSnapshot { + element: element.element.clone(), + parent: element.parent.clone(), + owner: element.owner.clone(), + owner_window_id: element.owner_window_id, + role: element.role.clone(), + role_proven: element.role_proven, + subrole: element.subrole.clone(), + orientation: element.orientation.clone(), + value_proof: element.value_proof.clone(), + value_query_proven: element.value_query_proven, + string_value: element.string_value.clone(), + value_settable: element.value_settable, + selected_text_range: element.selected_text_range, + selected_text_range_settable: element.selected_text_range_settable, + actions: element.actions.clone(), + actions_proven: element.actions_proven, + }) + .collect() + } +} + +fn render_ax_line(depth: usize, id: &ElementId, node: &RawAxNode) -> String { + let mut line = format!( + "{}- [{}] {}", + " ".repeat(depth), + id.as_str(), + sanitize(&node.role) + ); + if let Some(label) = &node.label { + line.push_str(&format!(" label={:?}", sanitize(label))); + } + if let Some(value) = &node.value { + line.push_str(&format!(" value={:?}", sanitize(value))); + } + if !node.actions.is_empty() { + line.push_str(&format!(" actions={:?}", node.actions)); + } + line +} + +fn sanitize(value: &str) -> String { + value + .chars() + .map(|character| { + if character.is_control() { + ' ' + } else { + character + } + }) + .collect() +} + +fn materialize_png( + root: &Path, + window_id: u32, + bytes: &[u8], +) -> Result<(String, ObservationArtifactHandle), NativeError> { + let mut directory = std::fs::DirBuilder::new(); + directory.recursive(true).mode(0o700); + directory.create(root).map_err(artifact_error)?; + std::fs::set_permissions(root, std::fs::Permissions::from_mode(0o700)) + .map_err(artifact_error)?; + let path = root.join(format!( + "observation-{window_id}-{}.png", + uuid::Uuid::new_v4() + )); + let temporary = root.join(format!( + ".observation-{window_id}-{}.partial", + uuid::Uuid::new_v4() + )); + let mut temporary_cleanup = TemporaryArtifact::new(temporary.clone()); + let mut file = OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&temporary) + .map_err(artifact_error)?; + file.write_all(bytes).map_err(artifact_error)?; + file.sync_data().map_err(artifact_error)?; + drop(file); + std::fs::rename(&temporary, &path).map_err(artifact_error)?; + temporary_cleanup.disarm(); + let url = format!("file://{}", path.display()); + let cleanup_path = path.clone(); + let artifact = ObservationArtifactHandle::new(path.display().to_string(), move || { + match std::fs::remove_file(&cleanup_path) { + Ok(()) => Ok(()), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(()), + Err(error) => Err(artifact_error(error)), + } + }); + Ok((url, artifact)) +} + +struct TemporaryArtifact(Option); + +impl TemporaryArtifact { + fn new(path: PathBuf) -> Self { + Self(Some(path)) + } + + fn disarm(&mut self) { + self.0 = None; + } +} + +impl Drop for TemporaryArtifact { + fn drop(&mut self) { + if let Some(path) = self.0.take() { + let _ = std::fs::remove_file(path); + } + } +} + +fn now_unix_ms() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .min(u128::from(u64::MAX)) as u64 +} + +fn capture_error(error: anyhow::Error) -> NativeError { + NativeError::new( + ErrorCode::UnsupportedInBackground, + ErrorPhase::Preflight, + true, + format!("native window capture unavailable: {error}"), + ) +} + +fn artifact_error(error: std::io::Error) -> NativeError { + NativeError::new( + ErrorCode::Internal, + ErrorPhase::Verify, + true, + format!("observation artifact I/O failed: {error}"), + ) +} + +fn ax_error(message: impl Into) -> NativeError { + NativeError::new( + ErrorCode::UnsupportedInBackground, + ErrorPhase::Preflight, + true, + message, + ) +} + +fn join_error(operation: &str, error: tokio::task::JoinError) -> NativeError { + NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + true, + format!("{operation} worker failed: {error}"), + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::video_sckit::{FrameRect, WindowFrameMetadata}; + use core_foundation::{base::TCFType, string::CFString}; + use cua_driver_core::api::{ + contracts::{AppId, GeometryRevision, WindowGeneration, WindowId}, + observation::{NativeProcessHandle, NativeWindowHandle}, + }; + + fn sample(status: SCFrameStatus, completion: u64, bytes: &[u8]) -> WindowFrameSample { + WindowFrameSample { + png_bytes: bytes.to_vec(), + pixel_width: 200, + pixel_height: 100, + source_frame: FrameRect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 50.0, + }, + metadata: WindowFrameMetadata { + completion_unix_ms: completion, + display_time: Some(completion), + frame_status: Some(status), + scale_factor: Some(2.0), + content_scale: Some(1.0), + content_rect: Some(FrameRect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 50.0, + }), + }, + } + } + + #[test] + fn static_pixels_require_fresh_completion_metadata() { + let mut history = MacFrameHistory::default(); + assert_eq!( + history.classify_and_commit(1, &sample(SCFrameStatus::Complete, 10, b"same"), 2.0), + Ok(CaptureFreshness::Fresh) + ); + assert_eq!( + history.classify_and_commit(1, &sample(SCFrameStatus::Idle, 11, b"same"), 2.0), + Ok(CaptureFreshness::ReusedWithFreshCompletion) + ); + assert_eq!( + history.classify_and_commit(1, &sample(SCFrameStatus::Idle, 11, b"same"), 2.0), + Ok(CaptureFreshness::Frozen) + ); + } + + #[test] + fn related_surface_transform_maps_pixels_into_target_window_points() { + let mut frame = sample(SCFrameStatus::Complete, 10, b"pixels"); + frame.source_frame = FrameRect { + x: 120.0, + y: 80.0, + width: 100.0, + height: 50.0, + }; + let transform = validated_surface_transform( + &frame, + Rect { + x: 120.0, + y: 80.0, + width: 100.0, + height: 50.0, + }, + Rect { + x: 100.0, + y: 50.0, + width: 400.0, + height: 300.0, + }, + 2.0, + ) + .unwrap(); + let point = + transform.transform(cua_driver_core::api::contracts::Point { x: 20.0, y: 10.0 }); + assert_eq!(point.x, 30.0); + assert_eq!(point.y, 35.0); + } + + #[test] + fn surface_transform_refuses_unrepresented_crop_or_source_geometry() { + let bounds = Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 50.0, + }; + let mut cropped = sample(SCFrameStatus::Complete, 10, b"pixels"); + cropped.metadata.content_rect.as_mut().unwrap().x = 2.0; + let crop_error = validated_surface_transform(&cropped, bounds, bounds, 2.0).unwrap_err(); + assert_eq!(crop_error.code, ErrorCode::SurfaceStale); + + let mut wrong_source = sample(SCFrameStatus::Complete, 11, b"pixels"); + wrong_source.source_frame.x = 10.0; + let source_error = + validated_surface_transform(&wrong_source, bounds, bounds, 2.0).unwrap_err(); + assert_eq!(source_error.code, ErrorCode::SurfaceStale); + } + + #[test] + fn materialized_artifact_is_private_atomic_and_cleanup_owned() { + let root = std::env::temp_dir().join(format!( + "cua-observation-artifact-test-{}", + uuid::Uuid::new_v4() + )); + let (_, artifact) = materialize_png(&root, 7, b"fixture-bytes").unwrap(); + let path = PathBuf::from(artifact.label()); + assert_eq!(std::fs::read(&path).unwrap(), b"fixture-bytes"); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + assert!(std::fs::read_dir(&root).unwrap().all(|entry| !entry + .unwrap() + .file_name() + .to_string_lossy() + .ends_with(".partial"))); + drop(artifact); + assert!(!path.exists()); + std::fs::remove_dir(&root).unwrap(); + } + + fn raw_snapshot(element: &CFString, label: &str) -> RawAxSnapshot { + let pointer = element.as_CFTypeRef().cast_mut().cast(); + RawAxSnapshot { + nodes: vec![RawAxNode { + element: unsafe { RetainedAxElement::retain(pointer) }, + parent: None, + depth: 0, + owner_window_id: 1, + role: "AXButton".to_owned(), + role_proven: true, + subrole: None, + orientation: None, + label: Some(label.to_owned()), + value: None, + value_proof: None, + value_query_proven: true, + string_value: None, + value_settable: Some(false), + selected_text_range: None, + selected_text_range_settable: Some(false), + bounds: None, + actions: vec!["AXPress".to_owned()], + actions_proven: true, + selected: false, + }], + focused: None, + selected_text: None, + document_text: None, + related_windows: Vec::new(), + truncated: false, + } + } + + #[test] + fn stable_ids_use_native_cf_identity_not_label_or_index() { + let first_native = CFString::new("native-identity-one"); + let replacement_native = CFString::new("native-identity-two"); + let mut registry = MacElementRegistry::default(); + let bounds = Rect { + x: 0.0, + y: 0.0, + width: 100.0, + height: 100.0, + }; + let owner = stamp("geometry", "native"); + let first = registry + .reconcile( + raw_snapshot(&first_native, "Same label"), + bounds, + 1, + &owner, + &[], + ) + .unwrap(); + let stable = registry + .reconcile( + raw_snapshot(&first_native, "Renamed"), + bounds, + 1, + &owner, + &[], + ) + .unwrap(); + let replacement = registry + .reconcile( + raw_snapshot(&replacement_native, "Same label"), + bounds, + 1, + &owner, + &[], + ) + .unwrap(); + + assert_eq!(first.update.elements[0].id, stable.update.elements[0].id); + assert_ne!( + stable.update.elements[0].id, + replacement.update.elements[0].id + ); + } + + fn stamp(geometry: &str, native: &str) -> ResolvedWindowStamp { + ResolvedWindowStamp { + app_id: AppId::parse("app").unwrap(), + window_id: WindowId::parse("window").unwrap(), + generation: WindowGeneration(7), + geometry_revision: GeometryRevision::parse(geometry).unwrap(), + native_window: NativeWindowHandle::new(native).unwrap(), + process: NativeProcessHandle::new("process").unwrap(), + } + } + + fn fake_facts(geometry: &str, x: f64) -> MacWindowFacts { + MacWindowFacts { + stamp: stamp(geometry, "native"), + pid: 42, + process_generation: 9, + cg_window_id: 7, + owner_name: "Fixture".to_owned(), + layer: 0, + bounds: Rect { + x, + y: 0.0, + width: 100.0, + height: 50.0, + }, + scale_factor: Some(2.0), + state: WindowStateKind::Visible, + is_on_screen: true, + on_current_space: Some(true), + space_ids: Some(vec![1]), + minimized: Some(false), + } + } + + #[test] + fn fake_stamp_or_epoch_race_invalidates_the_observation_bracket() { + let target = stamp("geometry-a", "native"); + let facts_a = fake_facts("geometry-a", 0.0); + assert!(coherent_window_bracket(&target, &facts_a, &facts_a, 4, 4)); + assert!(!coherent_window_bracket( + &target, + &facts_a, + &fake_facts("geometry-b", 1.0), + 4, + 4 + )); + assert!(!coherent_window_bracket(&target, &facts_a, &facts_a, 4, 5)); + } + + #[test] + fn refreshed_geometry_preserves_stable_identity_but_native_reuse_does_not() { + assert!(same_stable_identity( + &stamp("geometry-a", "native"), + &stamp("geometry-b", "native") + )); + assert!(!same_stable_identity( + &stamp("geometry-a", "native"), + &stamp("geometry-b", "replacement") + )); + } + + #[test] + fn missing_same_sample_metadata_is_not_action_safe() { + let mut history = MacFrameHistory::default(); + let mut incomplete = sample(SCFrameStatus::Complete, 10, b"pixels"); + incomplete.metadata.display_time = None; + assert_eq!( + history + .classify_and_commit(1, &incomplete, 2.0) + .unwrap_err() + .code, + ErrorCode::SurfaceStale + ); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/posture.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/posture.rs new file mode 100644 index 0000000000..3e531bc84b --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/posture.rs @@ -0,0 +1,1000 @@ +use std::{ + ffi::c_void, + sync::{ + atomic::{AtomicUsize, Ordering}, + mpsc, Arc, Mutex, + }, + thread, + time::{Duration, Instant}, +}; + +use core_foundation::{ + base::{CFGetTypeID, CFRelease, CFTypeRef, TCFType}, + runloop::{ + kCFRunLoopCommonModes, kCFRunLoopDefaultMode, CFRunLoop, CFRunLoopAddSource, + CFRunLoopGetCurrent, CFRunLoopRef, CFRunLoopRemoveSource, CFRunLoopWakeUp, + }, + string::{CFString, CFStringRef}, +}; +use core_graphics::event::{ + CGEventTap, CGEventTapLocation, CGEventTapOptions, CGEventTapPlacement, CGEventType, +}; +use cua_driver_core::api::{ + errors::{ErrorCode, ErrorPhase, NativeError}, + interaction::{NativeEvidence, PostureResult}, +}; + +use crate::{ + apps::{ + self, + nsworkspace::{WorkspaceEvent, WorkspaceEventHub, WorkspaceEventKind}, + }, + ax::bindings::{ + self, kAXErrorNoValue, kAXErrorSuccess, AXObserverAddNotification, AXObserverCreate, + AXObserverGetRunLoopSource, AXObserverRef, AXUIElementCopyAttributeValue, + AXUIElementCreateApplication, AXUIElementGetTypeID, AXUIElementRef, + }, + focus_steal::{DeadlineSuppression, SuppressionCloseOutcome, SuppressionOutcome}, +}; + +const NATIVE_BARRIER_TIMEOUT: Duration = Duration::from_millis(250); +const OBSERVER_HANDSHAKE_TIMEOUT: Duration = Duration::from_millis(750); +const OBSERVER_CLOSE_TIMEOUT: Duration = Duration::from_millis(750); +const OBSERVER_RUN_LOOP_INTERVAL: Duration = Duration::from_millis(10); + +/// Ordered, content-free posture evidence. The before/after samples are a +/// diagnostic and final-state cross-check; transient proof comes from the +/// native workspace, AX and HID streams. +#[derive(Debug, Default)] +struct PostureProofState { + frontmost_changed: bool, + key_window_changed: bool, + physical_cursor_moved: bool, + workspace_stream_complete: bool, + ax_stream_complete: bool, + hid_stream_complete: bool, + containment_stream_complete: bool, + activation_pids: Vec, + workspace_lagged_events: u64, + ax_focus_events: u64, + hid_cursor_events: u64, +} + +impl PostureProofState { + fn new() -> Self { + Self { + workspace_stream_complete: true, + ax_stream_complete: true, + hid_stream_complete: true, + containment_stream_complete: true, + ..Self::default() + } + } + + fn observe_workspace( + &mut self, + receiver: &mut tokio::sync::broadcast::Receiver, + baseline_frontmost: i32, + ) { + loop { + match receiver.try_recv() { + Ok(event) if event.kind == WorkspaceEventKind::Activated => { + if event.pid != baseline_frontmost { + self.frontmost_changed = true; + self.activation_pids.push(event.pid); + } + } + Ok(_) => {} + Err(tokio::sync::broadcast::error::TryRecvError::Lagged(skipped)) => { + self.workspace_stream_complete = false; + self.workspace_lagged_events = + self.workspace_lagged_events.saturating_add(skipped); + } + Err(tokio::sync::broadcast::error::TryRecvError::Closed) => { + self.workspace_stream_complete = false; + break; + } + Err(tokio::sync::broadcast::error::TryRecvError::Empty) => break, + } + } + } + + fn observe_final_sample(&mut self, baseline: PostureSample, final_sample: PostureSample) { + if final_sample.frontmost.is_none() + || final_sample.key_window.is_err() + || final_sample.cursor.is_err() + { + self.workspace_stream_complete = false; + self.ax_stream_complete = false; + self.hid_stream_complete = false; + } + self.frontmost_changed |= matches!( + (baseline.frontmost, final_sample.frontmost), + (Some(before), Some(after)) if before != after + ); + self.key_window_changed |= matches!( + (baseline.key_window, final_sample.key_window), + (Ok(before), Ok(after)) if before != after + ); + self.physical_cursor_moved |= matches!( + (baseline.cursor, final_sample.cursor), + (Ok(before), Ok(after)) if before != after + ); + } + + fn complete(&self) -> bool { + self.workspace_stream_complete + && self.ax_stream_complete + && self.hid_stream_complete + && self.containment_stream_complete + } + + fn posture(&self, baseline: PostureSample, final_sample: PostureSample) -> PostureResult { + let excursion = + self.frontmost_changed || self.key_window_changed || self.physical_cursor_moved; + let restored = excursion + && baseline.frontmost == final_sample.frontmost + && baseline.key_window == final_sample.key_window + && baseline.cursor == final_sample.cursor; + PostureResult { + held: self.complete() && !excursion, + frontmost_changed: self.frontmost_changed, + key_window_changed: self.key_window_changed, + physical_cursor_moved: self.physical_cursor_moved, + restored_after_violation: restored, + } + } +} + +enum ObserverCommand { + Close { + deadline: Instant, + reply: mpsc::SyncSender, + }, +} + +struct ObserverControl { + commands: mpsc::Sender, + run_loop: Arc, + thread: Option>, + closed: bool, +} + +impl ObserverControl { + fn close(&mut self, timeout: Duration) -> bool { + if self.closed { + return true; + } + self.closed = true; + let (reply_tx, reply_rx) = mpsc::sync_channel(1); + let native_deadline = Instant::now() + timeout.saturating_sub(OBSERVER_RUN_LOOP_INTERVAL); + let sent = self + .commands + .send(ObserverCommand::Close { + deadline: native_deadline, + reply: reply_tx, + }) + .is_ok(); + let run_loop = self.run_loop.load(Ordering::Acquire) as CFRunLoopRef; + if !run_loop.is_null() { + unsafe { CFRunLoopWakeUp(run_loop) }; + } + let closed = sent && reply_rx.recv_timeout(timeout).unwrap_or(false); + // Never join a possibly wedged AX or HID native thread. Dropping a + // JoinHandle detaches it; the thread owns its callback context and will + // free it if/when the native run loop returns. + self.thread.take(); + closed + } +} + +impl Drop for ObserverControl { + fn drop(&mut self) { + let _ = self.close(OBSERVER_CLOSE_TIMEOUT); + } +} + +struct AxFocusContext { + state: Arc>, +} + +unsafe extern "C" fn ax_focus_callback( + _observer: AXObserverRef, + _element: AXUIElementRef, + notification: CFStringRef, + refcon: *mut c_void, +) { + if notification.is_null() || refcon.is_null() { + return; + } + let notification = CFString::wrap_under_get_rule(notification).to_string(); + if !matches!( + notification.as_str(), + "AXFocusedWindowChanged" | "AXFocusedUIElementChanged" + ) { + return; + } + let context = &*(refcon as *const AxFocusContext); + let mut state = context + .state + .lock() + .expect("posture AX proof state poisoned"); + state.key_window_changed = true; + state.ax_focus_events = state.ax_focus_events.saturating_add(1); +} + +fn start_ax_focus_stream( + pid: i32, + state: Arc>, +) -> Result { + let run_loop = Arc::new(AtomicUsize::new(0)); + let thread_run_loop = Arc::clone(&run_loop); + let (started_tx, started_rx) = mpsc::sync_channel(1); + let (command_tx, command_rx) = mpsc::channel(); + let thread = thread::Builder::new() + .name(format!("cua-posture-ax-{pid}")) + .spawn(move || unsafe { + run_ax_focus_stream(pid, state, &thread_run_loop, &started_tx, &command_rx) + }) + .map_err(|error| { + posture_preflight(format!("failed to spawn AX posture stream: {error}")) + })?; + let mut control = ObserverControl { + commands: command_tx, + run_loop, + thread: Some(thread), + closed: false, + }; + match started_rx.recv_timeout(OBSERVER_HANDSHAKE_TIMEOUT) { + Ok(Ok(())) => Ok(control), + Ok(Err(error)) => { + let _ = control.close(OBSERVER_CLOSE_TIMEOUT); + Err(error) + } + Err(_) => { + let _ = control.close(OBSERVER_CLOSE_TIMEOUT); + Err(posture_preflight( + "AX posture stream registration exceeded its bounded deadline", + )) + } + } +} + +unsafe fn run_ax_focus_stream( + pid: i32, + state: Arc>, + run_loop_slot: &AtomicUsize, + started: &mpsc::SyncSender>, + commands: &mpsc::Receiver, +) { + let application = AXUIElementCreateApplication(pid); + if application.is_null() { + let _ = started.send(Err(posture_preflight( + "foreground AX application element is unavailable", + ))); + return; + } + let context = Box::into_raw(Box::new(AxFocusContext { state })); + let mut observer = std::ptr::null_mut(); + let create_error = AXObserverCreate(pid, ax_focus_callback, &mut observer); + if create_error != kAXErrorSuccess || observer.is_null() { + drop(Box::from_raw(context)); + CFRelease(application as CFTypeRef); + let _ = started.send(Err(posture_preflight(format!( + "foreground AX posture observer creation failed with {create_error}" + )))); + return; + } + let notifications = ["AXFocusedWindowChanged", "AXFocusedUIElementChanged"]; + let mut registered = 0usize; + for notification in notifications { + let notification = CFString::new(notification); + if AXObserverAddNotification( + observer, + application, + notification.as_concrete_TypeRef(), + context.cast(), + ) == kAXErrorSuccess + { + registered += 1; + } + } + let source = AXObserverGetRunLoopSource(observer); + if registered != notifications.len() || source.is_null() { + remove_ax_focus_notifications(observer, application, ¬ifications); + CFRelease(observer as CFTypeRef); + drop(Box::from_raw(context)); + CFRelease(application as CFTypeRef); + let _ = started.send(Err(posture_preflight( + "foreground AX focus/key event stream is incomplete", + ))); + return; + } + let run_loop = CFRunLoopGetCurrent(); + run_loop_slot.store(run_loop as usize, Ordering::Release); + CFRunLoopAddSource(run_loop, source, kCFRunLoopCommonModes); + if started.send(Ok(())).is_err() { + remove_ax_focus_notifications(observer, application, ¬ifications); + CFRunLoopRemoveSource(run_loop, source, kCFRunLoopCommonModes); + run_loop_slot.store(0, Ordering::Release); + CFRelease(observer as CFTypeRef); + drop(Box::from_raw(context)); + CFRelease(application as CFTypeRef); + return; + } + + let close_reply = loop { + CFRunLoop::run_in_mode( + unsafe { kCFRunLoopDefaultMode }, + OBSERVER_RUN_LOOP_INTERVAL, + true, + ); + match commands.try_recv() { + Ok(ObserverCommand::Close { deadline, reply }) => { + break Some((reply, drain_native_run_loop(deadline))) + } + Err(mpsc::TryRecvError::Disconnected) => break None, + Err(mpsc::TryRecvError::Empty) => {} + } + }; + remove_ax_focus_notifications(observer, application, ¬ifications); + CFRunLoopRemoveSource(run_loop, source, kCFRunLoopCommonModes); + run_loop_slot.store(0, Ordering::Release); + CFRelease(observer as CFTypeRef); + drop(Box::from_raw(context)); + CFRelease(application as CFTypeRef); + if let Some((reply, drained)) = close_reply { + let _ = reply.send(drained); + } +} + +unsafe fn remove_ax_focus_notifications( + observer: AXObserverRef, + application: AXUIElementRef, + notifications: &[&str], +) { + for notification in notifications { + let notification = CFString::new(notification); + let _ = bindings::AXObserverRemoveNotification( + observer, + application, + notification.as_concrete_TypeRef(), + ); + } +} + +fn start_hid_cursor_stream( + state: Arc>, +) -> Result { + let run_loop = Arc::new(AtomicUsize::new(0)); + let thread_run_loop = Arc::clone(&run_loop); + let (started_tx, started_rx) = mpsc::sync_channel(1); + let (command_tx, command_rx) = mpsc::channel(); + let thread = thread::Builder::new() + .name("cua-posture-hid-cursor".to_owned()) + .spawn(move || run_hid_cursor_stream(state, &thread_run_loop, &started_tx, &command_rx)) + .map_err(|error| { + posture_preflight(format!("failed to spawn HID posture stream: {error}")) + })?; + let mut control = ObserverControl { + commands: command_tx, + run_loop, + thread: Some(thread), + closed: false, + }; + match started_rx.recv_timeout(OBSERVER_HANDSHAKE_TIMEOUT) { + Ok(Ok(())) => Ok(control), + Ok(Err(error)) => { + let _ = control.close(OBSERVER_CLOSE_TIMEOUT); + Err(error) + } + Err(_) => { + let _ = control.close(OBSERVER_CLOSE_TIMEOUT); + Err(posture_preflight( + "HID cursor stream registration exceeded its bounded deadline", + )) + } + } +} + +fn run_hid_cursor_stream( + state: Arc>, + run_loop_slot: &AtomicUsize, + started: &mpsc::SyncSender>, + commands: &mpsc::Receiver, +) { + let callback_state = Arc::clone(&state); + let tap = match CGEventTap::new( + CGEventTapLocation::HID, + CGEventTapPlacement::HeadInsertEventTap, + CGEventTapOptions::ListenOnly, + vec![ + CGEventType::MouseMoved, + CGEventType::LeftMouseDragged, + CGEventType::RightMouseDragged, + CGEventType::OtherMouseDragged, + ], + move |_proxy, event_type, _event| { + let mut state = callback_state + .lock() + .expect("posture HID proof state poisoned"); + match event_type { + CGEventType::MouseMoved + | CGEventType::LeftMouseDragged + | CGEventType::RightMouseDragged + | CGEventType::OtherMouseDragged => { + state.physical_cursor_moved = true; + state.hid_cursor_events = state.hid_cursor_events.saturating_add(1); + } + CGEventType::TapDisabledByTimeout | CGEventType::TapDisabledByUserInput => { + state.hid_stream_complete = false; + } + _ => {} + } + None + }, + ) { + Ok(tap) => tap, + Err(()) => { + let _ = started.send(Err(posture_preflight( + "listen-only HID cursor event tap is unavailable", + ))); + return; + } + }; + let source = match tap.mach_port.create_runloop_source(0) { + Ok(source) => source, + Err(()) => { + let _ = started.send(Err(posture_preflight( + "listen-only HID cursor event tap has no run-loop source", + ))); + return; + } + }; + let run_loop = CFRunLoop::get_current(); + run_loop_slot.store(run_loop.as_concrete_TypeRef() as usize, Ordering::Release); + run_loop.add_source(&source, unsafe { kCFRunLoopCommonModes }); + tap.enable(); + if started.send(Ok(())).is_err() { + run_loop.remove_source(&source, unsafe { kCFRunLoopCommonModes }); + run_loop_slot.store(0, Ordering::Release); + return; + } + + let close_reply = loop { + CFRunLoop::run_in_mode( + unsafe { kCFRunLoopDefaultMode }, + OBSERVER_RUN_LOOP_INTERVAL, + true, + ); + match commands.try_recv() { + Ok(ObserverCommand::Close { deadline, reply }) => { + break Some((reply, drain_native_run_loop(deadline))) + } + Err(mpsc::TryRecvError::Disconnected) => break None, + Err(mpsc::TryRecvError::Empty) => {} + } + }; + run_loop.remove_source(&source, unsafe { kCFRunLoopCommonModes }); + run_loop_slot.store(0, Ordering::Release); + if let Some((reply, drained)) = close_reply { + let _ = reply.send(drained); + } +} + +fn drain_native_run_loop(deadline: Instant) -> bool { + while Instant::now() < deadline { + let result = CFRunLoop::run_in_mode( + unsafe { kCFRunLoopDefaultMode }, + Duration::from_millis(1), + true, + ); + if result != core_foundation::runloop::CFRunLoopRunResult::HandledSource { + return true; + } + } + false +} + +pub struct MacLaunchPostureWitness { + baseline: PostureSample, + target_pid: Option, + events: tokio::sync::broadcast::Receiver, + state: Arc>, + ax_stream: ObserverControl, + hid_stream: ObserverControl, + containment: DeadlineSuppression, + finished: bool, +} + +impl MacLaunchPostureWitness { + pub fn begin(deadline: Instant) -> Result { + let workspace = WorkspaceEventHub::shared(); + let mut events = workspace.subscribe(); + let initial_frontmost = apps::frontmost_pid().ok_or_else(|| { + posture_preflight("exact foreground baseline pid is unavailable before launch") + })?; + let state = Arc::new(Mutex::new(PostureProofState::new())); + let ax_stream = start_ax_focus_stream(initial_frontmost, Arc::clone(&state))?; + let hid_stream = start_hid_cursor_stream(Arc::clone(&state))?; + if !workspace.barrier(NATIVE_BARRIER_TIMEOUT) + || !crate::focus_steal::FocusStealPreventer::barrier(NATIVE_BARRIER_TIMEOUT) + { + return Err(posture_preflight( + "native posture callback queues were unavailable before launch dispatch", + )); + } + let baseline = exact_posture_sample(); + if !baseline.complete() || baseline.frontmost != Some(initial_frontmost) { + return Err(posture_preflight( + "foreground, AX focus/key, or cursor baseline changed during witness acquisition", + )); + } + { + let mut proof = state.lock().expect("launch posture proof state poisoned"); + proof.observe_workspace(&mut events, initial_frontmost); + if proof.frontmost_changed + || proof.key_window_changed + || proof.physical_cursor_moved + || !proof.complete() + { + return Err(posture_preflight( + "foreground, AX focus/key, or HID cursor changed during witness acquisition", + )); + } + } + let containment = crate::focus_steal::begin_deadline_owned_suppression_until( + None, + initial_frontmost, + "driver.v2.launch.deadline_owned", + deadline, + ); + Ok(Self { + baseline, + target_pid: None, + events, + state, + ax_stream, + hid_stream, + containment, + finished: false, + }) + } + + /// Clone for the LaunchServices completion block. If the awaiting task is + /// cancelled, this handle still narrows the detached wildcard and the + /// dispatcher retains it through the native deadline. + pub fn completion_containment(&self) -> DeadlineSuppression { + self.containment.clone() + } + + pub fn set_target(&mut self, pid: i32) -> Result<(), NativeError> { + self.target_pid = Some(pid); + if self + .containment + .narrow_to_target(pid, NATIVE_BARRIER_TIMEOUT) + { + Ok(()) + } else { + self.state + .lock() + .expect("launch posture proof state poisoned") + .containment_stream_complete = false; + Err(posture_preflight( + "launch containment wildcard could not be narrowed on its serial callback queue", + )) + } + } + + pub fn drain_events(&mut self) { + let baseline_frontmost = self + .baseline + .frontmost + .expect("launch posture baseline was validated"); + self.state + .lock() + .expect("launch posture proof state poisoned") + .observe_workspace(&mut self.events, baseline_frontmost); + } + + pub fn finish(mut self) -> (PostureResult, NativeEvidence) { + let containment = self.containment.close_with_evidence(NATIVE_BARRIER_TIMEOUT); + let workspace_barrier = WorkspaceEventHub::shared().barrier(NATIVE_BARRIER_TIMEOUT); + self.drain_events(); + let ax_barrier = self.ax_stream.close(OBSERVER_CLOSE_TIMEOUT); + let hid_barrier = self.hid_stream.close(OBSERVER_CLOSE_TIMEOUT); + let final_sample = exact_posture_sample(); + + let mut state = self + .state + .lock() + .expect("launch posture proof state poisoned"); + state.workspace_stream_complete &= workspace_barrier; + state.ax_stream_complete &= ax_barrier; + state.hid_stream_complete &= hid_barrier; + state.containment_stream_complete &= containment.callback_queue_drained; + if containment.evidence.activations > 0 { + state.frontmost_changed = true; + } + state.observe_final_sample(self.baseline, final_sample); + let posture = state.posture(self.baseline, final_sample); + let evidence = posture_evidence( + &state, + self.baseline, + final_sample, + self.target_pid, + containment, + ); + drop(state); + self.finished = true; + (posture, evidence) + } +} + +impl Drop for MacLaunchPostureWitness { + fn drop(&mut self) { + if self.finished { + return; + } + // Deliberately do not close deadline-owned containment. Cancellation + // can drop this witness while LaunchServices still owns the completion + // block; the dispatcher entry must survive until its native deadline. + let _ = WorkspaceEventHub::shared().barrier(NATIVE_BARRIER_TIMEOUT); + let _ = self.ax_stream.close(OBSERVER_CLOSE_TIMEOUT); + let _ = self.hid_stream.close(OBSERVER_CLOSE_TIMEOUT); + } +} + +pub struct MacInteractionPostureWitness { + baseline: PostureSample, + events: tokio::sync::broadcast::Receiver, + state: Arc>, + ax_stream: ObserverControl, + hid_stream: ObserverControl, + finished: Option<(PostureResult, NativeEvidence)>, +} + +impl MacInteractionPostureWitness { + pub fn begin() -> Result { + let workspace = WorkspaceEventHub::shared(); + let mut events = workspace.subscribe(); + let initial_frontmost = apps::frontmost_pid().ok_or_else(|| { + posture_preflight("exact foreground baseline pid is unavailable before interaction") + })?; + let state = Arc::new(Mutex::new(PostureProofState::new())); + let ax_stream = start_ax_focus_stream(initial_frontmost, Arc::clone(&state))?; + let hid_stream = start_hid_cursor_stream(Arc::clone(&state))?; + if !workspace.barrier(NATIVE_BARRIER_TIMEOUT) { + return Err(posture_preflight( + "workspace posture callback queue was unavailable before interaction", + )); + } + let baseline = exact_posture_sample(); + if !baseline.complete() || baseline.frontmost != Some(initial_frontmost) { + return Err(posture_preflight( + "foreground, AX focus/key, or cursor baseline changed during witness acquisition", + )); + } + { + let mut proof = state + .lock() + .expect("interaction posture proof state poisoned"); + proof.observe_workspace(&mut events, initial_frontmost); + if proof.frontmost_changed + || proof.key_window_changed + || proof.physical_cursor_moved + || !proof.complete() + { + return Err(posture_preflight( + "foreground, AX focus/key, or HID cursor changed during witness acquisition", + )); + } + } + Ok(Self { + baseline, + events, + state, + ax_stream, + hid_stream, + finished: None, + }) + } + + pub fn finish(&mut self, deadline: Instant) -> (PostureResult, NativeEvidence) { + if let Some(finished) = &self.finished { + return finished.clone(); + } + let workspace_barrier = + WorkspaceEventHub::shared().barrier(remaining_until(deadline, NATIVE_BARRIER_TIMEOUT)); + let baseline_frontmost = self + .baseline + .frontmost + .expect("interaction posture baseline was validated"); + let mut state = self + .state + .lock() + .expect("interaction posture proof state poisoned"); + state.observe_workspace(&mut self.events, baseline_frontmost); + drop(state); + let ax_barrier = self + .ax_stream + .close(remaining_until(deadline, OBSERVER_CLOSE_TIMEOUT)); + let hid_barrier = self + .hid_stream + .close(remaining_until(deadline, OBSERVER_CLOSE_TIMEOUT)); + let final_sample = exact_posture_sample(); + let mut state = self + .state + .lock() + .expect("interaction posture proof state poisoned"); + state.workspace_stream_complete &= workspace_barrier; + state.ax_stream_complete &= ax_barrier; + state.hid_stream_complete &= hid_barrier; + state.observe_final_sample(self.baseline, final_sample); + let posture = state.posture(self.baseline, final_sample); + let evidence = posture_evidence( + &state, + self.baseline, + final_sample, + None, + SuppressionCloseOutcome { + evidence: SuppressionOutcome::default(), + callback_queue_drained: true, + }, + ); + let finished = (posture, evidence); + drop(state); + self.finished = Some(finished.clone()); + finished + } + + pub fn prior_frontmost_pid(&self) -> i32 { + self.baseline + .frontmost + .expect("interaction posture baseline was validated") + } +} + +impl Drop for MacInteractionPostureWitness { + fn drop(&mut self) { + let _ = self.finish(Instant::now() + OBSERVER_CLOSE_TIMEOUT + OBSERVER_CLOSE_TIMEOUT); + } +} + +fn remaining_until(deadline: Instant, maximum: Duration) -> Duration { + deadline + .saturating_duration_since(Instant::now()) + .min(maximum) +} + +fn posture_evidence( + state: &PostureProofState, + baseline: PostureSample, + final_sample: PostureSample, + target_pid: Option, + containment: SuppressionCloseOutcome, +) -> NativeEvidence { + let mut evidence = NativeEvidence::default(); + evidence.fields.insert( + "prior_frontmost_pid".to_owned(), + serde_json::to_value(baseline.frontmost).unwrap_or_default(), + ); + evidence.fields.insert( + "final_frontmost_pid".to_owned(), + serde_json::to_value(final_sample.frontmost).unwrap_or_default(), + ); + evidence.fields.insert( + "prior_key_window_id".to_owned(), + serde_json::to_value(baseline.key_window.ok().flatten()).unwrap_or_default(), + ); + evidence.fields.insert( + "final_key_window_id".to_owned(), + serde_json::to_value(final_sample.key_window.ok().flatten()).unwrap_or_default(), + ); + evidence.fields.insert( + "prior_physical_cursor".to_owned(), + serde_json::to_value(baseline.cursor.ok()).unwrap_or_default(), + ); + evidence.fields.insert( + "final_physical_cursor".to_owned(), + serde_json::to_value(final_sample.cursor.ok()).unwrap_or_default(), + ); + evidence.fields.insert( + "activation_pids".to_owned(), + serde_json::to_value(&state.activation_pids).unwrap_or_default(), + ); + evidence.fields.insert( + "target_pid".to_owned(), + serde_json::to_value(target_pid).unwrap_or_default(), + ); + evidence.fields.insert( + "workspace_event_stream_complete".to_owned(), + state.workspace_stream_complete.into(), + ); + evidence.fields.insert( + "ax_focus_event_stream_complete".to_owned(), + state.ax_stream_complete.into(), + ); + evidence.fields.insert( + "hid_cursor_event_stream_complete".to_owned(), + state.hid_stream_complete.into(), + ); + evidence.fields.insert( + "containment_event_stream_complete".to_owned(), + state.containment_stream_complete.into(), + ); + evidence.fields.insert( + "workspace_event_lagged_count".to_owned(), + state.workspace_lagged_events.into(), + ); + evidence.fields.insert( + "ax_focus_event_count".to_owned(), + state.ax_focus_events.into(), + ); + evidence.fields.insert( + "hid_cursor_event_count".to_owned(), + state.hid_cursor_events.into(), + ); + evidence.fields.insert( + "containment_callback_queue_drained".to_owned(), + containment.callback_queue_drained.into(), + ); + insert_suppression_evidence(&mut evidence, containment.evidence); + evidence +} + +fn insert_suppression_evidence(evidence: &mut NativeEvidence, outcome: SuppressionOutcome) { + evidence.fields.insert( + "containment_activations".to_owned(), + outcome.activations.into(), + ); + evidence.fields.insert( + "containment_restore_attempts".to_owned(), + outcome.restore_attempts.into(), + ); + evidence.fields.insert( + "containment_restore_failures".to_owned(), + outcome.restore_failures.into(), + ); +} + +#[derive(Debug, Clone, Copy)] +struct PostureSample { + frontmost: Option, + key_window: Result, ()>, + cursor: Result<(f64, f64), ()>, +} + +impl PostureSample { + fn complete(self) -> bool { + self.frontmost.is_some() && self.key_window.is_ok() && self.cursor.is_ok() + } +} + +fn exact_posture_sample() -> PostureSample { + let frontmost = apps::frontmost_pid(); + PostureSample { + frontmost, + key_window: frontmost.map(focused_window_for_pid).unwrap_or(Err(())), + cursor: cursor_position().ok_or(()), + } +} + +fn focused_window_for_pid(pid: i32) -> Result, ()> { + unsafe { + let app = AXUIElementCreateApplication(pid); + if app.is_null() { + return Err(()); + } + let attribute = CFString::new("AXFocusedWindow"); + let mut value: CFTypeRef = std::ptr::null(); + let error = AXUIElementCopyAttributeValue(app, attribute.as_concrete_TypeRef(), &mut value); + CFRelease(app as CFTypeRef); + if error == kAXErrorNoValue { + return Ok(None); + } + if error != kAXErrorSuccess || value.is_null() { + return Err(()); + } + if CFGetTypeID(value) != AXUIElementGetTypeID() { + CFRelease(value); + return Err(()); + } + let window_id = bindings::ax_get_window_id(value.cast_mut().cast()); + CFRelease(value); + window_id.map(Some).ok_or(()) + } +} + +fn cursor_position() -> Option<(f64, f64)> { + use core_graphics::{ + event::CGEvent, + event_source::{CGEventSource, CGEventSourceStateID}, + }; + + let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState).ok()?; + let event = CGEvent::new(source).ok()?; + let point = event.location(); + Some((point.x, point.y)) +} + +fn posture_preflight(message: impl Into) -> NativeError { + NativeError::new( + ErrorCode::PostureUnverifiable, + ErrorPhase::Preflight, + true, + message, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn sample() -> PostureSample { + PostureSample { + frontmost: Some(10), + key_window: Ok(Some(100)), + cursor: Ok((5.0, 6.0)), + } + } + + #[test] + fn ordered_event_excursion_remains_a_violation_after_exact_restoration() { + let baseline = sample(); + let mut state = PostureProofState::new(); + state.frontmost_changed = true; + state.activation_pids.extend([20, 10]); + let posture = state.posture(baseline, baseline); + assert!(!posture.held); + assert!(posture.frontmost_changed); + assert!(posture.restored_after_violation); + } + + #[test] + fn each_incomplete_native_stream_refuses_a_clean_posture_claim() { + for stream in ["workspace", "ax", "hid", "containment"] { + let baseline = sample(); + let mut state = PostureProofState::new(); + match stream { + "workspace" => state.workspace_stream_complete = false, + "ax" => state.ax_stream_complete = false, + "hid" => state.hid_stream_complete = false, + "containment" => state.containment_stream_complete = false, + _ => unreachable!(), + } + assert!(!state.posture(baseline, baseline).held, "{stream}"); + } + } + + #[test] + fn ax_and_hid_events_are_primary_transient_proof_not_polling_samples() { + let baseline = sample(); + let mut state = PostureProofState::new(); + state.key_window_changed = true; + state.ax_focus_events = 1; + state.physical_cursor_moved = true; + state.hid_cursor_events = 1; + let posture = state.posture(baseline, baseline); + assert!(!posture.held); + assert!(posture.key_window_changed); + assert!(posture.physical_cursor_moved); + assert!(posture.restored_after_violation); + } + + #[test] + fn unreadable_final_diagnostic_marks_all_corresponding_streams_unverifiable() { + let baseline = sample(); + let mut state = PostureProofState::new(); + state.observe_final_sample( + baseline, + PostureSample { + frontmost: None, + key_window: Err(()), + cursor: Err(()), + }, + ); + assert!(!state.complete()); + assert!(!state.posture(baseline, baseline).held); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/settlement.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/settlement.rs new file mode 100644 index 0000000000..ec29305092 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/settlement.rs @@ -0,0 +1,912 @@ +//! Native target signals and notification-driven post-action settlement. + +use std::{ + collections::{BTreeSet, VecDeque}, + ffi::c_void, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + mpsc, Arc, Mutex, + }, + thread, + time::{Duration, Instant}, +}; + +use core_foundation::{ + base::{CFGetTypeID, CFRelease, CFTypeRef, TCFType}, + runloop::{ + kCFRunLoopCommonModes, kCFRunLoopDefaultMode, CFRunLoop, CFRunLoopAddSource, + CFRunLoopGetCurrent, CFRunLoopRef, CFRunLoopRemoveSource, CFRunLoopWakeUp, + }, + string::{CFString, CFStringRef}, +}; +use cua_driver_core::api::{ + errors::{ErrorCode, ErrorPhase, NativeError}, + settlement::{ + DirtyState, PendingSettlementEvidence, PendingSettlementState, SettledState, + SettlementAttempt, SettlementEvidence, SettlementSignal, + }, +}; +use tokio::sync::Notify; + +use crate::ax::bindings::{ + self, copy_ax_windows, kAXErrorSuccess, AXObserverAddNotification, AXObserverCreate, + AXObserverGetRunLoopSource, AXObserverRef, AXUIElementCopyAttributeValue, + AXUIElementCreateApplication, AXUIElementGetTypeID, AXUIElementRef, +}; + +use super::observation::RetainedAxElement; + +const SIGNAL_JOURNAL_CAPACITY: usize = 512; +const AX_OBSERVER_HANDSHAKE_TIMEOUT: Duration = Duration::from_millis(750); +const AX_OBSERVER_COMMAND_TIMEOUT: Duration = Duration::from_millis(750); +const AX_OBSERVER_POLL_INTERVAL: Duration = Duration::from_millis(20); + +type CGDirectDisplayId = u32; +type CGDisplayChangeSummaryFlags = u32; +type CGError = i32; +type CGDisplayReconfigurationCallback = unsafe extern "C" fn( + display: CGDirectDisplayId, + flags: CGDisplayChangeSummaryFlags, + user_info: *mut c_void, +); + +#[link(name = "CoreGraphics", kind = "framework")] +extern "C" { + fn CGDisplayRegisterReconfigurationCallback( + callback: CGDisplayReconfigurationCallback, + user_info: *mut c_void, + ) -> CGError; + fn CGDisplayRemoveReconfigurationCallback( + callback: CGDisplayReconfigurationCallback, + user_info: *mut c_void, + ) -> CGError; +} + +#[derive(Debug, Clone, Copy)] +struct RecordedSignal { + at: Instant, + signal: SettlementSignal, +} + +#[derive(Debug, Default)] +struct JournalState { + signals: VecDeque, + epoch: u64, +} + +/// Bounded, content-free native event journal for one target controller. +/// +/// The journal intentionally stores only signal kinds and monotonic times. AX +/// values, selected text and screenshots belong to observations and artifacts, +/// never logs or the settlement channel. +#[derive(Debug, Clone)] +pub struct MacSignalJournal { + state: Arc>, + changed: Arc, +} + +impl Default for MacSignalJournal { + fn default() -> Self { + Self { + state: Arc::new(Mutex::new(JournalState::default())), + changed: Arc::new(Notify::new()), + } + } +} + +impl MacSignalJournal { + pub fn record(&self, signal: SettlementSignal) { + let mut state = self.state.lock().expect("macOS signal journal poisoned"); + state.epoch = state.epoch.wrapping_add(1); + if state.signals.len() == SIGNAL_JOURNAL_CAPACITY { + state.signals.pop_front(); + } + state.signals.push_back(RecordedSignal { + at: Instant::now(), + signal, + }); + drop(state); + self.changed.notify_waiters(); + } + + /// Monotonic target-mutation bracket for coherent observations. + pub fn epoch(&self) -> u64 { + self.state + .lock() + .expect("macOS signal journal poisoned") + .epoch + } + + /// Run a synchronous publication while native callbacks are excluded from + /// advancing the epoch. `None` means evidence changed since the caller's + /// A-side sample and no publication occurred. + pub fn commit_if_epoch( + &self, + expected: u64, + commit: impl FnOnce() -> Result, + ) -> Result, NativeError> { + let state = self.state.lock().expect("macOS signal journal poisoned"); + if state.epoch != expected { + return Ok(None); + } + let result = commit()?; + drop(state); + Ok(Some(result)) + } + + fn snapshot_since(&self, since: Instant) -> Vec { + self.state + .lock() + .expect("macOS signal journal poisoned") + .signals + .iter() + .copied() + .filter(|record| record.at >= since) + .collect() + } + + /// Wait until terminal evidence exists and target-relevant native signals + /// have been quiet for the profile window. The timer is not a fixed sleep: + /// every relevant notification interrupts and resets it. + pub async fn settle( + &self, + dirty: &DirtyState, + relevant_signals: &BTreeSet, + deadline: Instant, + ) -> SettlementAttempt { + let quiet_window = Duration::from_millis(dirty.profile.quiet_window_ms); + let mut eligible_since: Option = None; + + loop { + // Register the waiter before reading state so a signal cannot land + // between the snapshot and the await and be lost. + let changed = self.changed.notified(); + let records = self.snapshot_since(dirty.since); + let mut observed = dirty.observed_signals.clone(); + observed.extend(records.iter().map(|record| record.signal)); + + let terminal = observed.contains(&SettlementSignal::DispatchComplete) + && dirty.profile.required_terminal_signals.is_subset(&observed); + let latest_relevant = latest_relevant_signal(&records, relevant_signals); + + if terminal { + let now = Instant::now(); + let quiet_since = match eligible_since { + Some(prior) => latest_relevant.map_or(prior, |latest| prior.max(latest)), + None => latest_relevant.map_or(now, |latest| latest.max(now)), + }; + eligible_since = Some(quiet_since); + if now.saturating_duration_since(quiet_since) >= quiet_window { + return SettlementAttempt::Settled(SettlementEvidence { + state: SettledState::Settled, + trigger_action_id: Some(dirty.action_id.clone()), + profile: dirty.profile.name.clone(), + elapsed_ms: dirty.since.elapsed().as_millis().min(u128::from(u64::MAX)) + as u64, + observed_signals: observed.iter().copied().collect(), + terminal_signal: "target_notifications_quiet".to_owned(), + quiet_window_ms: dirty.profile.quiet_window_ms, + resumed_from_prior_call: dirty.resumed_from_prior_call, + }); + } + } else { + eligible_since = None; + } + + let now = Instant::now(); + if now >= deadline { + return SettlementAttempt::Pending(pending_evidence(dirty, observed)); + } + + let wake_at = eligible_since + .map(|quiet_since| quiet_since + quiet_window) + .unwrap_or(deadline) + .min(deadline); + tokio::select! { + _ = changed => {} + _ = tokio::time::sleep_until(tokio::time::Instant::from_std(wake_at)) => {} + } + } + } +} + +fn latest_relevant_signal( + records: &[RecordedSignal], + relevant_signals: &BTreeSet, +) -> Option { + records + .iter() + .filter(|record| relevant_signals.contains(&record.signal)) + .map(|record| record.at) + .max() +} + +fn pending_evidence( + dirty: &DirtyState, + observed: BTreeSet, +) -> PendingSettlementEvidence { + let mut missing_signals: Vec<_> = dirty + .profile + .required_terminal_signals + .difference(&observed) + .copied() + .collect(); + if !observed.contains(&SettlementSignal::DispatchComplete) { + missing_signals.push(SettlementSignal::DispatchComplete); + } + PendingSettlementEvidence { + state: PendingSettlementState::Pending, + trigger_action_id: dirty.action_id.clone(), + profile: dirty.profile.name.clone(), + elapsed_ms: dirty.since.elapsed().as_millis().min(u128::from(u64::MAX)) as u64, + observed_signals: observed.iter().copied().collect(), + missing_signals, + } +} + +struct ObserverContext { + target_window_id: u32, + focused_target: AtomicBool, + journal: MacSignalJournal, + on_event: Arc, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum MacAxEvent { + ContentChanged, + FocusChanged, + TreeChanged, + WindowGeometryChanged, + WindowDestroyed, + MenuOpened, + MenuDismissed, + ScrollChanged, +} + +impl MacAxEvent { + fn settlement_signal(self) -> SettlementSignal { + match self { + Self::ContentChanged => SettlementSignal::AxValueChanged, + Self::FocusChanged => SettlementSignal::FocusChanged, + Self::TreeChanged => SettlementSignal::AxAction, + Self::WindowGeometryChanged => SettlementSignal::WindowGeometryChanged, + Self::WindowDestroyed => SettlementSignal::WindowListChanged, + Self::MenuOpened => SettlementSignal::MenuOpened, + Self::MenuDismissed => SettlementSignal::MenuDismissed, + Self::ScrollChanged => SettlementSignal::ScrollChanged, + } + } +} + +unsafe extern "C" fn observer_callback( + _observer: AXObserverRef, + element: AXUIElementRef, + notification: CFStringRef, + refcon: *mut c_void, +) { + if refcon.is_null() || element.is_null() || notification.is_null() { + return; + } + let notification = CFString::wrap_under_get_rule(notification).to_string(); + let context = &*(refcon as *const ObserverContext); + if matches!( + notification.as_str(), + "AXFocusedUIElementChanged" | "AXFocusedWindowChanged" + ) { + // Application-level focus callbacks carry the application object. + // Accept only a transition entering, staying within, or leaving this + // exact target window. + let focused_target = focused_window_id(element) == Some(context.target_window_id); + let was_focused = context + .focused_target + .swap(focused_target, Ordering::AcqRel); + if !was_focused && !focused_target { + return; + } + } + // Other callbacks can only originate from elements explicitly registered + // as this target or a retained descendant. Never query the callback + // element after AXUIElementDestroyed. + if let Some(event) = event_for_notification(¬ification) { + context.journal.record(event.settlement_signal()); + (context.on_event)(event); + } +} + +unsafe fn focused_window_id(application: AXUIElementRef) -> Option { + let attribute = CFString::new("AXFocusedWindow"); + let mut value: CFTypeRef = std::ptr::null(); + if AXUIElementCopyAttributeValue(application, attribute.as_concrete_TypeRef(), &mut value) + != kAXErrorSuccess + || value.is_null() + { + return None; + } + if CFGetTypeID(value) != AXUIElementGetTypeID() { + CFRelease(value); + return None; + } + let window_id = bindings::ax_get_window_id(value.cast_mut().cast()); + CFRelease(value); + window_id +} + +fn event_for_notification(notification: &str) -> Option { + match notification { + "AXFocusedUIElementChanged" | "AXFocusedWindowChanged" => Some(MacAxEvent::FocusChanged), + "AXValueChanged" | "AXSelectedTextChanged" | "AXSelectedChildrenChanged" => { + Some(MacAxEvent::ContentChanged) + } + "AXUIElementDestroyed" => Some(MacAxEvent::WindowDestroyed), + "AXMoved" | "AXResized" => Some(MacAxEvent::WindowGeometryChanged), + "AXMenuOpened" => Some(MacAxEvent::MenuOpened), + "AXMenuClosed" | "AXMenuItemSelected" => Some(MacAxEvent::MenuDismissed), + "AXSelectedRowsChanged" | "AXSelectedColumnsChanged" => Some(MacAxEvent::ScrollChanged), + "AXLayoutChanged" => Some(MacAxEvent::TreeChanged), + "AXTitleChanged" => Some(MacAxEvent::ContentChanged), + _ => None, + } +} + +struct DisplayObserverContext { + journal: MacSignalJournal, + on_change: Arc, +} + +unsafe extern "C" fn display_reconfiguration_callback( + _display: CGDirectDisplayId, + _flags: CGDisplayChangeSummaryFlags, + user_info: *mut c_void, +) { + if user_info.is_null() { + return; + } + let context = &*(user_info as *const DisplayObserverContext); + context + .journal + .record(SettlementSignal::WindowGeometryChanged); + (context.on_change)(); +} + +/// Exact-target invalidation bridge for global display topology/scale changes. +pub struct MacDisplayObserverRegistration { + context: *mut DisplayObserverContext, +} + +unsafe impl Send for MacDisplayObserverRegistration {} +unsafe impl Sync for MacDisplayObserverRegistration {} + +impl MacDisplayObserverRegistration { + pub fn start( + journal: MacSignalJournal, + on_change: Arc, + ) -> Result { + let context = Box::into_raw(Box::new(DisplayObserverContext { journal, on_change })); + let error = unsafe { + CGDisplayRegisterReconfigurationCallback( + display_reconfiguration_callback, + context.cast(), + ) + }; + if error != 0 { + unsafe { drop(Box::from_raw(context)) }; + return Err(observer_error(format!( + "CGDisplayRegisterReconfigurationCallback failed with {error}" + ))); + } + Ok(Self { context }) + } +} + +impl Drop for MacDisplayObserverRegistration { + fn drop(&mut self) { + unsafe { + let _ = CGDisplayRemoveReconfigurationCallback( + display_reconfiguration_callback, + self.context.cast(), + ); + drop(Box::from_raw(self.context)); + } + } +} + +/// Dedicated native AX observer bound to one exact process generation/window +/// identity. The caller must recreate it after any generation change. +pub struct MacAxObserverRegistration { + stopping: Arc, + run_loop: Arc, + commands: mpsc::Sender, + finished: mpsc::Receiver<()>, + thread: Option>, +} + +enum ObserverCommand { + ReplaceElements { + elements: Vec, + reply: mpsc::SyncSender>, + }, +} + +#[derive(Clone, Copy)] +struct AxObserverTarget { + pid: i32, + window_id: u32, +} + +impl MacAxObserverRegistration { + pub fn start( + pid: i32, + target_window_id: u32, + journal: MacSignalJournal, + on_event: Arc, + ) -> Result { + let stopping = Arc::new(AtomicBool::new(false)); + let run_loop = Arc::new(AtomicUsize::new(0)); + let thread_stopping = stopping.clone(); + let thread_run_loop = run_loop.clone(); + let (started_tx, started_rx) = mpsc::sync_channel(1); + let (finished_tx, finished_rx) = mpsc::sync_channel(1); + let (command_tx, command_rx) = mpsc::channel(); + + let thread = thread::Builder::new() + .name(format!("cua-ax-observer-{pid}-{target_window_id}")) + .spawn(move || { + let result = unsafe { + run_ax_observer( + AxObserverTarget { + pid, + window_id: target_window_id, + }, + journal, + on_event, + &thread_stopping, + &thread_run_loop, + &started_tx, + &command_rx, + ) + }; + if let Err(error) = result { + let _ = started_tx.send(Err(error)); + } + let _ = finished_tx.send(()); + }) + .map_err(|error| observer_error(format!("failed to spawn AX observer: {error}")))?; + + match started_rx.recv_timeout(AX_OBSERVER_HANDSHAKE_TIMEOUT) { + Ok(Ok(())) => Ok(Self { + stopping, + run_loop, + commands: command_tx, + finished: finished_rx, + thread: Some(thread), + }), + Ok(Err(error)) => { + let _ = finished_rx.recv_timeout(AX_OBSERVER_HANDSHAKE_TIMEOUT); + // Never join a native AX run-loop thread. The bounded finish + // acknowledgement is the teardown proof; dropping the handle + // detaches a wedged thread whose callback context it still owns. + drop(thread); + Err(error) + } + Err(mpsc::RecvTimeoutError::Disconnected) => { + drop(thread); + Err(observer_error("AX observer exited before registration")) + } + Err(mpsc::RecvTimeoutError::Timeout) => { + stopping.store(true, Ordering::Release); + let active_run_loop = run_loop.load(Ordering::Acquire) as CFRunLoopRef; + if !active_run_loop.is_null() { + unsafe { CFRunLoopWakeUp(active_run_loop) }; + } + Err(observer_error( + "AX observer registration exceeded its bounded deadline", + )) + } + } + } + + pub fn replace_elements(&self, elements: Vec) -> Result<(), NativeError> { + let (reply_tx, reply_rx) = mpsc::sync_channel(1); + self.commands + .send(ObserverCommand::ReplaceElements { + elements, + reply: reply_tx, + }) + .map_err(|_| observer_error("AX observer stopped before descendant registration"))?; + let run_loop = self.run_loop.load(Ordering::Acquire) as CFRunLoopRef; + if !run_loop.is_null() { + unsafe { CFRunLoopWakeUp(run_loop) }; + } + reply_rx + .recv_timeout(AX_OBSERVER_COMMAND_TIMEOUT) + .map_err(|_| { + observer_error("AX descendant registration exceeded its bounded deadline") + })? + } +} + +impl Drop for MacAxObserverRegistration { + fn drop(&mut self) { + self.stopping.store(true, Ordering::Release); + let run_loop = self.run_loop.load(Ordering::Acquire) as CFRunLoopRef; + if !run_loop.is_null() { + unsafe { CFRunLoopWakeUp(run_loop) }; + } + if let Some(thread) = self.thread.take() { + let _ = self.finished.recv_timeout(AX_OBSERVER_HANDSHAKE_TIMEOUT); + // The observer thread owns every retained AX object and callback + // context. Detach after the bounded acknowledgement instead of + // risking an unbounded join if CoreFoundation stalls on teardown. + drop(thread); + } + } +} + +unsafe fn run_ax_observer( + target_info: AxObserverTarget, + journal: MacSignalJournal, + on_event: Arc, + stopping: &AtomicBool, + run_loop_slot: &AtomicUsize, + started: &mpsc::SyncSender>, + commands: &mpsc::Receiver, +) -> Result<(), NativeError> { + let pid = target_info.pid; + let target_window_id = target_info.window_id; + let application = AXUIElementCreateApplication(pid); + if application.is_null() { + return Err(observer_error("AX application element is unavailable")); + } + let windows = copy_ax_windows(application); + let target = windows + .iter() + .copied() + .find(|element| bindings::ax_get_window_id(*element) == Some(target_window_id)); + for window in windows + .iter() + .copied() + .filter(|window| Some(*window) != target) + { + CFRelease(window as CFTypeRef); + } + let Some(target) = target else { + CFRelease(application as CFTypeRef); + return Err(observer_error("exact target AX window is unavailable")); + }; + + let context = Box::new(ObserverContext { + target_window_id, + focused_target: AtomicBool::new(focused_window_id(application) == Some(target_window_id)), + journal, + on_event, + }); + let context_ptr = Box::into_raw(context); + let mut observer = std::ptr::null_mut(); + let create_error = AXObserverCreate(pid, observer_callback, &mut observer); + if create_error != kAXErrorSuccess || observer.is_null() { + drop(Box::from_raw(context_ptr)); + CFRelease(target as CFTypeRef); + CFRelease(application as CFTypeRef); + return Err(observer_error(format!( + "AXObserverCreate failed with {create_error}" + ))); + } + + let target_notifications = ["AXUIElementDestroyed", "AXMoved", "AXResized"]; + let application_notifications = ["AXFocusedUIElementChanged", "AXFocusedWindowChanged"]; + let descendant_notifications = [ + "AXValueChanged", + "AXSelectedTextChanged", + "AXSelectedChildrenChanged", + "AXMenuOpened", + "AXMenuClosed", + "AXMenuItemSelected", + "AXSelectedRowsChanged", + "AXSelectedColumnsChanged", + "AXLayoutChanged", + "AXTitleChanged", + ]; + let mut target_registrations = 0usize; + for notification in target_notifications { + let notification = CFString::new(notification); + if AXObserverAddNotification( + observer, + target, + notification.as_concrete_TypeRef(), + context_ptr.cast(), + ) == kAXErrorSuccess + { + target_registrations += 1; + } + } + let mut focus_registrations = 0usize; + for notification in application_notifications { + let notification = CFString::new(notification); + if AXObserverAddNotification( + observer, + application, + notification.as_concrete_TypeRef(), + context_ptr.cast(), + ) == kAXErrorSuccess + { + focus_registrations += 1; + } + } + if target_registrations == 0 || focus_registrations == 0 { + CFRelease(observer as CFTypeRef); + drop(Box::from_raw(context_ptr)); + CFRelease(target as CFTypeRef); + CFRelease(application as CFTypeRef); + return Err(observer_error( + "target AX window lacks required exact window or application-focus notifications", + )); + } + + let source = AXObserverGetRunLoopSource(observer); + if source.is_null() { + CFRelease(observer as CFTypeRef); + drop(Box::from_raw(context_ptr)); + CFRelease(target as CFTypeRef); + CFRelease(application as CFTypeRef); + return Err(observer_error("AX observer has no run-loop source")); + } + let run_loop = CFRunLoopGetCurrent(); + run_loop_slot.store(run_loop as usize, Ordering::Release); + CFRunLoopAddSource(run_loop, source, kCFRunLoopCommonModes); + if started.send(Ok(())).is_err() { + remove_static_notifications( + observer, + target, + application, + &target_notifications, + &application_notifications, + ); + CFRunLoopRemoveSource(run_loop, source, kCFRunLoopCommonModes); + run_loop_slot.store(0, Ordering::Release); + CFRelease(observer as CFTypeRef); + drop(Box::from_raw(context_ptr)); + CFRelease(target as CFTypeRef); + CFRelease(application as CFTypeRef); + return Err(observer_error( + "AX observer owner disappeared during registration", + )); + } + + let mut observed_elements = Vec::new(); + while !stopping.load(Ordering::Acquire) { + while let Ok(command) = commands.try_recv() { + match command { + ObserverCommand::ReplaceElements { elements, reply } => { + remove_descendant_notifications( + observer, + &observed_elements, + &descendant_notifications, + ); + observed_elements = elements; + let registered = add_descendant_notifications( + observer, + context_ptr.cast(), + &observed_elements, + &descendant_notifications, + ); + let result = if observed_elements.is_empty() || registered > 0 { + Ok(()) + } else { + Err(observer_error( + "target descendants support no AX mutation notifications", + )) + }; + let _ = reply.send(result); + } + } + } + CFRunLoop::run_in_mode(kCFRunLoopDefaultMode, AX_OBSERVER_POLL_INTERVAL, true); + } + remove_descendant_notifications(observer, &observed_elements, &descendant_notifications); + remove_static_notifications( + observer, + target, + application, + &target_notifications, + &application_notifications, + ); + CFRunLoopRemoveSource(run_loop, source, kCFRunLoopCommonModes); + run_loop_slot.store(0, Ordering::Release); + CFRelease(observer as CFTypeRef); + drop(Box::from_raw(context_ptr)); + CFRelease(target as CFTypeRef); + CFRelease(application as CFTypeRef); + Ok(()) +} + +unsafe fn remove_static_notifications( + observer: AXObserverRef, + target: AXUIElementRef, + application: AXUIElementRef, + target_notifications: &[&str], + application_notifications: &[&str], +) { + for notification in target_notifications { + let notification = CFString::new(notification); + let _ = bindings::AXObserverRemoveNotification( + observer, + target, + notification.as_concrete_TypeRef(), + ); + } + for notification in application_notifications { + let notification = CFString::new(notification); + let _ = bindings::AXObserverRemoveNotification( + observer, + application, + notification.as_concrete_TypeRef(), + ); + } +} + +unsafe fn add_descendant_notifications( + observer: AXObserverRef, + context: *mut c_void, + elements: &[RetainedAxElement], + notifications: &[&str], +) -> usize { + let mut registrations = 0; + for element in elements { + for notification in notifications { + let notification = CFString::new(notification); + if AXObserverAddNotification( + observer, + element.as_ptr(), + notification.as_concrete_TypeRef(), + context, + ) == kAXErrorSuccess + { + registrations += 1; + } + } + } + registrations +} + +unsafe fn remove_descendant_notifications( + observer: AXObserverRef, + elements: &[RetainedAxElement], + notifications: &[&str], +) { + for element in elements { + for notification in notifications { + let notification = CFString::new(notification); + let _ = bindings::AXObserverRemoveNotification( + observer, + element.as_ptr(), + notification.as_concrete_TypeRef(), + ); + } + } +} + +fn observer_error(message: impl Into) -> NativeError { + NativeError::new(ErrorCode::Internal, ErrorPhase::Preflight, true, message) +} + +#[cfg(test)] +mod tests { + use super::*; + use cua_driver_core::api::{contracts::ActionId, settlement::SettlementProfile}; + + fn dirty(profile: SettlementProfile) -> DirtyState { + let mut observed_signals = BTreeSet::new(); + observed_signals.insert(SettlementSignal::DispatchStarted); + observed_signals.insert(SettlementSignal::DispatchComplete); + DirtyState { + action_id: ActionId::new(), + profile, + since: Instant::now(), + observed_signals, + resumed_from_prior_call: false, + } + } + + #[test] + fn notification_mapping_contains_no_ax_content() { + assert_eq!( + event_for_notification("AXSelectedTextChanged"), + Some(MacAxEvent::ContentChanged) + ); + assert_eq!(event_for_notification("secret document text"), None); + } + + #[test] + fn epoch_gate_refuses_publication_after_a_native_signal_race() { + let journal = MacSignalJournal::default(); + let epoch = journal.epoch(); + journal.record(SettlementSignal::WindowGeometryChanged); + let published = AtomicBool::new(false); + + let result = journal + .commit_if_epoch(epoch, || { + published.store(true, Ordering::Release); + Ok(()) + }) + .unwrap(); + + assert!(result.is_none()); + assert!(!published.load(Ordering::Acquire)); + } + + #[tokio::test] + async fn relevant_signal_resets_quiet_window() { + let journal = MacSignalJournal::default(); + let mut profile = SettlementProfile::dispatch_only("click"); + profile.quiet_window_ms = 25; + let dirty = dirty(profile); + let relevant = BTreeSet::from([SettlementSignal::AxValueChanged]); + let started = Instant::now(); + let settling = { + let journal = journal.clone(); + tokio::spawn(async move { + journal + .settle(&dirty, &relevant, started + Duration::from_millis(200)) + .await + }) + }; + tokio::task::yield_now().await; + journal.record(SettlementSignal::FreshFrame); + tokio::time::sleep(Duration::from_millis(10)).await; + journal.record(SettlementSignal::AxValueChanged); + let result = settling.await.unwrap(); + let SettlementAttempt::Settled(evidence) = result else { + panic!("expected settled evidence"); + }; + assert!(evidence + .observed_signals + .contains(&SettlementSignal::AxValueChanged)); + assert!(started.elapsed() >= Duration::from_millis(35)); + } + + #[test] + fn unrelated_signals_are_excluded_from_the_quiet_window_clock() { + let started = Instant::now(); + let records = [ + RecordedSignal { + at: started + Duration::from_millis(10), + signal: SettlementSignal::AxValueChanged, + }, + RecordedSignal { + at: started + Duration::from_millis(20), + signal: SettlementSignal::FreshFrame, + }, + ]; + assert_eq!( + latest_relevant_signal( + &records, + &BTreeSet::from([SettlementSignal::AxValueChanged]) + ), + Some(started + Duration::from_millis(10)) + ); + } + + #[tokio::test] + async fn timeout_returns_accumulated_progress() { + let journal = MacSignalJournal::default(); + let profile = SettlementProfile::requiring("menu", [SettlementSignal::MenuDismissed]); + let dirty = dirty(profile); + journal.record(SettlementSignal::AxAction); + let result = journal + .settle( + &dirty, + &BTreeSet::from([SettlementSignal::MenuDismissed]), + Instant::now() + Duration::from_millis(5), + ) + .await; + let SettlementAttempt::Pending(pending) = result else { + panic!("expected pending settlement"); + }; + assert!(pending + .observed_signals + .contains(&SettlementSignal::AxAction)); + assert!(pending + .missing_signals + .contains(&SettlementSignal::MenuDismissed)); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/target.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/target.rs new file mode 100644 index 0000000000..0ac0516a3a --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/target.rs @@ -0,0 +1,290 @@ +use std::sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, +}; + +use async_trait::async_trait; +use cua_driver_core::api::{ + errors::NativeError, + observation::{InvalidationReason, NativeProcessHandle, ResolvedWindowStamp}, + platform::{InvalidationSubscription, TargetFocusCoordinator, TargetInvalidation}, +}; + +use crate::apps::nsworkspace::{WorkspaceEventHub, WorkspaceEventKind}; + +use super::{ + observation::{MacElementRegistry, MacFrameHistory, RetainedAxElement}, + settlement::{ + MacAxEvent, MacAxObserverRegistration, MacDisplayObserverRegistration, MacSignalJournal, + }, +}; + +#[derive(Clone)] +pub struct MacInvalidationHub { + sender: tokio::sync::broadcast::Sender, +} + +impl Default for MacInvalidationHub { + fn default() -> Self { + let (sender, _) = tokio::sync::broadcast::channel(256); + Self { sender } + } +} + +impl MacInvalidationHub { + pub fn publish(&self, invalidation: TargetInvalidation) { + let _ = self.sender.send(invalidation); + } + + pub fn subscribe(&self) -> MacInvalidationSubscription { + MacInvalidationSubscription { + receiver: self.sender.subscribe(), + workspace: WorkspaceEventHub::shared().subscribe(), + registry_closed: false, + workspace_closed: false, + } + } +} + +pub struct MacInvalidationSubscription { + receiver: tokio::sync::broadcast::Receiver, + workspace: tokio::sync::broadcast::Receiver, + registry_closed: bool, + workspace_closed: bool, +} + +#[async_trait] +impl InvalidationSubscription for MacInvalidationSubscription { + async fn next(&mut self) -> Option { + loop { + if self.registry_closed && self.workspace_closed { + return None; + } + tokio::select! { + result = self.receiver.recv(), if !self.registry_closed => match result { + Ok(invalidation) => return Some(invalidation), + Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { + tracing::warn!(skipped, "macOS window invalidation subscriber lagged"); + return Some(TargetInvalidation::NativeStateResyncRequired); + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + self.registry_closed = true; + } + }, + result = self.workspace.recv(), if !self.workspace_closed => match result { + Ok(event) if event.kind == WorkspaceEventKind::Terminated => { + let Some(generation) = event.process_generation else { + tracing::warn!( + pid = event.pid, + "terminated app had no launch generation; refusing pid-only invalidation" + ); + continue; + }; + let process = NativeProcessHandle::new(format!( + "macos:{}:{generation:016x}", + event.pid + )) + .expect("constructed macOS process handle is nonempty"); + return Some(TargetInvalidation::ProcessExited { process }); + } + Ok(_) => {} + Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { + tracing::warn!(skipped, "macOS workspace invalidation subscriber lagged"); + return Some(TargetInvalidation::NativeStateResyncRequired); + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + self.workspace_closed = true; + } + } + } + } + } +} + +/// Native resources associated with one core target controller. Later plans +/// add AX, menu, window and frame observer registrations here. The invalidated +/// flag lets native callbacks revoke their own resources immediately even +/// before the core registry finishes removing the controller. +pub struct MacTargetState { + pub window: ResolvedWindowStamp, + pub signals: MacSignalJournal, + pub elements: MacElementRegistry, + pub frames: MacFrameHistory, + observer: Option, + display_observer: Option, + invalidated: Arc, +} + +impl MacTargetState { + pub fn new( + window: ResolvedWindowStamp, + pid: i32, + cg_window_id: u32, + invalidations: MacInvalidationHub, + ) -> Result { + let signals = MacSignalJournal::default(); + let invalidation_stamp = window.clone(); + let ax_invalidations = invalidations.clone(); + let on_event = Arc::new(move |event| { + let reason = match event { + MacAxEvent::ContentChanged + | MacAxEvent::FocusChanged + | MacAxEvent::MenuOpened + | MacAxEvent::ScrollChanged => InvalidationReason::ContentChanged, + MacAxEvent::TreeChanged | MacAxEvent::WindowDestroyed => { + InvalidationReason::AccessibilityInvalidated + } + MacAxEvent::WindowGeometryChanged => InvalidationReason::WindowChanged, + MacAxEvent::MenuDismissed => InvalidationReason::TransientDismissed, + }; + ax_invalidations.publish(TargetInvalidation::ObservationChanged { + app_id: invalidation_stamp.app_id.clone(), + window_id: invalidation_stamp.window_id.clone(), + generation: invalidation_stamp.generation, + reason, + }); + }); + let observer = + MacAxObserverRegistration::start(pid, cg_window_id, signals.clone(), on_event)?; + let display_stamp = window.clone(); + let on_display_change = Arc::new(move || { + invalidations.publish(TargetInvalidation::ObservationChanged { + app_id: display_stamp.app_id.clone(), + window_id: display_stamp.window_id.clone(), + generation: display_stamp.generation, + reason: InvalidationReason::DisplayChanged, + }); + }); + let display_observer = + MacDisplayObserverRegistration::start(signals.clone(), on_display_change)?; + Ok(Self { + window, + signals, + elements: MacElementRegistry::default(), + frames: MacFrameHistory::default(), + observer: Some(observer), + display_observer: Some(display_observer), + invalidated: Arc::new(AtomicBool::new(false)), + }) + } + + pub fn invalidated(&self) -> bool { + self.invalidated.load(Ordering::Acquire) + } + + pub(crate) fn replace_observed_elements( + &self, + elements: Vec, + ) -> Result<(), NativeError> { + self.observer + .as_ref() + .ok_or_else(|| { + NativeError::stale( + cua_driver_core::api::errors::ErrorCode::WindowIdentityChanged, + "macOS AX observer was invalidated before descendant registration", + ) + })? + .replace_elements(elements) + } + + pub fn invalidate(&mut self) { + self.invalidated.store(true, Ordering::Release); + self.observer.take(); + self.display_observer.take(); + } + + pub(crate) fn poison_handle(&self) -> Arc { + Arc::clone(&self.invalidated) + } +} + +impl Drop for MacTargetState { + fn drop(&mut self) { + self.invalidate(); + } +} + +#[derive(Debug, Default)] +pub(crate) struct MacActiveBelief { + pub action_id: cua_driver_core::api::contracts::ActionId, + pub pid: i32, + pub cg_window_id: u32, +} + +#[derive(Debug, Default)] +pub(crate) struct MacFocusState { + pub shutdown: bool, + pub active_belief: Option, +} + +#[derive(Default)] +pub struct MacTargetFocusCoordinator { + state: Arc>, +} + +impl MacTargetFocusCoordinator { + pub(crate) fn state_handle(&self) -> Arc> { + Arc::clone(&self.state) + } + + pub(crate) fn is_shutdown(&self) -> bool { + self.state + .lock() + .expect("macOS focus coordinator poisoned") + .shutdown + } +} + +#[async_trait] +impl TargetFocusCoordinator for MacTargetFocusCoordinator { + async fn shutdown(&mut self) -> Result<(), NativeError> { + let mut state = self.state.lock().expect("macOS focus coordinator poisoned"); + state.shutdown = true; + if let Some(active) = &state.active_belief { + crate::input::slps_make_key::post_target_only( + active.pid, + active.cg_window_id, + &[crate::input::slps_make_key::SlpsMakeKeyState::RemoveKey], + ) + .map_err(|error| { + NativeError::new( + cua_driver_core::api::errors::ErrorCode::Internal, + cua_driver_core::api::errors::ErrorPhase::Verify, + false, + format!("SLPS remove-key retry failed during target teardown: {error}"), + ) + .with_detail("action_id", active.action_id.to_string()) + .with_detail("pid", active.pid) + .with_detail("cg_window_id", active.cg_window_id) + })?; + state.active_belief = None; + } + Ok(()) + } +} + +impl Drop for MacTargetFocusCoordinator { + fn drop(&mut self) { + let Ok(mut state) = self.state.lock() else { + tracing::error!("macOS focus coordinator lock was poisoned during final drop"); + return; + }; + let Some(active) = &state.active_belief else { + return; + }; + match crate::input::slps_make_key::post_target_only( + active.pid, + active.cg_window_id, + &[crate::input::slps_make_key::SlpsMakeKeyState::RemoveKey], + ) { + Ok(()) => state.active_belief = None, + Err(error) => tracing::error!( + error = %error, + action_id = %active.action_id, + pid = active.pid, + cg_window_id = active.cg_window_id, + "final target-only SLPS remove-key record failed during coordinator drop" + ), + } + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/windows.rs new file mode 100644 index 0000000000..ba3f13378f --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/windows.rs @@ -0,0 +1,1389 @@ +use std::{ + collections::{HashMap, HashSet, VecDeque}, + sync::{Arc, Mutex}, +}; + +use async_trait::async_trait; +use core_foundation::base::{CFEqual, CFRelease, CFRetain, CFTypeRef}; +use cua_driver_core::api::{ + capabilities::{Framework, WindowStateKind}, + contracts::{AppId, AppRef, GeometryRevision, Rect, WindowGeneration, WindowId, WindowRef}, + errors::{ErrorCode, ErrorPhase, NativeError}, + observation::{ + NativeProcessHandle, NativeWindowHandle, ResolvedWindow, ResolvedWindowStamp, + WindowGeometry, + }, + platform::{TargetInvalidation, WindowProvider}, +}; + +use crate::{ + apps::nsworkspace::{self, RunningApplicationInfo}, + ax::bindings::{self, copy_ax_windows, AXUIElementCreateApplication, AXUIElementRef}, + permissions, windows, +}; + +use super::target::MacInvalidationHub; + +const MAX_WINDOW_TOMBSTONES: usize = 512; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct NativeWindowKey { + pid: i32, + process_generation: u64, + cg_window_id: u32, +} + +#[derive(Debug, Clone)] +struct ProcessSnapshot { + pid: i32, + generation: u64, + name: Option, + bundle_id: Option, + hidden: bool, +} + +#[derive(Debug)] +struct RetainedAxWindow(usize); + +impl RetainedAxWindow { + unsafe fn from_owned(element: AXUIElementRef) -> Self { + Self(element as usize) + } + + unsafe fn from_borrowed(element: AXUIElementRef) -> Self { + CFRetain(element as CFTypeRef); + Self(element as usize) + } + + fn same_identity(&self, other: &Self) -> bool { + unsafe { CFEqual(self.0 as CFTypeRef, other.0 as CFTypeRef) != 0 } + } + + fn as_ptr(&self) -> AXUIElementRef { + self.0 as AXUIElementRef + } +} + +impl Clone for RetainedAxWindow { + fn clone(&self) -> Self { + unsafe { CFRetain(self.0 as CFTypeRef) }; + Self(self.0) + } +} + +unsafe impl Send for RetainedAxWindow {} +unsafe impl Sync for RetainedAxWindow {} + +type AxWindowMatches = HashMap)>>; + +impl Drop for RetainedAxWindow { + fn drop(&mut self) { + unsafe { CFRelease(self.0 as CFTypeRef) }; + } +} + +#[derive(Debug, Clone)] +struct NativeWindowSnapshot { + key: NativeWindowKey, + process: ProcessSnapshot, + owner_name: String, + title: Option, + bounds: Rect, + layer: i32, + z_index: usize, + is_on_screen: bool, + on_current_space: Option, + space_ids: Option>, + minimized: Option, + scale_factor: Option, + ax_identity: RetainedAxWindow, +} + +/// Exact native facts for route preflight and observation setup. Callers get +/// these only through `facts_for_stamp`, which first revalidates every stable +/// identity component and the geometry revision against the live registry. +#[derive(Debug, Clone, PartialEq)] +pub struct MacWindowFacts { + pub stamp: ResolvedWindowStamp, + pub pid: i32, + pub process_generation: u64, + pub cg_window_id: u32, + pub owner_name: String, + pub layer: i32, + pub bounds: Rect, + pub scale_factor: Option, + pub state: WindowStateKind, + pub is_on_screen: bool, + pub on_current_space: Option, + pub space_ids: Option>, + pub minimized: Option, +} + +/// Exact registered identity for an AX-owned popover or sheet related to a +/// public target window. These entries never appear in public window listing; +/// they exist only so captured related pixels can be revalidated before use. +#[derive(Debug, Clone, PartialEq)] +pub struct MacRelatedWindowFacts { + pub public: WindowRef, + pub stamp: ResolvedWindowStamp, + pub parent: ResolvedWindowStamp, + pub pid: i32, + pub process_generation: u64, + pub cg_window_id: u32, + pub bounds: Rect, + pub scale_factor: f64, + pub layer: i32, + pub is_on_screen: bool, +} + +trait WindowSnapshotSource: Send + Sync { + fn snapshot(&self) -> Result, NativeError>; +} + +#[derive(Default)] +struct SystemWindowSnapshotSource; + +impl WindowSnapshotSource for SystemWindowSnapshotSource { + fn snapshot(&self) -> Result, NativeError> { + if !permissions::status::accessibility_granted() { + return Err(NativeError::new( + ErrorCode::PermissionDenied, + ErrorPhase::Preflight, + true, + "Accessibility permission is required to establish exact macOS window identity", + )); + } + + match self.snapshot_once() { + Ok(snapshots) => Ok(snapshots), + Err(_) => self.snapshot_once().map_err(|racing_pids| { + NativeError::new( + ErrorCode::WindowIdentityChanged, + ErrorPhase::Preflight, + true, + "process identity changed while joining WindowServer and Accessibility windows", + ) + .with_detail( + "pids", + serde_json::to_value(racing_pids).unwrap_or_default(), + ) + }), + } + } +} + +impl SystemWindowSnapshotSource { + fn snapshot_once(&self) -> Result, Vec> { + let applications_before: HashMap = + nsworkspace::running_applications() + .into_iter() + .map(|application| (application.pid, application)) + .collect(); + let server_windows = windows::all_windows(); + let mut ax_by_pid: HashMap = HashMap::new(); + let mut snapshots = Vec::new(); + + for window in &server_windows { + let Some(application) = applications_before.get(&window.pid) else { + continue; + }; + let Some(process_generation) = application.process_generation else { + continue; + }; + let matches = ax_by_pid + .entry(window.pid) + .or_insert_with(|| ax_windows_for_process(window.pid)); + let Some(candidates) = matches.get(&window.window_id) else { + continue; + }; + if candidates.len() != 1 { + tracing::warn!( + pid = window.pid, + window_id = window.window_id, + candidate_count = candidates.len(), + "ambiguous AX-to-WindowServer identity; omitting window" + ); + continue; + } + let (ax_identity, minimized) = &candidates[0]; + let space = windows::space_facts(window.window_id); + let scale_factor = windows::display_scale_for_bounds(&window.bounds); + snapshots.push(NativeWindowSnapshot { + key: NativeWindowKey { + pid: window.pid, + process_generation, + cg_window_id: window.window_id, + }, + process: ProcessSnapshot { + pid: window.pid, + generation: process_generation, + name: application.name.clone(), + bundle_id: application.bundle_id.clone(), + hidden: application.hidden, + }, + owner_name: window.app_name.clone(), + title: (!window.title.is_empty()).then_some(window.title.clone()), + bounds: Rect { + x: window.bounds.x, + y: window.bounds.y, + width: window.bounds.width, + height: window.bounds.height, + }, + layer: window.layer, + z_index: window.z_index, + is_on_screen: window.is_on_screen, + on_current_space: space.as_ref().map(|facts| facts.on_current_space), + space_ids: space.map(|facts| facts.space_ids), + minimized: *minimized, + scale_factor, + ax_identity: ax_identity.clone(), + }); + } + let applications_after: HashMap = + nsworkspace::running_applications() + .into_iter() + .map(|application| (application.pid, application)) + .collect(); + let racing_pids = racing_process_pids( + server_windows.iter().map(|window| window.pid), + &applications_before, + &applications_after, + ); + if racing_pids.is_empty() { + Ok(snapshots) + } else { + Err(racing_pids) + } + } +} + +fn racing_process_pids( + window_pids: impl IntoIterator, + before: &HashMap, + after: &HashMap, +) -> Vec { + let mut racing_pids = HashSet::new(); + for pid in window_pids { + let before = before.get(&pid); + let after = after.get(&pid); + if before.is_none() && after.is_none() { + continue; + } + let stable = matches!( + (before, after), + (Some(before), Some(after)) + if before.process_generation.is_some() + && before.process_generation == after.process_generation + ); + if !stable { + racing_pids.insert(pid); + } + } + let mut racing_pids: Vec<_> = racing_pids.into_iter().collect(); + racing_pids.sort_unstable(); + racing_pids +} + +fn ax_windows_for_process(pid: i32) -> AxWindowMatches { + let application = unsafe { AXUIElementCreateApplication(pid) }; + if application.is_null() { + return HashMap::new(); + } + let elements = unsafe { copy_ax_windows(application) }; + unsafe { CFRelease(application as CFTypeRef) }; + let mut result = AxWindowMatches::new(); + for element in elements { + let Some(window_id) = (unsafe { bindings::ax_get_window_id(element) }) else { + unsafe { CFRelease(element as CFTypeRef) }; + continue; + }; + let minimized = unsafe { bindings::copy_bool_attr(element, "AXMinimized") }; + let retained = unsafe { RetainedAxWindow::from_owned(element) }; + result + .entry(window_id) + .or_default() + .push((retained, minimized)); + } + result +} + +#[derive(Debug, Clone)] +struct RegistryEntry { + public: WindowRef, + key: NativeWindowKey, + process: NativeProcessHandle, + native: NativeWindowHandle, + generation: WindowGeneration, + geometry_revision: GeometryRevision, + snapshot: NativeWindowSnapshot, +} + +#[derive(Debug, Clone, Copy)] +enum WindowTombstone { + Missing, + IdentityChanged, +} + +#[derive(Debug, Clone)] +struct RelatedRegistryEntry { + public: WindowRef, + key: NativeWindowKey, + process: NativeProcessHandle, + native: NativeWindowHandle, + generation: WindowGeneration, + geometry_revision: GeometryRevision, + parent: ResolvedWindowStamp, + bounds: Rect, + scale_factor: f64, + layer: i32, + is_on_screen: bool, + ax_identity: RetainedAxWindow, +} + +#[derive(Default)] +struct RegistryState { + next_generation: u64, + by_id: HashMap, + by_native: HashMap, + related_by_id: HashMap, + related_by_native: HashMap, + tombstones: HashMap, + tombstone_order: VecDeque, +} + +impl RegistryState { + fn tombstone(&mut self, id: WindowId, reason: WindowTombstone) { + self.tombstone_order.retain(|candidate| candidate != &id); + self.tombstones.insert(id.clone(), reason); + self.tombstone_order.push_back(id); + while self.tombstones.len() > MAX_WINDOW_TOMBSTONES { + if let Some(expired) = self.tombstone_order.pop_front() { + self.tombstones.remove(&expired); + } + } + } + + fn remove_related_for_parent(&mut self, parent: &WindowId, generation: WindowGeneration) { + let stale: Vec<_> = self + .related_by_id + .iter() + .filter(|(_, entry)| { + &entry.parent.window_id == parent && entry.parent.generation == generation + }) + .map(|(id, _)| id.clone()) + .collect(); + for id in stale { + if let Some(entry) = self.related_by_id.remove(&id) { + self.related_by_native.remove(&entry.key); + } + } + } +} + +#[derive(Clone)] +pub struct MacWindowRegistry { + state: Arc>, + source: Arc, + invalidations: MacInvalidationHub, +} + +impl MacWindowRegistry { + pub fn new(invalidations: MacInvalidationHub) -> Self { + Self { + state: Arc::new(Mutex::new(RegistryState::default())), + source: Arc::new(SystemWindowSnapshotSource), + invalidations, + } + } + + #[cfg(test)] + fn with_source( + source: Arc, + invalidations: MacInvalidationHub, + ) -> Self { + Self { + state: Arc::new(Mutex::new(RegistryState::default())), + source, + invalidations, + } + } + + fn refresh(&self) -> Result<(), NativeError> { + let snapshots = self.source.snapshot()?; + let mut state = self + .state + .lock() + .expect("macOS window registry lock poisoned"); + let current_keys: HashSet<_> = snapshots + .iter() + .map(|snapshot| snapshot.key.clone()) + .collect(); + + let missing: Vec<_> = state + .by_native + .keys() + .filter(|key| !current_keys.contains(*key)) + .cloned() + .collect(); + for key in missing { + if let Some(id) = state.by_native.remove(&key) { + if let Some(entry) = state.by_id.remove(&id) { + self.invalidations + .publish(TargetInvalidation::WindowGenerationChanged { + app_id: entry.public.app.id.clone(), + window_id: entry.public.id.clone(), + previous: entry.generation, + current: WindowGeneration(entry.generation.0.saturating_add(1)), + }); + state.remove_related_for_parent(&entry.public.id, entry.generation); + state.tombstone(id, WindowTombstone::Missing); + } + } + } + + for snapshot in snapshots { + if let Some(existing_id) = state.by_native.get(&snapshot.key).cloned() { + let same_identity = state.by_id.get(&existing_id).is_some_and(|entry| { + entry + .snapshot + .ax_identity + .same_identity(&snapshot.ax_identity) + }); + if same_identity { + let entry = state + .by_id + .get_mut(&existing_id) + .expect("native index points to entry"); + if entry.snapshot.bounds != snapshot.bounds + || entry.snapshot.scale_factor != snapshot.scale_factor + { + entry.geometry_revision = GeometryRevision::new(); + } + entry.public.title = snapshot.title.clone(); + entry.snapshot = snapshot; + continue; + } + state.by_native.remove(&snapshot.key); + if let Some(entry) = state.by_id.remove(&existing_id) { + self.invalidations + .publish(TargetInvalidation::WindowGenerationChanged { + app_id: entry.public.app.id.clone(), + window_id: entry.public.id.clone(), + previous: entry.generation, + current: WindowGeneration(entry.generation.0.saturating_add(1)), + }); + state.remove_related_for_parent(&entry.public.id, entry.generation); + state.tombstone(existing_id, WindowTombstone::IdentityChanged); + } + } + + state.next_generation = state.next_generation.saturating_add(1).max(1); + let generation = WindowGeneration(state.next_generation); + let id = WindowId::new(); + let app = app_ref_for_process(&snapshot.process); + let public = WindowRef { + id: id.clone(), + app, + title: snapshot.title.clone(), + }; + let process = NativeProcessHandle::new(format!( + "macos:{}:{:016x}", + snapshot.key.pid, snapshot.key.process_generation + ))?; + let native = NativeWindowHandle::new(format!( + "macos:{}:{:016x}:{}", + snapshot.key.pid, snapshot.key.process_generation, snapshot.key.cg_window_id + ))?; + state.by_native.insert(snapshot.key.clone(), id.clone()); + state.by_id.insert( + id, + RegistryEntry { + public, + key: snapshot.key.clone(), + process, + native, + generation, + geometry_revision: GeometryRevision::new(), + snapshot, + }, + ); + } + Ok(()) + } + + fn list(&self, app: Option<&AppRef>) -> Result, NativeError> { + self.refresh()?; + let state = self + .state + .lock() + .expect("macOS window registry lock poisoned"); + let mut windows: Vec<_> = state + .by_id + .values() + .filter(|entry| app.is_none_or(|app| entry.public.app.id == app.id)) + .map(|entry| entry.public.clone()) + .collect(); + windows.sort_by_key(|window| { + state + .by_id + .get(&window.id) + .map(|entry| std::cmp::Reverse(entry.snapshot.z_index)) + }); + Ok(windows) + } + + fn entry(&self, id: &WindowId, app: Option<&AppRef>) -> Result { + self.refresh()?; + let state = self + .state + .lock() + .expect("macOS window registry lock poisoned"); + let entry = match state.by_id.get(id) { + Some(entry) => entry, + None => { + let (code, message) = match state.tombstones.get(id) { + Some(WindowTombstone::IdentityChanged) => ( + ErrorCode::WindowIdentityChanged, + "native window identity changed", + ), + Some(WindowTombstone::Missing) => ( + ErrorCode::WindowNotFound, + "window closed or disappeared from WindowServer", + ), + None => ( + ErrorCode::WindowNotFound, + "window id is not known to the macOS registry", + ), + }; + return Err(NativeError::new(code, ErrorPhase::Preflight, true, message) + .with_detail("window_id", id.to_string())); + } + }; + if let Some(app) = app { + if entry.public.app.id != app.id { + return Err(identity_error(entry, "window belongs to a different app")); + } + } + Ok(entry.clone()) + } + + /// Rehydrate native facts from an exact core observation stamp. A caller + /// may not silently roll a stale observation forward across replacement, + /// process reuse, or geometry changes. + pub async fn facts_for_stamp( + &self, + stamp: &ResolvedWindowStamp, + ) -> Result { + let facts = self.facts_for_identity(stamp).await?; + if facts.stamp.geometry_revision != stamp.geometry_revision { + return Err(NativeError::stale( + ErrorCode::ObservationStale, + "window geometry changed after the observation stamp was issued", + ) + .with_detail("window_id", stamp.window_id.to_string()) + .with_detail( + "expected_geometry_revision", + stamp.geometry_revision.to_string(), + ) + .with_detail( + "current_geometry_revision", + facts.stamp.geometry_revision.to_string(), + )); + } + Ok(facts) + } + + /// Revalidate the stable native identity while allowing the caller to + /// observe a newer geometry revision. This is used to bracket capture; + /// input routes continue to require `facts_for_stamp`. + pub async fn facts_for_identity( + &self, + stamp: &ResolvedWindowStamp, + ) -> Result { + let registry = self.clone(); + let stamp = stamp.clone(); + tokio::task::spawn_blocking(move || { + let entry = registry.entry(&stamp.window_id, None)?; + if entry.public.app.id != stamp.app_id + || entry.generation != stamp.generation + || entry.native != stamp.native_window + || entry.process != stamp.process + { + return Err(identity_error( + &entry, + "resolved window stamp no longer identifies this native window", + )); + } + Ok(facts_for_entry(&entry)) + }) + .await + .map_err(join_error)? + } + + /// Register the complete current set of AX-descendant related surfaces for + /// one exact target. The join is bracketed by process generation, requires + /// an exact AX native window id and WindowServer row, and retries once as a + /// whole. Missing/dismissed/reused candidates are never rolled forward. + pub async fn register_related_windows( + &self, + parent: &MacWindowFacts, + candidates: Vec<(u32, usize)>, + ) -> Result, NativeError> { + let current_parent = self.facts_for_identity(&parent.stamp).await?; + if current_parent != *parent { + return Err(NativeError::stale( + ErrorCode::ObservationRaced, + "target facts changed before related-window registration", + )); + } + let registry = self.clone(); + let parent = parent.clone(); + tokio::task::spawn_blocking(move || { + let snapshots = match related_snapshots_once(&parent, &candidates) { + Ok(snapshots) => snapshots, + Err(_) => related_snapshots_once(&parent, &candidates).map_err(|message| { + NativeError::new( + ErrorCode::ObservationRaced, + ErrorPhase::Preflight, + true, + format!( + "related AX/WindowServer ownership changed twice during registration: {message}" + ), + ) + })?, + }; + registry.replace_related(&parent, snapshots) + }) + .await + .map_err(join_error)? + } + + /// Revalidate a captured related transient immediately before it is used. + /// The parent stamp, retained AX identity, process generation, native + /// WindowServer id, geometry and display scale must all still match the + /// observation. A dismissed, moved or id-reused transient is stale; it is + /// never rolled forward to the new native object. + pub async fn facts_for_related_stamp( + &self, + stamp: &ResolvedWindowStamp, + parent: &ResolvedWindowStamp, + ) -> Result { + let parent_facts = self.facts_for_stamp(parent).await?; + let registry = self.clone(); + let stamp = stamp.clone(); + let parent = parent.clone(); + tokio::task::spawn_blocking(move || { + let entry = { + let state = registry + .state + .lock() + .expect("macOS window registry lock poisoned"); + state.related_by_id.get(&stamp.window_id).cloned() + } + .ok_or_else(|| related_stale(&stamp, "related window is not registered"))?; + if related_stamp_for_entry(&entry) != stamp || entry.parent != parent { + return Err(related_stale( + &stamp, + "related window stamp or parent ownership changed", + )); + } + let candidates = vec![(entry.key.cg_window_id, entry.ax_identity.as_ptr() as usize)]; + let mut snapshots = related_snapshots_once(&parent_facts, &candidates) + .map_err(|message| related_stale(&stamp, message))?; + if snapshots.len() != 1 || !related_snapshot_matches_entry(&snapshots.remove(0), &entry) + { + return Err(related_stale( + &stamp, + "related window identity, geometry or display scale changed", + )); + } + Ok(related_facts_for_entry(&entry)) + }) + .await + .map_err(join_error)? + } + + fn replace_related( + &self, + parent: &MacWindowFacts, + snapshots: Vec, + ) -> Result, NativeError> { + let live_keys: HashSet<_> = snapshots + .iter() + .map(|snapshot| snapshot.key.clone()) + .collect(); + let mut state = self + .state + .lock() + .expect("macOS window registry lock poisoned"); + let stale_ids: Vec<_> = state + .related_by_id + .iter() + .filter(|(_, entry)| { + same_stable_stamp(&entry.parent, &parent.stamp) && !live_keys.contains(&entry.key) + }) + .map(|(id, _)| id.clone()) + .collect(); + for id in stale_ids { + if let Some(entry) = state.related_by_id.remove(&id) { + state.related_by_native.remove(&entry.key); + self.invalidations + .publish(TargetInvalidation::ObservationChanged { + app_id: parent.stamp.app_id.clone(), + window_id: parent.stamp.window_id.clone(), + generation: parent.stamp.generation, + reason: + cua_driver_core::api::observation::InvalidationReason::TransientDismissed, + }); + } + } + + let mut result = Vec::with_capacity(snapshots.len()); + for snapshot in snapshots { + let existing_id = state.related_by_native.get(&snapshot.key).cloned(); + if let Some(existing_id) = existing_id { + let same_identity = state + .related_by_id + .get(&existing_id) + .is_some_and(|entry| entry.ax_identity.same_identity(&snapshot.ax_identity)); + if same_identity { + let entry = state + .related_by_id + .get_mut(&existing_id) + .expect("related native index points to entry"); + if entry.bounds != snapshot.bounds + || entry.scale_factor != snapshot.scale_factor + { + entry.geometry_revision = GeometryRevision::new(); + } + entry.public.title = snapshot.title; + entry.parent = parent.stamp.clone(); + entry.bounds = snapshot.bounds; + entry.scale_factor = snapshot.scale_factor; + entry.layer = snapshot.layer; + entry.is_on_screen = snapshot.is_on_screen; + entry.ax_identity = snapshot.ax_identity; + result.push(related_facts_for_entry(entry)); + continue; + } + state.related_by_native.remove(&snapshot.key); + state.related_by_id.remove(&existing_id); + } + + state.next_generation = state.next_generation.saturating_add(1).max(1); + let generation = WindowGeneration(state.next_generation); + let id = WindowId::new(); + let public = WindowRef { + id: id.clone(), + app: AppRef { + id: parent.stamp.app_id.clone(), + name: Some(parent.owner_name.clone()), + pid: u32::try_from(parent.pid).ok(), + running: true, + }, + title: snapshot.title, + }; + let process = parent.stamp.process.clone(); + let native = NativeWindowHandle::new(format!( + "macos:{}:{:016x}:{}", + snapshot.key.pid, snapshot.key.process_generation, snapshot.key.cg_window_id + ))?; + let entry = RelatedRegistryEntry { + public, + key: snapshot.key.clone(), + process, + native, + generation, + geometry_revision: GeometryRevision::new(), + parent: parent.stamp.clone(), + bounds: snapshot.bounds, + scale_factor: snapshot.scale_factor, + layer: snapshot.layer, + is_on_screen: snapshot.is_on_screen, + ax_identity: snapshot.ax_identity, + }; + result.push(related_facts_for_entry(&entry)); + state.related_by_native.insert(snapshot.key, id.clone()); + state.related_by_id.insert(id, entry); + } + result.sort_by_key(|facts| facts.cg_window_id); + Ok(result) + } +} + +struct RelatedNativeSnapshot { + key: NativeWindowKey, + title: Option, + bounds: Rect, + scale_factor: f64, + layer: i32, + is_on_screen: bool, + ax_identity: RetainedAxWindow, +} + +fn related_snapshots_once( + parent: &MacWindowFacts, + candidates: &[(u32, usize)], +) -> Result, String> { + let application_before = nsworkspace::running_applications() + .into_iter() + .find(|application| application.pid == parent.pid) + .ok_or_else(|| "owner process disappeared before related-window join".to_owned())?; + if application_before.process_generation != Some(parent.process_generation) { + return Err("owner process generation changed before related-window join".to_owned()); + } + let server_windows = windows::all_windows_including_transients(); + let mut seen = HashSet::new(); + let mut snapshots = Vec::with_capacity(candidates.len()); + for (window_id, element) in candidates { + if !seen.insert(*window_id) { + return Err(format!("duplicate related AX window id {window_id}")); + } + let element = *element as AXUIElementRef; + if element.is_null() || unsafe { bindings::ax_get_window_id(element) } != Some(*window_id) { + return Err(format!( + "related AX identity no longer maps to window {window_id}" + )); + } + let role = unsafe { bindings::copy_string_attr(element, "AXRole") }; + if !matches!(role.as_deref(), Some("AXPopover" | "AXSheet")) { + return Err(format!( + "related window {window_id} is not an AX popover or sheet" + )); + } + let matches: Vec<_> = server_windows + .iter() + .filter(|window| window.pid == parent.pid && window.window_id == *window_id) + .collect(); + if matches.len() != 1 { + return Err(format!( + "related window {window_id} has {} WindowServer rows", + matches.len() + )); + } + let window = matches[0]; + let scale_factor = windows::display_scale_for_bounds(&window.bounds) + .ok_or_else(|| format!("related window {window_id} has no backing scale"))?; + let bounds = Rect { + x: window.bounds.x, + y: window.bounds.y, + width: window.bounds.width, + height: window.bounds.height, + }; + bounds + .validate() + .map_err(|error| format!("related window {window_id}: {error}"))?; + snapshots.push(RelatedNativeSnapshot { + key: NativeWindowKey { + pid: parent.pid, + process_generation: parent.process_generation, + cg_window_id: *window_id, + }, + title: (!window.title.is_empty()).then_some(window.title.clone()), + bounds, + scale_factor, + layer: window.layer, + is_on_screen: window.is_on_screen, + ax_identity: unsafe { RetainedAxWindow::from_borrowed(element) }, + }); + } + let application_after = nsworkspace::running_applications() + .into_iter() + .find(|application| application.pid == parent.pid) + .ok_or_else(|| "owner process disappeared after related-window join".to_owned())?; + if application_after.process_generation != Some(parent.process_generation) { + return Err("owner process generation changed during related-window join".to_owned()); + } + Ok(snapshots) +} + +fn same_stable_stamp(left: &ResolvedWindowStamp, right: &ResolvedWindowStamp) -> bool { + left.app_id == right.app_id + && left.window_id == right.window_id + && left.generation == right.generation + && left.native_window == right.native_window + && left.process == right.process +} + +fn related_facts_for_entry(entry: &RelatedRegistryEntry) -> MacRelatedWindowFacts { + MacRelatedWindowFacts { + public: entry.public.clone(), + stamp: ResolvedWindowStamp { + app_id: entry.public.app.id.clone(), + window_id: entry.public.id.clone(), + generation: entry.generation, + geometry_revision: entry.geometry_revision.clone(), + native_window: entry.native.clone(), + process: entry.process.clone(), + }, + parent: entry.parent.clone(), + pid: entry.key.pid, + process_generation: entry.key.process_generation, + cg_window_id: entry.key.cg_window_id, + bounds: entry.bounds, + scale_factor: entry.scale_factor, + layer: entry.layer, + is_on_screen: entry.is_on_screen, + } +} + +fn related_stamp_for_entry(entry: &RelatedRegistryEntry) -> ResolvedWindowStamp { + ResolvedWindowStamp { + app_id: entry.public.app.id.clone(), + window_id: entry.public.id.clone(), + generation: entry.generation, + geometry_revision: entry.geometry_revision.clone(), + native_window: entry.native.clone(), + process: entry.process.clone(), + } +} + +fn related_snapshot_matches_entry( + snapshot: &RelatedNativeSnapshot, + entry: &RelatedRegistryEntry, +) -> bool { + snapshot.key == entry.key + && snapshot.ax_identity.same_identity(&entry.ax_identity) + && snapshot.bounds == entry.bounds + && snapshot.scale_factor == entry.scale_factor + && snapshot.layer == entry.layer + && snapshot.is_on_screen == entry.is_on_screen +} + +fn related_stale(stamp: &ResolvedWindowStamp, message: impl Into) -> NativeError { + NativeError::stale(ErrorCode::SurfaceStale, message) + .with_detail("window_id", stamp.window_id.to_string()) +} + +#[async_trait] +impl WindowProvider for MacWindowRegistry { + async fn list_windows(&self, app: Option<&AppRef>) -> Result, NativeError> { + let registry = self.clone(); + let app = app.cloned(); + tokio::task::spawn_blocking(move || registry.list(app.as_ref())) + .await + .map_err(join_error)? + } + + async fn rehydrate( + &self, + id: &WindowId, + app: Option<&AppRef>, + ) -> Result { + let registry = self.clone(); + let id = id.clone(); + let app = app.cloned(); + tokio::task::spawn_blocking(move || { + registry.entry(&id, app.as_ref()).map(|entry| entry.public) + }) + .await + .map_err(join_error)? + } + + async fn resolve(&self, window: &WindowRef) -> Result { + let registry = self.clone(); + let window = window.clone(); + tokio::task::spawn_blocking(move || { + let entry = registry.entry(&window.id, Some(&window.app))?; + let scale_factor = entry.snapshot.scale_factor.ok_or_else(|| { + NativeError::unsupported( + "window does not intersect a display with a known backing scale", + ) + .with_detail("window_id", entry.public.id.to_string()) + })?; + Ok(ResolvedWindow { + public: entry.public, + native: entry.native, + process: entry.process, + framework: classify_framework(entry.snapshot.process.bundle_id.as_deref()), + geometry: WindowGeometry { + bounds: entry.snapshot.bounds, + scale_factor, + revision: entry.geometry_revision, + }, + generation: entry.generation, + state: window_state(&entry.snapshot), + }) + }) + .await + .map_err(join_error)? + } +} + +fn app_ref_for_process(process: &ProcessSnapshot) -> AppRef { + let identity = format!("macos:process:{}:{:016x}", process.pid, process.generation); + AppRef { + id: AppId::parse(identity).expect("constructed macOS app id is nonempty"), + name: process.name.clone(), + pid: u32::try_from(process.pid).ok(), + running: true, + } +} + +fn classify_framework(bundle_id: Option<&str>) -> Framework { + match bundle_id { + Some( + "com.google.Chrome" + | "com.google.Chrome.beta" + | "com.google.Chrome.dev" + | "com.google.Chrome.canary" + | "org.chromium.Chromium" + | "com.microsoft.edgemac" + | "com.microsoft.edgemac.Beta" + | "com.microsoft.edgemac.Dev" + | "com.microsoft.edgemac.Canary" + | "com.brave.Browser" + | "com.brave.Browser.beta" + | "com.brave.Browser.nightly", + ) => Framework::Chromium, + Some("com.apple.Safari" | "com.apple.SafariTechnologyPreview") => Framework::WebKit, + _ => Framework::Unknown, + } +} + +fn window_state(snapshot: &NativeWindowSnapshot) -> WindowStateKind { + if snapshot.minimized == Some(true) { + WindowStateKind::Minimized + } else if snapshot.minimized.is_none() { + WindowStateKind::Unknown + } else if snapshot.on_current_space == Some(false) { + WindowStateKind::OffSpace + } else if snapshot.process.hidden || snapshot.on_current_space.is_none() { + WindowStateKind::Unknown + } else if snapshot.is_on_screen { + WindowStateKind::Visible + } else { + WindowStateKind::Unknown + } +} + +fn facts_for_entry(entry: &RegistryEntry) -> MacWindowFacts { + MacWindowFacts { + stamp: ResolvedWindowStamp { + app_id: entry.public.app.id.clone(), + window_id: entry.public.id.clone(), + generation: entry.generation, + geometry_revision: entry.geometry_revision.clone(), + native_window: entry.native.clone(), + process: entry.process.clone(), + }, + pid: entry.key.pid, + process_generation: entry.key.process_generation, + cg_window_id: entry.key.cg_window_id, + owner_name: entry.snapshot.owner_name.clone(), + layer: entry.snapshot.layer, + bounds: entry.snapshot.bounds, + scale_factor: entry.snapshot.scale_factor, + state: window_state(&entry.snapshot), + is_on_screen: entry.snapshot.is_on_screen, + on_current_space: entry.snapshot.on_current_space, + space_ids: entry.snapshot.space_ids.clone(), + minimized: entry.snapshot.minimized, + } +} + +fn identity_error(entry: &RegistryEntry, message: &str) -> NativeError { + NativeError::new( + ErrorCode::WindowIdentityChanged, + ErrorPhase::Preflight, + false, + message, + ) + .with_detail("window_id", entry.public.id.to_string()) + .with_detail("pid", entry.key.pid) + .with_detail("cg_window_id", entry.key.cg_window_id) + .with_detail("generation", entry.generation.0) +} + +fn join_error(error: tokio::task::JoinError) -> NativeError { + NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + true, + format!("macOS blocking lifecycle task failed: {error}"), + ) +} + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Mutex}; + + use core_foundation::{base::TCFType, string::CFString}; + use cua_driver_core::api::{ + contracts::{AppId, AppRef}, + errors::ErrorCode, + platform::{InvalidationSubscription, TargetInvalidation, WindowProvider}, + }; + + use super::*; + + #[derive(Default)] + struct FakeSnapshotSource { + snapshots: Mutex>, + } + + impl FakeSnapshotSource { + fn replace(&self, snapshots: Vec) { + *self.snapshots.lock().expect("fake source lock poisoned") = snapshots; + } + } + + impl WindowSnapshotSource for FakeSnapshotSource { + fn snapshot(&self) -> Result, NativeError> { + Ok(self + .snapshots + .lock() + .expect("fake source lock poisoned") + .clone()) + } + } + + fn ax_identity(label: &str) -> RetainedAxWindow { + let value = CFString::new(label); + let raw = value.as_CFTypeRef(); + unsafe { CFRetain(raw) }; + unsafe { RetainedAxWindow::from_owned(raw.cast_mut().cast()) } + } + + fn snapshot( + bundle_id: &str, + process_generation: u64, + cg_window_id: u32, + title: Option<&str>, + ax_label: &str, + ) -> NativeWindowSnapshot { + NativeWindowSnapshot { + key: NativeWindowKey { + pid: 101, + process_generation, + cg_window_id, + }, + process: ProcessSnapshot { + pid: 101, + generation: process_generation, + name: Some("Fixture".to_owned()), + bundle_id: Some(bundle_id.to_owned()), + hidden: false, + }, + owner_name: "Fixture".to_owned(), + title: title.map(str::to_owned), + bounds: Rect { + x: 10.0, + y: 20.0, + width: 800.0, + height: 600.0, + }, + layer: 0, + z_index: 4, + is_on_screen: true, + on_current_space: Some(true), + space_ids: Some(vec![7]), + minimized: Some(false), + scale_factor: Some(2.0), + ax_identity: ax_identity(ax_label), + } + } + + fn registry() -> ( + MacWindowRegistry, + Arc, + MacInvalidationHub, + ) { + let source = Arc::new(FakeSnapshotSource::default()); + let invalidations = MacInvalidationHub::default(); + ( + MacWindowRegistry::with_source(source.clone(), invalidations.clone()), + source, + invalidations, + ) + } + + #[tokio::test] + async fn no_title_window_keeps_opaque_identity_across_metadata_changes() { + let (registry, source, _) = registry(); + source.replace(vec![snapshot("com.example.fixture", 1, 44, None, "ax-1")]); + let first = registry.list_windows(None).await.unwrap(); + assert_eq!(first.len(), 1); + assert_eq!(first[0].title, None); + + source.replace(vec![snapshot( + "com.example.fixture", + 1, + 44, + Some("Document"), + "ax-1", + )]); + let second = registry.list_windows(None).await.unwrap(); + assert_eq!(second[0].id, first[0].id); + assert_eq!(second[0].title.as_deref(), Some("Document")); + } + + #[test] + fn framework_classification_is_exact_and_never_infers_electron() { + assert_eq!( + classify_framework(Some("com.google.Chrome")), + Framework::Chromium + ); + assert_eq!( + classify_framework(Some("com.apple.Safari")), + Framework::WebKit + ); + assert_eq!( + classify_framework(Some("com.example.electron-looking-app")), + Framework::Unknown + ); + assert_eq!(classify_framework(None), Framework::Unknown); + } + + #[tokio::test] + async fn native_facts_require_the_exact_live_geometry_stamp() { + let (registry, source, _) = registry(); + let first = snapshot("com.example.fixture", 1, 44, None, "ax-1"); + source.replace(vec![first.clone()]); + let public = registry.list_windows(None).await.unwrap().remove(0); + let resolved = registry.resolve(&public).await.unwrap(); + let facts = registry.facts_for_stamp(&resolved.stamp()).await.unwrap(); + assert_eq!(facts.pid, 101); + assert_eq!(facts.cg_window_id, 44); + assert_eq!(facts.layer, 0); + assert_eq!(facts.space_ids, Some(vec![7])); + + let mut moved = first; + moved.bounds.x += 20.0; + source.replace(vec![moved]); + let refreshed = registry + .facts_for_identity(&resolved.stamp()) + .await + .unwrap(); + assert_eq!(refreshed.bounds.x, 30.0); + let error = registry + .facts_for_stamp(&resolved.stamp()) + .await + .unwrap_err(); + assert_eq!(error.code, ErrorCode::ObservationStale); + } + + #[tokio::test] + async fn reused_native_key_with_different_ax_identity_revokes_old_window() { + let (registry, source, invalidations) = registry(); + let mut events = invalidations.subscribe(); + source.replace(vec![snapshot("com.example.fixture", 1, 44, None, "ax-1")]); + let old = registry.list_windows(None).await.unwrap().remove(0); + + source.replace(vec![snapshot("com.example.fixture", 1, 44, None, "ax-2")]); + let replacement = registry.list_windows(None).await.unwrap().remove(0); + assert_ne!(replacement.id, old.id); + let error = registry.rehydrate(&old.id, None).await.unwrap_err(); + assert_eq!(error.code, ErrorCode::WindowIdentityChanged); + assert!(matches!( + events.next().await, + Some(TargetInvalidation::WindowGenerationChanged { window_id, .. }) + if window_id == old.id + )); + } + + #[tokio::test] + async fn identity_churn_releases_live_entries_and_bounds_tombstones() { + let (registry, source, _) = registry(); + for generation in 0..(MAX_WINDOW_TOMBSTONES + 8) { + source.replace(vec![snapshot( + "com.example.fixture", + 1, + 44, + None, + &format!("ax-{generation}"), + )]); + assert_eq!(registry.list_windows(None).await.unwrap().len(), 1); + } + + let state = registry + .state + .lock() + .expect("macOS window registry lock poisoned"); + assert_eq!(state.by_id.len(), 1); + assert_eq!(state.by_native.len(), 1); + assert_eq!(state.tombstones.len(), MAX_WINDOW_TOMBSTONES); + assert_eq!(state.tombstone_order.len(), MAX_WINDOW_TOMBSTONES); + } + + #[tokio::test] + async fn owner_generation_change_never_rehydrates_through_pid_and_cg_id_reuse() { + let (registry, source, _) = registry(); + source.replace(vec![snapshot("com.example.fixture", 10, 44, None, "ax-1")]); + let old = registry.list_windows(None).await.unwrap().remove(0); + + source.replace(vec![snapshot("com.example.fixture", 11, 44, None, "ax-1")]); + let replacement = registry.list_windows(None).await.unwrap().remove(0); + assert_ne!(replacement.id, old.id); + assert_ne!(replacement.app.id, old.app.id); + let error = registry.rehydrate(&old.id, None).await.unwrap_err(); + assert_eq!(error.code, ErrorCode::WindowNotFound); + } + + #[tokio::test] + async fn duplicate_bundle_instances_have_distinct_running_app_ids() { + let (registry, source, _) = registry(); + let first = snapshot("com.example.fixture", 10, 44, None, "ax-1"); + let mut second = snapshot("com.example.fixture", 20, 45, None, "ax-2"); + second.key.pid = 202; + second.process.pid = 202; + source.replace(vec![first, second]); + + let windows = registry.list_windows(None).await.unwrap(); + assert_eq!(windows.len(), 2); + assert_ne!(windows[0].app.id, windows[1].app.id); + } + + #[tokio::test] + async fn cross_app_rehydration_is_an_identity_error() { + let (registry, source, _) = registry(); + source.replace(vec![snapshot("com.example.fixture", 1, 44, None, "ax-1")]); + let window = registry.list_windows(None).await.unwrap().remove(0); + let other_app = AppRef { + id: AppId::parse("macos:bundle:com.example.other").unwrap(), + name: Some("Other".to_owned()), + pid: Some(102), + running: true, + }; + + let error = registry + .rehydrate(&window.id, Some(&other_app)) + .await + .unwrap_err(); + assert_eq!(error.code, ErrorCode::WindowIdentityChanged); + } + + #[test] + fn process_generation_bracket_rejects_missing_or_reused_processes() { + fn application(pid: i32, generation: Option) -> RunningApplicationInfo { + RunningApplicationInfo { + pid, + name: Some("Fixture".to_owned()), + bundle_id: Some("com.example.fixture".to_owned()), + executable_path: None, + process_generation: generation, + active: false, + hidden: false, + finished_launching: true, + regular: true, + } + } + + let before = HashMap::from([(101, application(101, Some(7)))]); + let stable = HashMap::from([(101, application(101, Some(7)))]); + assert!(racing_process_pids([101], &before, &stable).is_empty()); + + let reused = HashMap::from([(101, application(101, Some(8)))]); + assert_eq!(racing_process_pids([101], &before, &reused), vec![101]); + + let unknown = HashMap::from([(101, application(101, None))]); + assert_eq!(racing_process_pids([101], &unknown, &unknown), vec![101]); + } + + #[test] + fn unreadable_ax_minimized_state_is_unknown() { + let mut fixture = snapshot("com.example.fixture", 1, 44, None, "ax-1"); + fixture.minimized = None; + assert_eq!(window_state(&fixture), WindowStateKind::Unknown); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs index 65ccf875eb..a4d6ab1bfc 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/focus_steal.rs @@ -56,12 +56,16 @@ //! queue's own thread regardless of run-loop state. use std::collections::HashMap; -use std::sync::{Arc, Mutex, OnceLock}; +use std::sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, Mutex, OnceLock, +}; use std::time::{Duration, Instant}; +use objc2::rc::Retained; use objc2_app_kit::{ - NSApplicationActivationOptions, NSRunningApplication, NSWorkspace, - NSWorkspaceDidActivateApplicationNotification, NSWorkspaceApplicationKey, + NSApplicationActivationOptions, NSRunningApplication, NSWorkspace, NSWorkspaceApplicationKey, + NSWorkspaceDidActivateApplicationNotification, }; use objc2_foundation::NSOperationQueue; use uuid::Uuid; @@ -96,17 +100,39 @@ struct Entry { /// Provenance for tracing — e.g. `"LaunchAppTool.pre"`. #[allow(dead_code)] origin: &'static str, + evidence: Arc, +} + +#[derive(Debug, Default)] +struct SuppressionEvidenceState { + activations: AtomicUsize, + restore_attempts: AtomicUsize, + restore_failures: AtomicUsize, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SuppressionOutcome { + pub activations: usize, + pub restore_attempts: usize, + pub restore_failures: usize, +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct SuppressionCloseOutcome { + pub evidence: SuppressionOutcome, + /// True only when removal ran on the serial observer queue after every + /// containment callback already queued there. + pub callback_queue_drained: bool, } /// Singleton focus-steal preventer. /// /// Constructed lazily on first `shared()` call. Owns the dispatcher state -/// (Sync via the inner Mutex); the NSWorkspace observer + queue are -/// intentionally retained-and-forgotten on install so their lifetime is -/// the whole process and we don't need to thread `!Send` Cocoa handles -/// through this struct. +/// (Sync via the inner Mutex); the observer token is process-lifetime and the +/// serial callback queue is retained for bounded evidence barriers. pub struct FocusStealPreventer { dispatcher: Arc, + observer_queue: Retained, } impl FocusStealPreventer { @@ -116,8 +142,11 @@ impl FocusStealPreventer { SINGLETON .get_or_init(|| { let dispatcher = Arc::new(Dispatcher::new()); - install_observer(&dispatcher); - Arc::new(FocusStealPreventer { dispatcher }) + let observer_queue = install_observer(&dispatcher); + Arc::new(FocusStealPreventer { + dispatcher, + observer_queue, + }) }) .clone() } @@ -132,18 +161,58 @@ impl FocusStealPreventer { target_pid: Option, restore_to: i32, origin: &'static str, + ) -> SuppressionLease { + Self::begin_suppression_until( + target_pid, + restore_to, + origin, + Instant::now() + ENTRY_DEADLINE, + ) + } + + /// Begin a caller-bounded suppression whose leak deadline covers the + /// complete native operation span. Long-running v2 operations use their + /// own deadline instead of silently losing containment after five seconds. + pub fn begin_suppression_until( + target_pid: Option, + restore_to: i32, + origin: &'static str, + deadline: Instant, ) -> SuppressionLease { let shared = Self::shared(); let handle = shared .dispatcher - .add(target_pid, restore_to, origin); + .add_until(target_pid, restore_to, origin, deadline); SuppressionLease { handle, dispatcher: Arc::clone(&shared.dispatcher), + evidence: shared.dispatcher.evidence(handle), released: false, } } + /// Register containment whose dispatcher entry, rather than a future or + /// callback stack frame, owns the native deadline. Dropping every Rust + /// handle intentionally leaves the entry armed until its deadline; normal + /// completion must call [`DeadlineSuppression::close_with_evidence`]. + pub fn begin_deadline_owned_suppression_until( + target_pid: Option, + restore_to: i32, + origin: &'static str, + deadline: Instant, + ) -> DeadlineSuppression { + let shared = Self::shared(); + let handle = shared + .dispatcher + .add_until(target_pid, restore_to, origin, deadline); + DeadlineSuppression { + handle, + dispatcher: Arc::clone(&shared.dispatcher), + evidence: shared.dispatcher.evidence(handle), + closed: Arc::new(AtomicBool::new(false)), + } + } + /// Run `f` with a suppression entry active. Equivalent to /// `begin_suppression(...)` + run `f` + drop the lease — but expressed /// as a single async call site. @@ -161,10 +230,73 @@ impl FocusStealPreventer { f().await } - /// For tests. Returns the singleton's dispatcher arc. - #[cfg(test)] - fn dispatcher(&self) -> &Arc { - &self.dispatcher + /// Wait until every focus-containment callback queued before this call has + /// run. Callers fail closed when the bounded barrier does not complete. + pub fn barrier(timeout: Duration) -> bool { + let shared = Self::shared(); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let block = block2::RcBlock::new(move || { + let _ = sender.send(()); + }); + unsafe { shared.observer_queue.addBarrierBlock(&block) }; + receiver.recv_timeout(timeout).is_ok() + } + + fn close_handle( + dispatcher: Arc, + handle: SuppressionHandle, + evidence: Arc, + timeout: Duration, + ) -> SuppressionCloseOutcome { + let shared = Self::shared(); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let queued_dispatcher = Arc::clone(&dispatcher); + let queued_evidence = Arc::clone(&evidence); + let block = block2::RcBlock::new(move || { + queued_dispatcher.remove(handle); + let _ = sender.send(suppression_outcome(&queued_evidence)); + }); + unsafe { shared.observer_queue.addBarrierBlock(&block) }; + match receiver.recv_timeout(timeout) { + Ok(evidence) => SuppressionCloseOutcome { + evidence, + callback_queue_drained: true, + }, + Err(_) => { + // Never strand containment just because the proof barrier + // timed out. Removal and the best evidence currently visible + // are still returned; the caller must classify this as + // posture_unverifiable rather than success. + dispatcher.remove(handle); + SuppressionCloseOutcome { + evidence: suppression_outcome(&evidence), + callback_queue_drained: false, + } + } + } + } + + fn narrow_handle( + dispatcher: Arc, + handle: SuppressionHandle, + target_pid: i32, + timeout: Duration, + ) -> bool { + let shared = Self::shared(); + let (sender, receiver) = std::sync::mpsc::sync_channel(1); + let queued_dispatcher = Arc::clone(&dispatcher); + let block = block2::RcBlock::new(move || { + queued_dispatcher.retarget(handle, Some(target_pid)); + let _ = sender.send(()); + }); + unsafe { shared.observer_queue.addBarrierBlock(&block) }; + if receiver.recv_timeout(timeout).is_ok() { + true + } else { + // Preserve containment even when the ordering proof is lost. + dispatcher.retarget(handle, Some(target_pid)); + false + } } } @@ -177,11 +309,79 @@ pub fn begin_suppression( FocusStealPreventer::begin_suppression(target_pid, restore_to, origin) } +/// Begin suppression through a caller-owned bounded native deadline. +pub fn begin_suppression_until( + target_pid: Option, + restore_to: i32, + origin: &'static str, + deadline: Instant, +) -> SuppressionLease { + FocusStealPreventer::begin_suppression_until(target_pid, restore_to, origin, deadline) +} + +pub fn begin_deadline_owned_suppression_until( + target_pid: Option, + restore_to: i32, + origin: &'static str, + deadline: Instant, +) -> DeadlineSuppression { + FocusStealPreventer::begin_deadline_owned_suppression_until( + target_pid, restore_to, origin, deadline, + ) +} + +/// Cloneable handle to a dispatcher-owned containment entry. `Drop` is +/// intentionally a no-op: cancellation must not disarm a LaunchServices +/// callback that can still activate an app before the native deadline. +#[derive(Clone)] +pub struct DeadlineSuppression { + handle: SuppressionHandle, + dispatcher: Arc, + evidence: Arc, + closed: Arc, +} + +impl DeadlineSuppression { + /// Narrow a launch wildcard to the pid returned by LaunchServices without + /// a remove/add gap. The serial-queue result is evidence, not best effort. + pub fn narrow_to_target(&self, target_pid: i32, timeout: Duration) -> bool { + if self.closed.load(Ordering::Acquire) { + return false; + } + FocusStealPreventer::narrow_handle( + Arc::clone(&self.dispatcher), + self.handle, + target_pid, + timeout, + ) + } + + pub fn close_with_evidence(&self, timeout: Duration) -> SuppressionCloseOutcome { + if self.closed.swap(true, Ordering::AcqRel) { + return SuppressionCloseOutcome { + evidence: suppression_outcome(&self.evidence), + callback_queue_drained: true, + }; + } + FocusStealPreventer::close_handle( + Arc::clone(&self.dispatcher), + self.handle, + Arc::clone(&self.evidence), + timeout, + ) + } + + pub fn outcome(&self) -> SuppressionOutcome { + suppression_outcome(&self.evidence) + } +} + /// RAII lease. `Drop` ends the entry synchronously, so the entry is /// removed even if a future is cancelled mid-await. pub struct SuppressionLease { handle: SuppressionHandle, dispatcher: Arc, + evidence: Arc, released: bool, } @@ -192,6 +392,35 @@ impl SuppressionLease { self.dispatcher.remove(self.handle); self.released = true; } + + pub fn release_with_evidence(mut self) -> SuppressionOutcome { + self.dispatcher.remove(self.handle); + self.released = true; + suppression_outcome(&self.evidence) + } + + pub fn close_with_evidence(mut self, timeout: Duration) -> SuppressionCloseOutcome { + let outcome = FocusStealPreventer::close_handle( + Arc::clone(&self.dispatcher), + self.handle, + Arc::clone(&self.evidence), + timeout, + ); + self.released = true; + outcome + } + + pub fn outcome(&self) -> SuppressionOutcome { + suppression_outcome(&self.evidence) + } +} + +fn suppression_outcome(state: &SuppressionEvidenceState) -> SuppressionOutcome { + SuppressionOutcome { + activations: state.activations.load(Ordering::Acquire), + restore_attempts: state.restore_attempts.load(Ordering::Acquire), + restore_failures: state.restore_failures.load(Ordering::Acquire), + } } impl Drop for SuppressionLease { @@ -236,18 +465,35 @@ impl Dispatcher { /// subsequent adds would skip the kick and the janitor never /// started, leaving deadline-reaping entirely up to the /// `snapshot_matches` reap fallback (only fires on an activation). + #[cfg(test)] fn add( self: &Arc, target_pid: Option, restore_to: i32, origin: &'static str, + ) -> SuppressionHandle { + self.add_until( + target_pid, + restore_to, + origin, + Instant::now() + ENTRY_DEADLINE, + ) + } + + fn add_until( + self: &Arc, + target_pid: Option, + restore_to: i32, + origin: &'static str, + deadline: Instant, ) -> SuppressionHandle { let id = Uuid::new_v4(); let entry = Entry { target_pid, restore_to, - deadline: Instant::now() + ENTRY_DEADLINE, + deadline, origin, + evidence: Arc::new(SuppressionEvidenceState::default()), }; { let mut guard = self.entries.lock().unwrap(); @@ -261,6 +507,18 @@ impl Dispatcher { SuppressionHandle(id) } + fn evidence(&self, handle: SuppressionHandle) -> Arc { + Arc::clone( + &self + .entries + .lock() + .expect("focus suppression dispatcher poisoned") + .get(&handle.0) + .expect("new suppression entry must retain evidence") + .evidence, + ) + } + /// Remove an entry. When the map drains to empty, signals the janitor /// to stop until the next add. fn remove(&self, handle: SuppressionHandle) { @@ -274,10 +532,21 @@ impl Dispatcher { } } + fn retarget(&self, handle: SuppressionHandle, target_pid: Option) { + if let Some(entry) = self + .entries + .lock() + .expect("focus suppression dispatcher poisoned") + .get_mut(&handle.0) + { + entry.target_pid = target_pid; + } + } + /// Snapshot the entries (cloned to a small Vec) — used by tests /// and the activation handler to evaluate matches without holding /// the lock across the restore call. - fn snapshot_matches(&self, activated_pid: i32) -> Vec { + fn snapshot_matches(&self, activated_pid: i32) -> Vec<(i32, Arc)> { let mut guard = self.entries.lock().unwrap(); // Reap expired entries first — keeps the dispatcher honest even // if the janitor hasn't ticked yet. @@ -294,7 +563,7 @@ impl Dispatcher { None => activated_pid != e.restore_to, } }) - .map(|e| e.restore_to) + .map(|e| (e.restore_to, Arc::clone(&e.evidence))) .collect() } @@ -393,7 +662,7 @@ impl Dispatcher { /// down. Forgetting avoids having to thread `!Send` `Retained<...>` /// handles through `FocusStealPreventer` (which lives in `Arc<...>` / /// `OnceLock<...>` and therefore needs to be `Send + Sync`). -fn install_observer(dispatcher: &Arc) { +fn install_observer(dispatcher: &Arc) -> Retained { use block2::RcBlock; use objc2_foundation::NSNotification; use std::ptr::NonNull; @@ -430,11 +699,11 @@ fn install_observer(dispatcher: &Arc) { ) }; - // Intentionally leak both — the observer needs to outlive any - // particular `Arc` and the singleton has - // process lifetime. + // The observer token is process-lifetime. The singleton retains the queue + // so bounded callers can enqueue an exact barrier on this same callback + // stream before reading suppression evidence. std::mem::forget(token); - std::mem::forget(queue); + queue } /// Match a single activation notification against the dispatcher and, @@ -442,10 +711,7 @@ fn install_observer(dispatcher: &Arc) { /// /// Runs on the observer queue's background thread — safe to call /// blocking system APIs. -fn handle_activation( - dispatcher: &Arc, - note: &objc2_foundation::NSNotification, -) { +fn handle_activation(dispatcher: &Arc, note: &objc2_foundation::NSNotification) { use objc2::msg_send; use objc2::runtime::AnyObject; @@ -457,8 +723,7 @@ fn handle_activation( // userInfo[NSWorkspaceApplicationKey] -> NSRunningApplication*. // We go through a raw msg_send to avoid Retained generic // bookkeeping for the cross-cast. - let app_ptr: *mut AnyObject = - msg_send![&*info, objectForKey: NSWorkspaceApplicationKey]; + let app_ptr: *mut AnyObject = msg_send![&*info, objectForKey: NSWorkspaceApplicationKey]; if app_ptr.is_null() { return; } @@ -467,21 +732,24 @@ fn handle_activation( }; let restore_pids = dispatcher.snapshot_matches(activated_pid); - for pid in restore_pids { - restore_focus(pid); + for (pid, evidence) in restore_pids { + evidence.activations.fetch_add(1, Ordering::AcqRel); + evidence.restore_attempts.fetch_add(1, Ordering::AcqRel); + if !restore_focus(pid) { + evidence.restore_failures.fetch_add(1, Ordering::AcqRel); + } } } /// Re-activate `pid` if it's still running. Safe to call from any /// thread — Apple documents `activateWithOptions:` as thread-safe. -fn restore_focus(pid: i32) { +fn restore_focus(pid: i32) -> bool { unsafe { - if let Some(app) = - NSRunningApplication::runningApplicationWithProcessIdentifier(pid) - { - let _ = app.activateWithOptions(NSApplicationActivationOptions(0)); + if let Some(app) = NSRunningApplication::runningApplicationWithProcessIdentifier(pid) { + return app.activateWithOptions(NSApplicationActivationOptions(0)); } } + false } // ── Tests ──────────────────────────────────────────────────────────────────── @@ -503,7 +771,10 @@ mod tests { let h = d.add(Some(42), 7, "test.add"); assert_eq!(d.len(), 1); let matches = d.snapshot_matches(42); - assert_eq!(matches, vec![7]); + assert_eq!( + matches.into_iter().map(|(pid, _)| pid).collect::>(), + vec![7] + ); // Non-matching pid: no restore candidates. assert!(d.snapshot_matches(99).is_empty()); d.remove(h); @@ -517,7 +788,13 @@ mod tests { let d = Arc::new(Dispatcher::new()); let _h = d.add(None, 7, "test.wild"); // pid 99 != restore_to 7 → should match. - assert_eq!(d.snapshot_matches(99), vec![7]); + assert_eq!( + d.snapshot_matches(99) + .into_iter() + .map(|(pid, _)| pid) + .collect::>(), + vec![7] + ); // pid 7 == restore_to → must NOT match (don't fight ourselves). assert!(d.snapshot_matches(7).is_empty()); } @@ -531,6 +808,7 @@ mod tests { let lease = SuppressionLease { handle: h, dispatcher: Arc::clone(&d), + evidence: d.evidence(h), released: false, }; assert_eq!(d.len(), 1); @@ -546,6 +824,7 @@ mod tests { let lease = SuppressionLease { handle: h, dispatcher: Arc::clone(&d), + evidence: d.evidence(h), released: false, }; lease.release(); @@ -568,6 +847,7 @@ mod tests { restore_to: 7, deadline: Instant::now() - Duration::from_secs(1), origin: "test.leak", + evidence: Arc::new(SuppressionEvidenceState::default()), }, ); } @@ -578,6 +858,59 @@ mod tests { assert_eq!(d.len(), 0, "snapshot_matches should purge expired"); } + #[test] + fn caller_deadline_covers_long_native_operation_span() { + let d = Arc::new(Dispatcher::new()); + let requested = Instant::now() + Duration::from_secs(12); + let handle = d.add_until(Some(42), 7, "test.long_operation", requested); + let recorded = d.entries.lock().unwrap().get(&handle.0).unwrap().deadline; + assert_eq!(recorded, requested); + d.remove(handle); + } + + #[test] + fn deadline_owned_containment_survives_owner_cancellation_until_deadline() { + let d = Arc::new(Dispatcher::new()); + let deadline = Instant::now() + Duration::from_secs(12); + let handle = d.add_until(None, 7, "test.detached_launch", deadline); + let containment = DeadlineSuppression { + handle, + dispatcher: Arc::clone(&d), + evidence: d.evidence(handle), + closed: Arc::new(AtomicBool::new(false)), + }; + drop(containment); + assert_eq!(d.len(), 1, "Drop must not disarm a late LS callback"); + assert_eq!(d.entries.lock().unwrap()[&handle.0].deadline, deadline); + d.remove(handle); + } + + #[test] + fn wildcard_narrows_in_place_without_a_remove_add_gap() { + let d = Arc::new(Dispatcher::new()); + let handle = d.add(None, 7, "test.narrow"); + d.retarget(handle, Some(42)); + assert!(d.snapshot_matches(99).is_empty()); + assert_eq!(d.snapshot_matches(42).len(), 1); + assert_eq!(d.len(), 1); + d.remove(handle); + } + + #[test] + fn serial_close_removes_after_preceding_containment_callbacks() { + let d = Arc::new(Dispatcher::new()); + let handle = d.add(Some(42), 7, "test.serial_close"); + let lease = SuppressionLease { + handle, + dispatcher: Arc::clone(&d), + evidence: d.evidence(handle), + released: false, + }; + let close = lease.close_with_evidence(Duration::from_secs(1)); + assert!(close.callback_queue_drained); + assert_eq!(d.len(), 0); + } + /// Janitor lifecycle: starts on first add, stops when empty, /// restarts on next add. Spin up a tokio runtime to host the task. #[tokio::test(flavor = "multi_thread", worker_threads = 2)] @@ -661,7 +994,13 @@ mod tests { let matches = d.snapshot_matches(42); assert_eq!(matches.len(), 2); // Set equality — order is HashMap-dependent. - assert!(matches.contains(&1)); - assert!(matches.contains(&2)); + let restore_pids: Vec<_> = matches.into_iter().map(|(pid, _)| pid).collect(); + assert!(restore_pids.contains(&1)); + assert!(restore_pids.contains(&2)); + } + + #[test] + fn retained_observer_queue_exposes_a_bounded_teardown_barrier() { + assert!(FocusStealPreventer::barrier(Duration::from_secs(1))); } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/mod.rs index 3bb24ffe51..cc3e0921d4 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/input/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/input/mod.rs @@ -8,11 +8,12 @@ //! public `CGEventPostToPid` to reach Catalyst/Chromium apps and trigger //! the activity-monitor tickle required for live-input detection. -pub mod mouse; -pub mod keyboard; pub mod ax_actions; +pub mod keyboard; +pub mod mouse; pub mod skylight; +pub mod slps_make_key; pub use ax_actions::perform_ax_action; +pub use keyboard::{hotkey, press_key, type_text}; pub use mouse::click_at_xy; -pub use keyboard::{press_key, type_text, hotkey}; diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs index ec6dd54148..e6b21d960d 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs @@ -16,10 +16,10 @@ //! If anything fails to resolve the functions return `false` and callers //! fall back to the public `CGEvent::post_to_pid`. -use std::ffi::{CStr, c_void}; -use std::os::raw::{c_int, c_uint, c_char}; -use std::sync::OnceLock; use libc::pid_t; +use std::ffi::{c_void, CStr}; +use std::os::raw::{c_char, c_int, c_uint}; +use std::sync::OnceLock; // ── Function-pointer typedefs ────────────────────────────────────────────── @@ -87,7 +87,10 @@ fn ensure_skylight_loaded() { LOADED.get_or_init(|| { let path = b"/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight\0"; unsafe { - libc::dlopen(path.as_ptr() as *const c_char, libc::RTLD_LAZY | libc::RTLD_GLOBAL); + libc::dlopen( + path.as_ptr() as *const c_char, + libc::RTLD_LAZY | libc::RTLD_GLOBAL, + ); } }); } @@ -96,10 +99,12 @@ fn ensure_skylight_loaded() { /// Returns `None` when the symbol doesn't resolve. fn find_sym(name: &[u8]) -> Option<*mut c_void> { ensure_skylight_loaded(); - let ptr = unsafe { - libc::dlsym(libc::RTLD_DEFAULT, name.as_ptr() as *const c_char) - }; - if ptr.is_null() { None } else { Some(ptr) } + let ptr = unsafe { libc::dlsym(libc::RTLD_DEFAULT, name.as_ptr() as *const c_char) }; + if ptr.is_null() { + None + } else { + Some(ptr) + } } /// Reinterpret a raw symbol pointer as a function pointer of type `T`. @@ -182,6 +187,46 @@ pub fn is_focus_without_raise_available() -> bool { && post_event_record_to_fn().is_some() } +/// `true` when the two symbols needed by the v2 target-only SLPS make-key route +/// resolved. Unlike `is_focus_without_raise_available`, this deliberately +/// does not require or call the front-process SPI. +pub fn is_target_slps_make_key_available() -> bool { + get_process_for_pid_fn().is_some() && post_event_record_to_fn().is_some() +} + +pub const EVENT_RECORD_LEN: usize = 0xF8; +pub const SLPS_MAKE_KEY_MARKER: u8 = 0x01; +pub const SLPS_REMOVE_KEY_MARKER: u8 = 0x02; + +/// Build the recovered SLPS make-key/remove-key record without selecting a destination. +/// The caller must post it only to the intended target process PSN. +pub fn build_slps_make_key_record(window_id: u32, marker: u8) -> [u8; EVENT_RECORD_LEN] { + let mut record = [0u8; EVENT_RECORD_LEN]; + record[0x04] = EVENT_RECORD_LEN as u8; + record[0x08] = 0x0D; + record[0x3C..=0x3F].copy_from_slice(&window_id.to_le_bytes()); + record[0x8A] = marker; + record +} + +/// Post one recovered make-key/remove-key record to the target process only. +/// This deliberately does not defocus the real foreground process and never +/// calls a global event-posting or front-process primitive. +pub fn post_slps_make_key_record_to_pid(pid: pid_t, window_id: u32, marker: u8) -> bool { + let Some(post) = post_event_record_to_fn() else { + return false; + }; + let Some(get_process) = get_process_for_pid_fn() else { + return false; + }; + let mut target_psn = [0u8; 8]; + if unsafe { get_process(pid, target_psn.as_mut_ptr().cast()) } != 0 { + return false; + } + let record = build_slps_make_key_record(window_id, marker); + unsafe { post(target_psn.as_ptr().cast(), record.as_ptr()) == 0 } +} + // ── ObjC runtime helpers ─────────────────────────────────────────────────── /// Look up an ObjC class by C-string name via `objc_getClass`. @@ -220,8 +265,8 @@ fn class_responds_to_selector(cls: *mut c_void, sel: *mut c_void) -> bool { } type RespondsToFn = unsafe extern "C" fn(*mut c_void, *mut c_void) -> bool; static SYM: OnceLock> = OnceLock::new(); - let f = *SYM - .get_or_init(|| find_sym(b"class_respondsToSelector\0").map(|p| unsafe { as_fn(p) })); + let f = + *SYM.get_or_init(|| find_sym(b"class_respondsToSelector\0").map(|p| unsafe { as_fn(p) })); match f { Some(f) => unsafe { f(cls, sel) }, None => false, @@ -238,9 +283,7 @@ fn class_responds_to_selector(cls: *mut c_void, sel: *mut c_void) -> bool { /// We probe offsets 24, 32, 16 for resilience across OS versions (same as Swift). unsafe fn extract_event_record(event_ptr: *mut c_void) -> *mut c_void { for &offset in &[24usize, 32, 16] { - let slot = (event_ptr as *const u8) - .add(offset) - .cast::<*mut c_void>(); + let slot = (event_ptr as *const u8).add(offset).cast::<*mut c_void>(); let p = std::ptr::read_unaligned(slot); if !p.is_null() { return p; @@ -301,7 +344,10 @@ pub fn post_to_pid(pid: pid_t, event_ptr: *mut c_void, attach_auth_message: bool /// `CGEventSetWindowLocation` SPI. Returns `true` when the SPI resolved. pub fn set_window_location(event_ptr: *mut c_void, x: f64, y: f64) -> bool { match set_window_loc_fn() { - Some(f) => { unsafe { f(event_ptr, x, y) }; true } + Some(f) => { + unsafe { f(event_ptr, x, y) }; + true + } None => false, } } @@ -310,7 +356,10 @@ pub fn set_window_location(event_ptr: *mut c_void, x: f64, y: f64) -> bool { /// `SLEventSetIntegerValueField`. Returns `false` when SPI absent. pub fn set_integer_field(event_ptr: *mut c_void, field: u32, value: i64) -> bool { match set_int_field_fn() { - Some(f) => { unsafe { f(event_ptr, field, value) }; true } + Some(f) => { + unsafe { f(event_ptr, field, value) }; + true + } None => false, } } @@ -357,10 +406,14 @@ pub fn activate_without_raise(target_pid: pid_t, target_wid: u32) -> bool { let mut target_psn = [0u8; 8]; let ok_prev = unsafe { get_front(prev_psn.as_mut_ptr() as *mut c_void) } == 0; - if !ok_prev { return false; } + if !ok_prev { + return false; + } let ok_target = unsafe { get_pid_psn(target_pid, target_psn.as_mut_ptr() as *mut c_void) } == 0; - if !ok_target { return false; } + if !ok_target { + return false; + } // Build the 248-byte event buffer. let mut buf = [0u8; 0xF8]; @@ -374,15 +427,11 @@ pub fn activate_without_raise(target_pid: pid_t, target_wid: u32) -> bool { // Step 3: defocus previous front. buf[0x8A] = 0x02; - let defocus_ok = unsafe { - post_fn(prev_psn.as_ptr() as *const c_void, buf.as_ptr()) == 0 - }; + let defocus_ok = unsafe { post_fn(prev_psn.as_ptr() as *const c_void, buf.as_ptr()) == 0 }; // Step 4: focus target. buf[0x8A] = 0x01; - let focus_ok = unsafe { - post_fn(target_psn.as_ptr() as *const c_void, buf.as_ptr()) == 0 - }; + let focus_ok = unsafe { post_fn(target_psn.as_ptr() as *const c_void, buf.as_ptr()) == 0 }; defocus_ok && focus_ok } @@ -394,15 +443,19 @@ pub fn activate_without_raise(target_pid: pid_t, target_wid: u32) -> bool { /// Falls back to `GetProcessForPID(pid)` when the SkyLight path fails. pub fn get_process_psn_for_window(window_id: u32, pid: libc::pid_t, out_psn: &mut [u8; 8]) -> bool { // Try modern path: CGSMainConnectionID → SLSGetWindowOwner → SLSGetConnectionPSN - if let (Some(get_owner), Some(get_psn), Some(conn_id_fn)) = - (get_window_owner_fn(), get_connection_psn_fn(), connection_id_fn()) - { + if let (Some(get_owner), Some(get_psn), Some(conn_id_fn)) = ( + get_window_owner_fn(), + get_connection_psn_fn(), + connection_id_fn(), + ) { let main_cid = unsafe { conn_id_fn() }; let mut owner_cid: u32 = 0; let ok = unsafe { get_owner(main_cid, window_id, &mut owner_cid) } == 0; if ok && owner_cid != 0 { let psn_ok = unsafe { get_psn(owner_cid, out_psn.as_mut_ptr() as *mut c_void) } == 0; - if psn_ok { return true; } + if psn_ok { + return true; + } } } // Fallback: GetProcessForPID diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/slps_make_key.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/slps_make_key.rs new file mode 100644 index 0000000000..824298570a --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/input/slps_make_key.rs @@ -0,0 +1,90 @@ +//! Exact target-only `SLPSPostEventRecordTo` make-key/remove-key records. +//! +//! The recovered bytes prove only these two marker states. They do not prove +//! distinct CPS `new-front`, `key-focus-returned`, or other event subtypes, so +//! this module deliberately does not name or claim them. + +use super::skylight; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SlpsMakeKeyState { + MakeKey, + RemoveKey, +} + +impl SlpsMakeKeyState { + pub fn marker(self) -> u8 { + match self { + Self::MakeKey => skylight::SLPS_MAKE_KEY_MARKER, + Self::RemoveKey => skylight::SLPS_REMOVE_KEY_MARKER, + } + } + + pub fn evidence_name(self) -> &'static str { + match self { + Self::MakeKey => "slps_make_key", + Self::RemoveKey => "slps_remove_key", + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum SlpsMakeKeyError { + #[error("target-only SLPS make-key symbols are unavailable")] + SymbolsUnavailable, + #[error("target-only SLPS {state:?} record failed for pid {pid}, window {window_id}")] + PostFailed { + pid: i32, + window_id: u32, + state: SlpsMakeKeyState, + }, +} + +pub fn available() -> bool { + skylight::is_target_slps_make_key_available() +} + +pub fn post_target_only( + pid: i32, + window_id: u32, + states: &[SlpsMakeKeyState], +) -> Result<(), SlpsMakeKeyError> { + if !available() { + return Err(SlpsMakeKeyError::SymbolsUnavailable); + } + for state in states { + if !skylight::post_slps_make_key_record_to_pid(pid, window_id, state.marker()) { + return Err(SlpsMakeKeyError::PostFailed { + pid, + window_id, + state: *state, + }); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn record_evidence_claims_only_make_key_and_remove_key_marker_shapes() { + let window_id = 0x0102_0304; + let grant = + skylight::build_slps_make_key_record(window_id, SlpsMakeKeyState::MakeKey.marker()); + let revoke = + skylight::build_slps_make_key_record(window_id, SlpsMakeKeyState::RemoveKey.marker()); + assert_eq!(&grant[0x3C..=0x3F], &window_id.to_le_bytes()); + assert_eq!(grant[0x8A], skylight::SLPS_MAKE_KEY_MARKER); + assert_eq!(revoke[0x8A], skylight::SLPS_REMOVE_KEY_MARKER); + assert_eq!( + grant + .iter() + .zip(revoke.iter()) + .filter(|(left, right)| left != right) + .count(), + 1 + ); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs index f5015378ff..79dd4d9a90 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs @@ -40,6 +40,8 @@ pub mod video_sckit; pub mod pip; #[cfg(target_os = "macos")] pub mod session; +#[cfg(target_os = "macos")] +pub mod driver; use cua_driver_core::tool::ToolRegistry; diff --git a/libs/cua-driver/rust/crates/platform-macos/src/permissions/status.rs b/libs/cua-driver/rust/crates/platform-macos/src/permissions/status.rs index 2721d17ff4..df2b8770df 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/permissions/status.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/permissions/status.rs @@ -66,6 +66,17 @@ pub fn screen_recording_granted() -> bool { unsafe { CGPreflightScreenCaptureAccess() } } +/// Live ScreenCaptureKit capability probe. Unlike the CoreGraphics preflight +/// boolean, this asks the capture framework whether the responsible process +/// can enumerate shareable displays right now. +pub fn screen_recording_capturable() -> bool { + use screencapturekit::prelude::SCShareableContent; + + SCShareableContent::get() + .map(|content| !content.displays().is_empty()) + .unwrap_or(false) +} + /// Raise the Accessibility TCC prompt if not yet granted. No-op when /// already active. Mirrors Swift `Permissions.requestAccessibility()`. pub fn request_accessibility() -> bool { diff --git a/libs/cua-driver/rust/crates/platform-macos/src/video_sckit.rs b/libs/cua-driver/rust/crates/platform-macos/src/video_sckit.rs index ce3841ec99..4380862750 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/video_sckit.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/video_sckit.rs @@ -23,7 +23,7 @@ //! atom) and returns the elapsed-time metadata. use std::path::Path; -use std::time::Instant; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; use cua_driver_core::video::{VideoBackend, VideoBackendFactory, VideoMetadata}; @@ -34,9 +34,154 @@ use screencapturekit::recording_output::{ SCRecordingOutput, SCRecordingOutputCodec, SCRecordingOutputConfiguration, SCRecordingOutputFileType, }; +use screencapturekit::{ + cm::{CMSampleBufferExt, CMSampleBufferSCExt, SCFrameStatus}, + screenshot_manager::SCScreenshotManager, +}; pub struct SckitVideoBackendFactory; +/// Metadata read from the same `CMSampleBuffer` that produced the returned +/// PNG. Keeping pixels and freshness evidence in one value prevents callers +/// from accidentally pairing a new image with stale stream metadata. +#[derive(Debug, Clone, PartialEq)] +pub struct WindowFrameMetadata { + pub completion_unix_ms: u64, + pub display_time: Option, + pub frame_status: Option, + pub scale_factor: Option, + pub content_scale: Option, + pub content_rect: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct FrameRect { + pub x: f64, + pub y: f64, + pub width: f64, + pub height: f64, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct WindowFrameSample { + pub png_bytes: Vec, + pub pixel_width: u32, + pub pixel_height: u32, + /// ScreenCaptureKit's exact source-window frame at sample construction. + pub source_frame: FrameRect, + pub metadata: WindowFrameMetadata, +} + +struct TemporaryCapturePath(std::path::PathBuf); + +impl Drop for TemporaryCapturePath { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + } +} + +/// Capture exactly one desktop-independent window through ScreenCaptureKit. +/// +/// This is intentionally separate from the long-lived recording stream. A +/// v2 observation needs one bounded sample whose pixels, frame status, +/// display timestamp and scale attachments are coherent. The old +/// `screencapture` CLI helper cannot provide that contract and is never called +/// here. +pub fn capture_window_sample( + window_id: u32, + expected_scale_factor: f64, +) -> anyhow::Result { + if !expected_scale_factor.is_finite() || expected_scale_factor <= 0.0 { + anyhow::bail!("invalid expected scale factor for window {window_id}"); + } + + let content = SCShareableContent::get() + .map_err(|error| anyhow::anyhow!("SCShareableContent::get failed: {error}"))?; + let window = content + .windows() + .into_iter() + .find(|candidate| candidate.window_id() == window_id) + .ok_or_else(|| anyhow::anyhow!("ScreenCaptureKit window {window_id} is unavailable"))?; + let frame = window.frame(); + let width = scaled_dimension(frame.size.width, expected_scale_factor, "width")?; + let height = scaled_dimension(frame.size.height, expected_scale_factor, "height")?; + let filter = SCContentFilter::create().with_window(&window).build(); + let configuration = SCStreamConfiguration::new() + .with_width(width) + .with_height(height) + .with_ignores_shadows_single_window(true) + .with_shows_cursor(false) + .with_captures_audio(false); + + let sample = SCScreenshotManager::capture_sample_buffer(&filter, &configuration) + .map_err(|error| anyhow::anyhow!("ScreenCaptureKit sample failed: {error}"))?; + let frame_info = sample.frame_info(); + let image = sample + .cg_image() + .map_err(|status| anyhow::anyhow!("ScreenCaptureKit sample has no image: {status}"))?; + let pixel_width = u32::try_from(image.width()) + .map_err(|_| anyhow::anyhow!("captured image width exceeds u32"))?; + let pixel_height = u32::try_from(image.height()) + .map_err(|_| anyhow::anyhow!("captured image height exceeds u32"))?; + + let temporary_path = TemporaryCapturePath(std::env::temp_dir().join(format!( + "cua-v2-sck-sample-{}-{}.png", + std::process::id(), + uuid::Uuid::new_v4() + ))); + image.save_png(&temporary_path.0).map_err(|error| { + anyhow::anyhow!("failed to encode ScreenCaptureKit sample for window {window_id}: {error}") + })?; + let png_bytes = std::fs::read(&temporary_path.0).map_err(|error| { + anyhow::anyhow!("failed to read encoded ScreenCaptureKit sample: {error}") + })?; + if png_bytes.is_empty() { + anyhow::bail!("ScreenCaptureKit produced an empty PNG for window {window_id}"); + } + + let completion_unix_ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .min(u128::from(u64::MAX)) as u64; + let metadata = WindowFrameMetadata { + completion_unix_ms, + display_time: frame_info.as_ref().and_then(|info| info.display_time), + frame_status: frame_info.as_ref().and_then(|info| info.frame_status), + scale_factor: frame_info.as_ref().and_then(|info| info.scale_factor), + content_scale: frame_info.as_ref().and_then(|info| info.content_scale), + content_rect: frame_info + .as_ref() + .and_then(|info| info.content_rect) + .map(|rect| FrameRect { + x: rect.origin.x, + y: rect.origin.y, + width: rect.size.width, + height: rect.size.height, + }), + }; + Ok(WindowFrameSample { + png_bytes, + pixel_width, + pixel_height, + source_frame: FrameRect { + x: frame.origin.x, + y: frame.origin.y, + width: frame.size.width, + height: frame.size.height, + }, + metadata, + }) +} + +fn scaled_dimension(points: f64, scale: f64, name: &str) -> anyhow::Result { + let pixels = (points * scale).round(); + if !pixels.is_finite() || pixels < 1.0 || pixels > f64::from(u32::MAX) { + anyhow::bail!("invalid ScreenCaptureKit {name}: {points} points at {scale}x"); + } + Ok(pixels as u32) +} + impl VideoBackendFactory for SckitVideoBackendFactory { fn start(&self, output_path: &Path) -> anyhow::Result> { SckitVideoBackend::start(output_path).map(|b| Box::new(b) as Box) diff --git a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs index 0785c6d46e..84c92c0374 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/windows.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/windows.rs @@ -4,6 +4,8 @@ //! of CFDictionary objects describing each window. use serde::{Deserialize, Serialize}; +use std::ffi::{c_char, c_void}; +use std::sync::OnceLock; #[derive(Debug, Clone, Serialize, Deserialize)] pub struct WindowBounds { @@ -27,6 +29,13 @@ pub struct WindowInfo { pub space_ids: Option>, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SpaceFacts { + pub space_ids: Vec, + pub current_space_ids: Vec, + pub on_current_space: bool, +} + // ── CGWindow option flags ───────────────────────────────────────────────────── // Apple-canonical kCG* naming preserved to match the public Apple headers — the // upper-case-globals lint would rename them to KCG_..., which would silently @@ -58,15 +67,27 @@ extern "C" { /// Enumerate all windows (including off-screen). pub fn all_windows() -> Vec { - enumerate_windows(kCGWindowListExcludeDesktopElements) + enumerate_windows(kCGWindowListExcludeDesktopElements, false) +} + +/// Enumerate non-desktop WindowServer surfaces on every layer. +/// +/// This is intentionally not used for public window discovery. Observation +/// joins it against AX-owned menu/popover/sheet window ids so transient +/// surfaces are included only with structural ownership evidence. +pub fn all_windows_including_transients() -> Vec { + enumerate_windows(kCGWindowListExcludeDesktopElements, true) } /// Enumerate only on-screen windows. pub fn visible_windows() -> Vec { - enumerate_windows(kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements) + enumerate_windows( + kCGWindowListOptionOnScreenOnly | kCGWindowListExcludeDesktopElements, + false, + ) } -fn enumerate_windows(options: u32) -> Vec { +fn enumerate_windows(options: u32, include_nonzero_layers: bool) -> Vec { use core_foundation::{ array::CFArray, base::{CFGetTypeID, TCFType, CFTypeRef}, @@ -145,7 +166,7 @@ fn enumerate_windows(options: u32) -> Vec { let is_on_screen = get_bool("kCGWindowIsOnscreen"); // Only include layer-0 windows. - if layer != 0 { continue; } + if !include_nonzero_layers && layer != 0 { continue; } // Parse bounds dict. let bounds = { @@ -238,3 +259,171 @@ pub fn resolve_main_window_id(pid: i32) -> anyhow::Result { }); Ok(largest.unwrap().window_id) } + +/// Backing scale for the display containing the largest area of `bounds`. +/// Returns `None` when no active display intersects the window. +pub fn display_scale_for_bounds(bounds: &WindowBounds) -> Option { + use core_graphics::display::CGDisplay; + + let mut best: Option<(f64, f64)> = None; + for display_id in CGDisplay::active_displays().ok()? { + let display = CGDisplay::new(display_id); + let frame = display.bounds(); + let left = bounds.x.max(frame.origin.x); + let top = bounds.y.max(frame.origin.y); + let right = (bounds.x + bounds.width).min(frame.origin.x + frame.size.width); + let bottom = (bounds.y + bounds.height).min(frame.origin.y + frame.size.height); + let area = (right - left).max(0.0) * (bottom - top).max(0.0); + if area <= 0.0 || frame.size.width <= 0.0 { + continue; + } + let scale = display.pixels_wide() as f64 / frame.size.width; + if best.map_or(true, |(best_area, _)| area > best_area) { + best = Some((area, scale)); + } + } + best.map(|(_, scale)| scale) +} + +type CopySpacesForWindowsFn = unsafe extern "C" fn( + u32, + u32, + core_foundation::array::CFArrayRef, +) -> core_foundation::array::CFArrayRef; +type CopyManagedDisplaySpacesFn = unsafe extern "C" fn(u32) -> core_foundation::array::CFArrayRef; + +fn load_skylight_symbol(names: &[&[u8]]) -> Option<*mut c_void> { + static LOADED: OnceLock<()> = OnceLock::new(); + LOADED.get_or_init(|| unsafe { + let path = b"/System/Library/PrivateFrameworks/SkyLight.framework/SkyLight\0"; + libc::dlopen( + path.as_ptr() as *const c_char, + libc::RTLD_LAZY | libc::RTLD_GLOBAL, + ); + }); + names.iter().find_map(|name| { + let ptr = unsafe { libc::dlsym(libc::RTLD_DEFAULT, name.as_ptr() as *const c_char) }; + (!ptr.is_null()).then_some(ptr) + }) +} + +unsafe fn symbol_as(pointer: *mut c_void) -> T { + std::mem::transmute_copy::<*mut c_void, T>(&pointer) +} + +fn copy_spaces_for_windows_fn() -> Option { + static SYMBOL: OnceLock> = OnceLock::new(); + *SYMBOL.get_or_init(|| { + load_skylight_symbol(&[b"SLSCopySpacesForWindows\0", b"CGSCopySpacesForWindows\0"]) + .map(|pointer| unsafe { symbol_as(pointer) }) + }) +} + +fn copy_managed_display_spaces_fn() -> Option { + static SYMBOL: OnceLock> = OnceLock::new(); + *SYMBOL.get_or_init(|| { + load_skylight_symbol(&[ + b"SLSCopyManagedDisplaySpaces\0", + b"CGSCopyManagedDisplaySpaces\0", + ]) + .map(|pointer| unsafe { symbol_as(pointer) }) + }) +} + +pub fn space_query_available() -> bool { + crate::input::skylight::main_connection_id().is_some() + && copy_spaces_for_windows_fn().is_some() + && copy_managed_display_spaces_fn().is_some() +} + +/// Read a window's managed Space ids and the active Space on every display. +/// All symbols are private SkyLight reads and are resolved dynamically, so an +/// unavailable API is reported as `None` rather than guessed from visibility. +pub fn space_facts(window_id: u32) -> Option { + let connection = crate::input::skylight::main_connection_id()?; + let window_spaces = copy_window_space_ids(connection, window_id)?; + let current_spaces = copy_current_space_ids(connection)?; + let on_current_space = window_spaces + .iter() + .any(|space| current_spaces.contains(space)); + Some(SpaceFacts { + space_ids: window_spaces, + current_space_ids: current_spaces, + on_current_space, + }) +} + +fn copy_window_space_ids(connection: u32, window_id: u32) -> Option> { + use core_foundation::{array::CFArray, base::TCFType, number::CFNumber}; + + let window = CFNumber::from(window_id as i64); + let windows = CFArray::from_CFTypes(&[window]); + let raw = + unsafe { copy_spaces_for_windows_fn()?(connection, 0x7, windows.as_concrete_TypeRef()) }; + if raw.is_null() { + return None; + } + let spaces = unsafe { CFArray::::wrap_under_create_rule(raw) }; + Some( + spaces + .iter() + .filter_map(|space| space.to_i64()) + .filter_map(|space| u64::try_from(space).ok()) + .collect(), + ) +} + +fn copy_current_space_ids(connection: u32) -> Option> { + use core_foundation::{ + array::CFArray, + base::{CFGetTypeID, CFTypeRef, TCFType}, + dictionary::CFDictionary, + number::CFNumber, + string::CFString, + }; + + let raw = unsafe { copy_managed_display_spaces_fn()?(connection) }; + if raw.is_null() { + return None; + } + let displays = unsafe { CFArray::::wrap_under_create_rule(raw) }; + let mut result = Vec::new(); + for display in displays.iter() { + let display = *display; + if unsafe { CFGetTypeID(display) } + != CFDictionary::<*const c_void, *const c_void>::type_id() + { + continue; + } + let display = unsafe { + CFDictionary::<*const c_void, *const c_void>::wrap_under_get_rule(display as _) + }; + let current_key = CFString::new("Current Space"); + let Some(current) = display.find(current_key.as_concrete_TypeRef() as *const c_void) else { + continue; + }; + let current = *current; + if unsafe { CFGetTypeID(current) } + != CFDictionary::<*const c_void, *const c_void>::type_id() + { + continue; + } + let current = unsafe { + CFDictionary::<*const c_void, *const c_void>::wrap_under_get_rule(current as _) + }; + let id = ["ManagedSpaceID", "id64"].iter().find_map(|key| { + let key = CFString::new(key); + let value = *current.find(key.as_concrete_TypeRef() as *const c_void)?; + if unsafe { CFGetTypeID(value) } != CFNumber::type_id() { + return None; + } + unsafe { CFNumber::wrap_under_get_rule(value as _) } + .to_i64() + .and_then(|value| u64::try_from(value).ok()) + }); + if let Some(id) = id { + result.push(id); + } + } + Some(result) +}