diff --git a/libs/cua-driver/rust/crates/cua-driver-core/examples/v2_protocol_fixture.rs b/libs/cua-driver/rust/crates/cua-driver-core/examples/v2_protocol_fixture.rs new file mode 100644 index 0000000000..da62aa307e --- /dev/null +++ b/libs/cua-driver/rust/crates/cua-driver-core/examples/v2_protocol_fixture.rs @@ -0,0 +1,288 @@ +//! Bounded native protocol fixture for the Python v2 transport contract test. +//! +//! This binary does not dispatch platform work. It accepts the canonical v2 +//! request types and emits canonical API results through the canonical v2 +//! response envelope, then records that stdin reached EOF. + +use std::{ + collections::BTreeMap, + env, fs, + io::{self, BufRead, Write}, + path::PathBuf, +}; + +use cua_driver_core::{ + api::{ + AccessibilityElement, AccessibilityState, ActionId, ActionReceipt, AppId, AppRef, + AxRevision, AxTreeUpdate, CapturedSurface, ElementId, ElementRef, ErrorCode, ErrorPhase, + MenuRevision, MenuState, NativeError, NativeEvidence, ObservationId, PartialEvidence, + PartialNativeDispatch, PendingSettlementEvidence, PendingSettlementState, PostureResult, + Rect, ReplaceAxLines, Route, SettledState, SettlementEvidence, SettlementSignal, Size, + SurfaceId, SurfaceKind, VerificationLevel, WindowId, WindowRef, WindowState, + }, + protocol::{ + V2Command, V2Failure, V2HandshakeRequest, V2HandshakeResponse, V2RequestEnvelope, + V2ResponseBody, V2ResponseEnvelope, V2Success, V2_METHODS, V2_PROTOCOL_VERSION, + }, +}; +use serde::Serialize; + +const MAX_FIXTURE_LINE_BYTES: usize = 64 * 1024; +const MAX_FIXTURE_REQUESTS: usize = 4; + +fn main() -> anyhow::Result<()> { + let eof_marker = env::args_os() + .nth(1) + .map(PathBuf::from) + .ok_or_else(|| anyhow::anyhow!("expected EOF marker path"))?; + let stdin = io::stdin(); + let mut lines = stdin.lock().lines(); + let handshake_line = lines + .next() + .transpose()? + .ok_or_else(|| anyhow::anyhow!("expected handshake"))?; + anyhow::ensure!( + handshake_line.len() <= MAX_FIXTURE_LINE_BYTES, + "handshake exceeded fixture line bound" + ); + let handshake: V2HandshakeRequest = serde_json::from_str(&handshake_line)?; + write_line(&V2HandshakeResponse { + request_id: handshake.request_id, + minimum_version: V2_PROTOCOL_VERSION, + maximum_version: V2_PROTOCOL_VERSION, + driver_name: "cua-driver-v2-protocol-fixture".to_owned(), + driver_version: env!("CARGO_PKG_VERSION").to_owned(), + build: "test-only".to_owned(), + methods: V2_METHODS + .iter() + .map(|method| (*method).to_owned()) + .collect(), + })?; + + let mut observation_count = 0_u8; + let mut request_count = 0_usize; + for line in lines { + request_count += 1; + anyhow::ensure!( + request_count <= MAX_FIXTURE_REQUESTS, + "request count exceeded fixture bound" + ); + let line = line?; + anyhow::ensure!( + line.len() <= MAX_FIXTURE_LINE_BYTES, + "request exceeded fixture line bound" + ); + let request: V2RequestEnvelope = serde_json::from_str(&line)?; + request + .validate_version() + .map_err(|error| anyhow::anyhow!(error))?; + match request.command { + V2Command::GetWindowState(_) => { + observation_count += 1; + let state = fixture_state(observation_count)?; + write_result(request.request_id, state)?; + } + V2Command::Click(_) => { + write_result(request.request_id, fixture_receipt()?)?; + } + V2Command::TypeText(_) => { + write_error(request.request_id, fixture_partial_error()?)?; + } + command => { + let error = NativeError::invalid(format!( + "protocol fixture does not implement {}", + command.method() + )); + write_error(request.request_id, error)?; + } + } + } + + fs::write(eof_marker, b"eof\n")?; + Ok(()) +} + +fn fixture_state(sequence: u8) -> anyhow::Result { + let observation_id = parse_id(ObservationId::parse(format!("observation-{sequence}")))?; + let revision = parse_id(AxRevision::parse(format!("revision-{sequence}")))?; + let tree_update = match sequence { + 1 => AxTreeUpdate::Full { + revision, + tree: "root\nold".to_owned(), + }, + 2 => AxTreeUpdate::Diff { + base_revision: parse_id(AxRevision::parse("revision-1"))?, + revision, + operations: vec![ReplaceAxLines { + start_line: 1, + delete_count: 1, + lines: vec!["new".to_owned()], + }], + }, + _ => anyhow::bail!("fixture serves exactly two observations"), + }; + let window = fixture_window()?; + let surface_id = parse_id(SurfaceId::parse("surface-1"))?; + let element_ref = ElementRef { + observation_id: observation_id.clone(), + id: parse_id(ElementId::parse("element-1"))?, + }; + Ok(WindowState { + observation_id, + window: window.clone(), + surfaces: vec![CapturedSurface { + id: surface_id, + owner_window: window, + kind: SurfaceKind::Window, + image_url: "file:///opaque/native-protocol-fixture.png".to_owned(), + size: Size { + width: 640, + height: 480, + }, + window_bounds: Some(Rect { + x: 10.0, + y: 20.0, + width: 640.0, + height: 480.0, + }), + }], + accessibility: Some(AccessibilityState { + tree_update, + elements: vec![AccessibilityElement { + element_ref: element_ref.clone(), + role: Some("AXButton".to_owned()), + label: Some("Fixture".to_owned()), + value: None, + bounds: None, + actions: vec!["AXPress".to_owned()], + }], + focused_element: Some(element_ref), + selected_text: None, + selected_elements: Vec::new(), + document_text: None, + }), + menu: MenuState::Closed { + revision: parse_id(MenuRevision::parse(format!("menu-revision-{sequence}")))?, + }, + settlement: SettlementEvidence::initial(), + captured_at_unix_ms: 1_753_200_000_000 + u64::from(sequence), + warnings: Vec::new(), + }) +} + +fn fixture_receipt() -> anyhow::Result { + let action_id = parse_id(ActionId::parse("action-click-1"))?; + let mut fields = BTreeMap::new(); + fields.insert("fixture".to_owned(), serde_json::json!("rust-serde")); + Ok(ActionReceipt { + action_id: action_id.clone(), + window: fixture_window()?, + consumed_observation_id: parse_id(ObservationId::parse("observation-2"))?, + route: Route::TargetedPointer, + verification: VerificationLevel::DispatchVerified, + posture: PostureResult::default(), + settlement: settled_after(action_id), + native_evidence: NativeEvidence { + fields, + interaction_scope: None, + }, + warnings: Vec::new(), + }) +} + +fn fixture_partial_error() -> anyhow::Result { + let action_id = parse_id(ActionId::parse("action-type-1"))?; + let observation_id = parse_id(ObservationId::parse("observation-2"))?; + let pending = PendingSettlementEvidence { + state: PendingSettlementState::Pending, + trigger_action_id: action_id.clone(), + profile: "fixture-keyboard".to_owned(), + elapsed_ms: 3, + observed_signals: vec![SettlementSignal::DispatchStarted], + missing_signals: vec![SettlementSignal::DispatchComplete], + }; + let mut fields = BTreeMap::new(); + fields.insert("delivered_events".to_owned(), serde_json::json!(1)); + let evidence = NativeEvidence { + fields, + interaction_scope: None, + }; + let mut error = NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "fixture keyboard sequence stopped after its first targeted event", + ) + .with_detail("delivered_events", 1); + error.partial_evidence = Some(Box::new(PartialEvidence::Action { + action_id, + window: fixture_window()?, + consumed_observation_id: observation_id, + route: Route::TargetedKeyboard, + dispatch: Some(PartialNativeDispatch { + verification: VerificationLevel::DispatchUnverified, + native_evidence: evidence.clone(), + warnings: vec!["fixture forced a typed partial dispatch".to_owned()], + }), + posture: PostureResult::default(), + native_evidence: evidence, + pending_settlement: Some(Box::new(pending.clone())), + })); + error.pending_settlement = Some(Box::new(pending)); + Ok(error) +} + +fn fixture_window() -> anyhow::Result { + Ok(WindowRef { + id: parse_id(WindowId::parse("window-1"))?, + app: AppRef { + id: parse_id(AppId::parse("app-1"))?, + name: Some("Fixture".to_owned()), + pid: Some(42), + running: true, + }, + title: Some("Fixture window".to_owned()), + }) +} + +fn parse_id(result: Result) -> anyhow::Result { + result.map_err(anyhow::Error::msg) +} + +fn settled_after(action_id: ActionId) -> SettlementEvidence { + SettlementEvidence { + state: SettledState::Settled, + trigger_action_id: Some(action_id), + profile: "fixture-pointer".to_owned(), + elapsed_ms: 2, + observed_signals: vec![SettlementSignal::DispatchComplete], + terminal_signal: "dispatch_complete".to_owned(), + quiet_window_ms: 1, + resumed_from_prior_call: false, + } +} + +fn write_result(request_id: String, result: T) -> anyhow::Result<()> { + write_line(&V2ResponseEnvelope { + request_id, + protocol_version: V2_PROTOCOL_VERSION, + body: V2ResponseBody::Result(V2Success { result }), + }) +} + +fn write_error(request_id: String, error: NativeError) -> anyhow::Result<()> { + write_line(&V2ResponseEnvelope:: { + request_id, + protocol_version: V2_PROTOCOL_VERSION, + body: V2ResponseBody::Error(V2Failure { error }), + }) +} + +fn write_line(value: &impl Serialize) -> anyhow::Result<()> { + let stdout = io::stdout(); + let mut stdout = stdout.lock(); + serde_json::to_writer(&mut stdout, value)?; + stdout.write_all(b"\n")?; + stdout.flush()?; + Ok(()) +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/controller.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/controller.rs index 8576b5e162..791eb7b816 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/api/controller.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/controller.rs @@ -21,10 +21,9 @@ use super::{ TypeTextCommand, VerificationLevel, WindowRef, WindowState, }, errors::{ErrorCode, ErrorPhase, NativeError, PartialEvidence, PartialNativeDispatch}, - interaction::{MutationDeadline, ScopePlan, ScopeRequirements}, + interaction::{MutationDeadline, NativeSideEffectBoundary, ScopePlan, ScopeRequirements}, observation::{ - revision_accessibility, InvalidationReason, ObservationRecord, ResolvedDrag, - ResolvedScroll, ResolvedWindow, + revision_accessibility, ObservationRecord, ResolvedDrag, ResolvedScroll, ResolvedWindow, }, platform::{ ClickSpec, ElementScrollSpec, InteractionProvider, KeyboardActionProvider, @@ -46,6 +45,12 @@ const DEFAULT_LOCK_TIMEOUT: Duration = Duration::from_secs(2); const DEFAULT_MUTATION_WORK_TIMEOUT: Duration = Duration::from_secs(10); const DEFAULT_MUTATION_TEARDOWN_TIMEOUT: Duration = Duration::from_secs(2); +enum PreparedDispatch { + Semantic(>::PreparedAction), + Pointer(>::PreparedAction), + Keyboard(>::PreparedAction), +} + pub struct DriverController { platform: Arc

, platform_name: PlatformName, @@ -151,13 +156,9 @@ impl DriverController

{ } }; if !launch.posture.held { - let mut error = NativeError::new( - ErrorCode::PostureViolated, - ErrorPhase::Verify, - false, - "background launch changed the user's foreground posture", - ) - .with_detail("action_id", action_id.to_string()); + let mut error = NativeError::from_posture(&launch.posture) + .expect("a non-held launch posture must be unverifiable or violated") + .with_detail("action_id", action_id.to_string()); error.partial_evidence = Some(Box::new(PartialEvidence::Launch { action_id, app: Some(launch.app), @@ -211,6 +212,11 @@ impl DriverController

{ .targets .get_or_create(&self.platform, key, resolved.clone()) .await?; + tracing::debug!( + target_instance_id = %target.instance_id, + operation = "get_window_state", + "v2 target controller acquired" + ); let _process_guard = tokio::time::timeout( self.lock_timeout, Arc::clone(&target.mutation_lock).lock_owned(), @@ -247,6 +253,7 @@ impl DriverController

{ .as_ref() .map(|accessibility| accessibility.normalized_tree.len()) .unwrap_or(0); + let ax_base_revision = state.ax_revisions.last_revision().map(ToString::to_string); let revisioned = native .accessibility .map(|accessibility| { @@ -258,6 +265,9 @@ impl DriverController

{ ) }) .transpose()?; + let ax_result_revision = revisioned + .as_ref() + .map(|revisioned| revisioned.public.tree_update.revision().to_string()); let mut surfaces = HashMap::new(); for surface in native.surfaces { @@ -313,6 +323,13 @@ impl DriverController

{ if let Some(revisioned) = revisioned { state.ax_revisions.commit(revisioned.prepared_revision); } + tracing::info!( + target_instance_id = %target.instance_id, + ax_base_revision = ax_base_revision.as_deref().unwrap_or("none"), + ax_result_revision = ax_result_revision.as_deref().unwrap_or("none"), + settlement_profile = %public.settlement.profile, + "v2 window observation committed" + ); target.touch(); Ok(public) } @@ -397,13 +414,13 @@ impl DriverController

{ &command.request.surface_id, command.request.end, )?; - Ok(ResolvedAction::Drag(ResolvedDrag { + Ok(ResolvedAction::Drag(Box::new(ResolvedDrag { start, end, duration_ms: command.request.duration_ms, button: command.request.button, modifiers: command.request.modifiers, - })) + }))) }, ) .await @@ -664,24 +681,116 @@ impl DriverController

{ let prepared = prepare(&mut state, &resolved)?; validate_current_menu_target(&state, &prepared)?; let targeted_menu_id = resolved_action_menu_id(&prepared).cloned(); - let capability_key = CapabilityKey { - platform: self.platform_name.clone(), - os_version: self.os_version.clone(), - action: action.clone(), - addressing, - framework: resolved.framework.clone(), - window_state: resolved.state.clone(), - }; - let route = match self.capabilities.read().await.decision(&capability_key) { - RouteDecision::Supported { route } => route, - RouteDecision::Unsupported { reason } => { - return Err(NativeError::unsupported(reason) - .with_detail("action", format!("{action:?}")) - .with_detail("framework", format!("{:?}", resolved.framework))) + let (route, prepared) = if let ResolvedAction::ElementClick { + source, + element, + spec, + } = prepared + { + let semantic_usable = self + .platform + .semantic() + .element_click_candidate(&mut state.platform, &element, &spec) + .await?; + let semantic_key = capability_key( + &self.platform_name, + &self.os_version, + &action, + AddressingMode::Element, + &resolved, + ); + let semantic_decision = self.capabilities.read().await.decision(&semantic_key); + match (semantic_usable, semantic_decision) { + (true, RouteDecision::Supported { route: Route::Semantic }) => ( + Route::Semantic, + ResolvedAction::ElementClick { + source, + element, + spec, + }, + ), + (true, RouteDecision::Supported { route }) => { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "element capability published a non-semantic route for an exact semantic click candidate", + ) + .with_detail("published_route", format!("{route:?}"))) + } + (semantic_usable, semantic_decision) => { + let point = state.observations.resolve_element_point(&resolved, &source)?; + let point_key = capability_key( + &self.platform_name, + &self.os_version, + &action, + AddressingMode::CapturedPoint, + &resolved, + ); + match self.capabilities.read().await.decision(&point_key) { + RouteDecision::Supported { + route: Route::TargetedPointer, + } => ( + Route::TargetedPointer, + ResolvedAction::PointClick { point, spec }, + ), + RouteDecision::Supported { route } => { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "captured-point capability published a non-pointer route", + ) + .with_detail("published_route", format!("{route:?}"))) + } + RouteDecision::Unsupported { reason } => { + let semantic_reason = match semantic_decision { + RouteDecision::Unsupported { reason } => reason, + RouteDecision::Supported { .. } if !semantic_usable => { + "element did not retain an exact usable semantic AXPress candidate" + .to_owned() + } + RouteDecision::Supported { .. } => unreachable!(), + }; + return Err(NativeError::unsupported( + "no exact background element-click route is currently usable", + ) + .with_detail("semantic_reason", semantic_reason) + .with_detail("captured_point_reason", reason) + .with_detail("framework", format!("{:?}", resolved.framework))); + } + } + } } + } else { + let capability_key = capability_key( + &self.platform_name, + &self.os_version, + &action, + addressing, + &resolved, + ); + let route = match self.capabilities.read().await.decision(&capability_key) { + RouteDecision::Supported { route } => route, + RouteDecision::Unsupported { reason } => { + return Err(NativeError::unsupported(reason) + .with_detail("action", format!("{action:?}")) + .with_detail("framework", format!("{:?}", resolved.framework))) + } + }; + ( + route, + adapt_action_route(&mut state, &resolved, route, prepared)?, + ) }; - let prepared = adapt_action_route(&mut state, &resolved, route, prepared)?; let action_id = ActionId::new(); + tracing::info!( + target_instance_id = %target.instance_id, + action_id = %action_id, + action_kind = ?action, + route = ?route, + "v2 mutation target and route acquired" + ); let deadline = mutation_deadline(self.mutation_work_timeout, self.mutation_teardown_timeout)?; let mut requirements = ScopeRequirements::for_route(route); @@ -717,6 +826,12 @@ impl DriverController

{ }; ensure_scope_plan_matches(&scope_plan, &action_id, &resolved, route, deadline)?; let pending_menu_id = menu_opening.then(|| { + tracing::debug!( + target_instance_id = %target.instance_id, + action_id = %action_id, + menu_transition = "closed_to_opening", + "v2 menu lifecycle transition" + ); state .menu .begin_open(action_id.clone(), resolved.public.clone(), resolved.stamp()) @@ -739,10 +854,30 @@ impl DriverController

{ }; let mut scope = match scope_result { Ok(Ok(scope)) => scope, - Ok(Err(error)) => { + Ok(Err(mut error)) => { if menu_opening { state.menu.close(); } + if error.target_invalidated() { + target.invalidate(); + drop(state); + match tokio::time::timeout_at( + deadline.teardown.into(), + self.targets.remove_invalid_target(&self.platform, &key), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(removal_error)) => error = error.with_related(&removal_error), + Err(_) => { + error = error.with_related(&mutation_deadline_error( + &action_id, + ErrorPhase::Verify, + "poisoned_target_teardown", + )); + } + } + } return Err(error); } Err(_) => { @@ -778,94 +913,88 @@ impl DriverController

{ } }; scope.bind_target_validity(target.validity_handle()); - let semantic_plan = if route == Route::Semantic { - let prepare_result = { - let platform = &mut state.platform; - tokio::time::timeout_at( - deadline.work.into(), - self.platform - .semantic() - .prepare(platform, &mut scope, &prepared), - ) - .await - }; - match prepare_result { - Ok(Ok(plan)) => Some(plan), - Ok(Err(error)) => { - let teardown = scope.release(); - let teardown_failed = !teardown.failures.is_empty(); - if teardown_failed { - target.invalidate(); - } - let mut failures = vec![error]; - failures.extend(teardown.failures); - if let Some(error) = NativeError::from_posture(&scope.posture) { - failures.push(error); - } - let scope_evidence = serde_json::to_value(&scope.native_evidence) - .expect("typed scope evidence must serialize"); - if teardown_failed { - drop(state); - match tokio::time::timeout_at( - deadline.teardown.into(), - self.targets.remove_invalid_target(&self.platform, &key), - ) - .await - { - Ok(Ok(_)) => {} - Ok(Err(error)) => failures.push(error), - Err(_) => failures.push(mutation_deadline_error( - &action_id, - ErrorPhase::Verify, - "poisoned_target_teardown", - )), - } - } - let mut primary = NativeError::primary(failures) - .expect("semantic prepare failure is nonempty"); - primary - .details - .insert("interaction_scope".to_owned(), scope_evidence); - return Err(primary); + let dispatch_plan_result = { + let platform = &mut state.platform; + tokio::time::timeout_at( + deadline.work.into(), + self.prepare_dispatch(platform, &mut scope, &prepared), + ) + .await + }; + let dispatch_plan = match dispatch_plan_result { + Ok(Ok(plan)) => plan, + Ok(Err(error)) => { + let teardown = scope.release(); + let teardown_failed = !teardown.failures.is_empty(); + if teardown_failed { + target.invalidate(); } - Err(_) => { - let mut failures = vec![mutation_deadline_error( - &action_id, - ErrorPhase::Preflight, - "semantic_prepare", - )]; - let teardown = scope.release(); - let teardown_failed = !teardown.failures.is_empty(); - if teardown_failed { - target.invalidate(); - } - failures.extend(teardown.failures); - if let Some(error) = NativeError::from_posture(&scope.posture) { - failures.push(error); + let mut failures = vec![error]; + failures.extend(teardown.failures); + if let Some(error) = NativeError::from_posture(&scope.posture) { + failures.push(error); + } + let scope_evidence = serde_json::to_value(&scope.native_evidence) + .expect("typed scope evidence must serialize"); + if teardown_failed { + drop(state); + match tokio::time::timeout_at( + deadline.teardown.into(), + self.targets.remove_invalid_target(&self.platform, &key), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => failures.push(error), + Err(_) => failures.push(mutation_deadline_error( + &action_id, + ErrorPhase::Verify, + "poisoned_target_teardown", + )), } - if teardown_failed { - drop(state); - match tokio::time::timeout_at( - deadline.teardown.into(), - self.targets.remove_invalid_target(&self.platform, &key), - ) - .await - { - Ok(Ok(_)) => {} - Ok(Err(error)) => failures.push(error), - Err(_) => failures.push(mutation_deadline_error( - &action_id, - ErrorPhase::Verify, - "poisoned_target_teardown", - )), - } + } + let mut primary = NativeError::primary(failures) + .expect("native action prepare failure is nonempty"); + primary + .details + .insert("interaction_scope".to_owned(), scope_evidence); + return Err(primary); + } + Err(_) => { + let mut failures = vec![mutation_deadline_error( + &action_id, + ErrorPhase::Preflight, + "native_action_prepare", + )]; + let teardown = scope.release(); + let teardown_failed = !teardown.failures.is_empty(); + if teardown_failed { + target.invalidate(); + } + failures.extend(teardown.failures); + if let Some(error) = NativeError::from_posture(&scope.posture) { + failures.push(error); + } + if teardown_failed { + drop(state); + match tokio::time::timeout_at( + deadline.teardown.into(), + self.targets.remove_invalid_target(&self.platform, &key), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => failures.push(error), + Err(_) => failures.push(mutation_deadline_error( + &action_id, + ErrorPhase::Verify, + "poisoned_target_teardown", + )), } - return Err(NativeError::primary(failures) - .expect("semantic prepare timeout failure is nonempty")); } + return Err(NativeError::primary(failures) + .expect("native action prepare timeout failure is nonempty")); } - } else { - None }; if !menu_opening { if let Some(menu_id) = &targeted_menu_id { @@ -905,28 +1034,101 @@ impl DriverController

{ .insert("interaction_scope".to_owned(), scope_evidence); return Err(primary); } + tracing::debug!( + target_instance_id = %target.instance_id, + action_id = %action_id, + menu_transition = "open_to_targeting", + "v2 menu lifecycle transition" + ); } } - // Core owns the dispatch boundary. Once provider code is entered the - // observation is consumed and target dirty even if the task is - // cancelled while awaiting native dispatch. - state - .observations - .consume(&observation_id, action_id.clone())?; - state - .observations - .invalidate_all(InvalidationReason::MutationDispatched); - let profile = settlement_profile(&action, menu_opening, requires_effect_verification); - state.settlement.mark_dirty(action_id.clone(), profile)?; - let dispatch_result = { - let platform = &mut state.platform; - tokio::time::timeout_at( + let profile = + settlement_profile(&action, route, menu_opening, requires_effect_verification); + // Refresh the store's current proof immediately before handing the + // provider its explicit first-native-side-effect boundary. The target + // lock prevents any concurrent core mutation of this record. + state.observations.current(&observation_id, &resolved)?; + let (dispatch_result, side_effect_started) = { + let TargetControllerState { + platform, + observations, + settlement, + .. + } = &mut *state; + let mut boundary = NativeSideEffectBoundary::new( + observations, + settlement, + observation_id.clone(), + action_id.clone(), + profile, + ); + let result = tokio::time::timeout_at( deadline.work.into(), - self.dispatch(platform, &mut scope, prepared, semantic_plan), + self.dispatch(platform, &mut scope, &mut boundary, prepared, dispatch_plan), ) - .await + .await; + (result, boundary.started()) }; + if !side_effect_started { + if menu_opening || targeted_menu_id.is_some() { + state.menu.close(); + } + let provider_failure = match dispatch_result { + Ok(Err(error)) => error, + Err(_) => mutation_deadline_error( + &action_id, + ErrorPhase::Preflight, + "final_native_validation", + ) + .with_detail("native_side_effect_started", false), + Ok(Ok(_)) => NativeError::new( + ErrorCode::Internal, + ErrorPhase::Dispatch, + false, + "native provider returned success without entering the first-side-effect boundary", + ), + }; + let provider_invalidated = provider_failure.target_invalidated(); + let teardown = scope.release(); + let teardown_failed = !teardown.failures.is_empty(); + if provider_invalidated || teardown_failed { + target.invalidate(); + } + let mut failures = vec![provider_failure]; + failures.extend(teardown.failures); + if let Some(error) = NativeError::from_posture(&scope.posture) { + failures.push(error); + } + let scope_evidence = serde_json::to_value(&scope.native_evidence) + .expect("typed scope evidence must serialize"); + if provider_invalidated || teardown_failed { + drop(state); + match tokio::time::timeout_at( + deadline.teardown.into(), + self.targets.remove_invalid_target(&self.platform, &key), + ) + .await + { + Ok(Ok(_)) => {} + Ok(Err(error)) => failures.push(error), + Err(_) => failures.push(mutation_deadline_error( + &action_id, + ErrorPhase::Verify, + "poisoned_target_teardown", + )), + } + } + let mut primary = + NativeError::primary(failures).expect("pre-side-effect failure is nonempty"); + primary + .details + .insert("interaction_scope".to_owned(), scope_evidence); + primary + .details + .insert("native_side_effect_started".to_owned(), false.into()); + return Err(primary); + } let mut failures = Vec::new(); let mut dispatch_timed_out = false; let dispatch = match dispatch_result { @@ -939,6 +1141,9 @@ impl DriverController

{ match result { Ok(dispatch) => Some(dispatch), Err(error) => { + if error.target_invalidated() { + target.invalidate(); + } if menu_opening || targeted_menu_id.is_some() { state.menu.close(); } @@ -1002,6 +1207,7 @@ impl DriverController

{ let teardown = scope.release(); let teardown_failed = !teardown.failures.is_empty(); + let target_invalidated = target.ensure_valid().is_err(); if teardown_failed { target.invalidate(); } @@ -1042,7 +1248,7 @@ impl DriverController

{ native_evidence: scope.native_evidence.clone(), pending_settlement, }; - if teardown_failed { + if teardown_failed || target_invalidated { drop(state); match tokio::time::timeout_at( deadline.teardown.into(), @@ -1060,6 +1266,17 @@ impl DriverController

{ } } if let Some(mut error) = NativeError::primary(failures) { + tracing::warn!( + target_instance_id = %target.instance_id, + action_id = %action_id, + action_kind = ?action, + route = ?route, + verification = ?dispatch.as_ref().map(|dispatch| dispatch.verification), + posture_held = scope.posture.held, + settlement_outcome = settlement.as_ref().map_or("pending_or_failed", |_| "settled"), + error_code = ?error.code, + "v2 mutation completed with typed failure" + ); error.partial_evidence = Some(Box::new(partial)); return Err(error); } @@ -1079,6 +1296,19 @@ impl DriverController

{ _ => None, }; target.touch(); + if let Some(receipt) = &receipt { + tracing::info!( + target_instance_id = %target.instance_id, + action_id = %receipt.action_id, + action_kind = ?action, + route = ?receipt.route, + verification = ?receipt.verification, + posture_held = receipt.posture.held, + settlement_profile = %receipt.settlement.profile, + settlement_outcome = "settled", + "v2 mutation completed" + ); + } receipt.ok_or_else(|| { NativeError::new( ErrorCode::Internal, @@ -1155,63 +1385,85 @@ impl DriverController

{ } } - async fn dispatch( + async fn prepare_dispatch( &self, target: &mut P::TargetState, scope: &mut super::interaction::InteractionScope, - mutation: ResolvedAction, - semantic_plan: Option< - >::PreparedAction, - >, - ) -> Result { - if scope.route == Route::Semantic { - let plan = semantic_plan.ok_or_else(|| { - NativeError::new( - ErrorCode::Internal, - ErrorPhase::Dispatch, - false, - "semantic dispatch entered without its retained prepare plan", - ) - })?; - return self.platform.semantic().dispatch(target, scope, plan).await; - } - if semantic_plan.is_some() { - return Err(NativeError::new( + mutation: &ResolvedAction, + ) -> Result, NativeError> { + match mutation { + ResolvedAction::PressKey { .. } | ResolvedAction::TypeText { .. } => self + .platform + .keyboard() + .prepare(target, scope, mutation) + .await + .map(PreparedDispatch::Keyboard), + _ if scope.route == Route::Semantic => self + .platform + .semantic() + .prepare(target, scope, mutation) + .await + .map(PreparedDispatch::Semantic), + ResolvedAction::PointClick { .. } + | ResolvedAction::Drag(_) + | ResolvedAction::DeltaScroll(_) + if scope.route == Route::TargetedPointer => + { + self.platform + .pointer() + .prepare(target, scope, mutation) + .await + .map(PreparedDispatch::Pointer) + } + _ => Err(NativeError::new( ErrorCode::Internal, - ErrorPhase::Dispatch, + ErrorPhase::Preflight, false, - "non-semantic dispatch received a semantic prepare plan", - )); + "resolved action and selected route have no native prepare provider", + )), } - match mutation { - ResolvedAction::PointClick { point, spec } => { - self.platform.pointer().click(scope, point, spec).await - } - ResolvedAction::Drag(drag) => self.platform.pointer().drag(scope, drag).await, - ResolvedAction::DeltaScroll(scroll) => { - self.platform.pointer().scroll(scope, scroll).await - } - ResolvedAction::PressKey { focus, stroke } => { + } + + async fn dispatch( + &self, + target: &mut P::TargetState, + scope: &mut super::interaction::InteractionScope, + boundary: &mut NativeSideEffectBoundary<'_>, + mutation: ResolvedAction, + plan: PreparedDispatch

, + ) -> Result { + match (mutation, plan) { + ( + ResolvedAction::PressKey { .. } | ResolvedAction::TypeText { .. }, + PreparedDispatch::Keyboard(plan), + ) if matches!(scope.route, Route::Semantic | Route::TargetedKeyboard) => { self.platform .keyboard() - .press_key(scope, &focus, stroke) + .dispatch(target, scope, boundary, plan) .await } - ResolvedAction::TypeText { focus, text } => { + ( + ResolvedAction::PointClick { .. } + | ResolvedAction::Drag(_) + | ResolvedAction::DeltaScroll(_), + PreparedDispatch::Pointer(plan), + ) if scope.route == Route::TargetedPointer => { self.platform - .keyboard() - .type_text(scope, &focus, &text) + .pointer() + .dispatch(target, scope, boundary, plan) .await } - ResolvedAction::ElementClick { .. } - | ResolvedAction::ElementScroll { .. } - | ResolvedAction::SetValue { .. } - | ResolvedAction::SelectText { .. } - | ResolvedAction::Secondary { .. } => Err(NativeError::new( + (_, PreparedDispatch::Semantic(plan)) if scope.route == Route::Semantic => { + self.platform + .semantic() + .dispatch(target, scope, boundary, plan) + .await + } + _ => Err(NativeError::new( ErrorCode::Internal, ErrorPhase::Dispatch, false, - "element semantic action reached a non-semantic dispatch route", + "prepared native action did not match the resolved action provider and route", )), } } @@ -1298,6 +1550,23 @@ fn resolved_action_menu_id(action: &ResolvedAction) -> Option<&super::contracts: } } +fn capability_key( + platform: &PlatformName, + os_version: &str, + action: &ActionKind, + addressing: AddressingMode, + window: &ResolvedWindow, +) -> CapabilityKey { + CapabilityKey { + platform: platform.clone(), + os_version: os_version.to_owned(), + action: action.clone(), + addressing, + framework: window.framework.clone(), + window_state: window.state.clone(), + } +} + fn ensure_public_window_matches( requested: &WindowRef, resolved: &ResolvedWindow, @@ -1397,6 +1666,7 @@ fn mutation_deadline_error( fn settlement_profile( action: &ActionKind, + route: Route, menu_opening: bool, exact_readback: bool, ) -> SettlementProfile { @@ -1425,6 +1695,7 @@ fn settlement_profile( } let relevant = match action { ActionKind::Click => vec![ + SettlementSignal::PointerSequenceComplete, SettlementSignal::AxAction, SettlementSignal::AxValueChanged, SettlementSignal::FocusChanged, @@ -1433,11 +1704,13 @@ fn settlement_profile( SettlementSignal::MenuDismissed, ], ActionKind::Drag => vec![ + SettlementSignal::PointerSequenceComplete, SettlementSignal::AxAction, SettlementSignal::AxValueChanged, SettlementSignal::WindowGeometryChanged, ], ActionKind::Scroll => vec![ + SettlementSignal::PointerSequenceComplete, SettlementSignal::ScrollChanged, SettlementSignal::AxValueChanged, ], @@ -1456,8 +1729,13 @@ fn settlement_profile( SettlementSignal::WindowListChanged, ], }; - SettlementProfile::dispatch_only(format!("{action:?}").to_lowercase()) - .with_relevant_signals(relevant) + let name = format!("{action:?}").to_lowercase(); + if route == Route::TargetedPointer { + SettlementProfile::requiring(name, [SettlementSignal::PointerSequenceComplete]) + .with_relevant_signals(relevant) + } else { + SettlementProfile::dispatch_only(name).with_relevant_signals(relevant) + } } fn target_busy(window: &ResolvedWindow) -> NativeError { diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/errors.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/errors.rs index 6b0f7ab40d..a04d9d0bad 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/api/errors.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/errors.rs @@ -65,6 +65,12 @@ pub struct NativeError { pub pending_settlement: Option>, #[serde(default)] pub related_failures: Vec, + /// Internal platform-to-core containment signal. This is deliberately not + /// serialized: callers need the durable failure evidence above, while the + /// in-process controller needs a typed instruction never to reuse native + /// state whose cleanup could not be proved. + #[serde(skip)] + target_invalidated: bool, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] @@ -133,6 +139,7 @@ impl NativeError { partial_evidence: None, pending_settlement: None, related_failures: Vec::new(), + target_invalidated: false, } } @@ -196,6 +203,15 @@ impl NativeError { self } + pub fn with_target_invalidated(mut self) -> Self { + self.target_invalidated = true; + self + } + + pub fn target_invalidated(&self) -> bool { + self.target_invalidated + } + pub fn precedence(&self) -> u8 { match self.code { ErrorCode::PostureViolated => 5, @@ -217,6 +233,7 @@ impl NativeError { .max_by_key(|(_, error)| error.precedence()) .map(|(index, _)| index)?; let mut primary = failures.remove(primary_index); + primary.target_invalidated |= failures.iter().any(Self::target_invalidated); primary .related_failures .extend(failures.iter().map(|failure| NativeErrorSummary { diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/interaction.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/interaction.rs index a773ec4b3a..2406bdb5e1 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/api/interaction.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/interaction.rs @@ -10,9 +10,10 @@ use serde::{Deserialize, Serialize}; use serde_json::Value; use super::{ - contracts::{ActionId, MenuId, Point, Route}, + contracts::{ActionId, MenuId, ObservationId, Point, Route}, errors::NativeError, - observation::{ResolvedWindow, ResolvedWindowStamp}, + observation::{InvalidationReason, ObservationStore, ResolvedWindow, ResolvedWindowStamp}, + settlement::{SettlementProfile, SettlementState}, target::TargetValidityHandle, }; @@ -119,6 +120,65 @@ pub struct MutationDeadline { pub teardown: Instant, } +/// The single controller-owned transition into possibly mutating native UI. +/// +/// Platform providers keep every fallible no-side-effect validation and native +/// event construction before `begin`, then call it immediately before the +/// first AX write/action or targeted event post. The transition is idempotent +/// so cleanup posts can safely pass through the same boundary. While the +/// target lock is held, no other core path can change these stores between +/// action resolution and this transition. +pub struct NativeSideEffectBoundary<'a> { + observations: &'a mut ObservationStore, + settlement: &'a mut SettlementState, + observation_id: ObservationId, + action_id: ActionId, + profile: SettlementProfile, + started: bool, +} + +impl<'a> NativeSideEffectBoundary<'a> { + pub fn new( + observations: &'a mut ObservationStore, + settlement: &'a mut SettlementState, + observation_id: ObservationId, + action_id: ActionId, + profile: SettlementProfile, + ) -> Self { + debug_assert!(settlement.settled_evidence().is_some()); + Self { + observations, + settlement, + observation_id, + action_id, + profile, + started: false, + } + } + + /// Consume perception and mark the target dirty exactly once. + /// + /// A provider must call this only after its last no-side-effect check and + /// immediately before its first native mutation primitive. + pub fn begin(&mut self) -> Result<(), NativeError> { + if self.started { + return Ok(()); + } + self.observations + .consume(&self.observation_id, self.action_id.clone())?; + self.observations + .invalidate_all(InvalidationReason::MutationDispatched); + self.settlement + .mark_dirty(self.action_id.clone(), self.profile.clone())?; + self.started = true; + Ok(()) + } + + pub fn started(&self) -> bool { + self.started + } +} + impl MutationDeadline { pub fn new(work: Instant, teardown: Instant) -> Result { if teardown < work { @@ -312,6 +372,15 @@ impl InteractionScope { self.target_validity = Some(target_validity); } + /// Fail closed when a native provider cannot prove that partially posted + /// state was released. Core removes the poisoned controller after scope + /// teardown so the next observation constructs fresh native state. + pub fn invalidate_target(&self) { + if let Some(target_validity) = &self.target_validity { + target_validity.invalidate(); + } + } + pub fn release(&mut self) -> ScopeTeardownOutcome { if let Some(outcome) = &self.teardown { return outcome.clone(); diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/observation.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/observation.rs index 70add43a05..dd806ec2e5 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/api/observation.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/observation.rs @@ -125,6 +125,12 @@ pub enum CaptureFreshness { Unavailable, } +/// Opaque monotonic epoch produced by the native observation journal. Core +/// carries it with captured pixels so the platform can reject a point if a +/// native content/focus/AX signal races before the first post. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct NativeObservationEpoch(pub u64); + impl CaptureFreshness { pub fn action_safe(self) -> bool { matches!(self, Self::Fresh | Self::ReusedWithFreshCompletion) @@ -143,6 +149,7 @@ pub struct SurfaceRecord { pub raster_size: Size, pub window_bounds: Option, pub capture_revision: CaptureRevision, + pub observation_epoch: Option, pub transform: SurfaceToWindowTransform, pub freshness: CaptureFreshness, /// Exact native ownership for the pixels. Related transient surfaces are @@ -357,6 +364,7 @@ pub struct ResolvedPoint { pub surface_id: SurfaceId, pub surface_owner: SurfaceOwner, pub capture_revision: CaptureRevision, + pub observation_epoch: Option, pub surface_point: Point, pub window_point: Point, pub screen_point: Point, @@ -588,6 +596,7 @@ impl ObservationStore { surface_id: surface_id.clone(), surface_owner: surface.owner.clone(), capture_revision: surface.capture_revision.clone(), + observation_epoch: surface.observation_epoch, surface_point: point, window_point, screen_point, @@ -1324,6 +1333,7 @@ mod store_tests { }, window_bounds: None, capture_revision: CaptureRevision::parse("related-capture").unwrap(), + observation_epoch: None, transform: SurfaceToWindowTransform { scale_x: 0.5, scale_y: 0.5, diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/platform.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/platform.rs index 21e24b9805..ac8fdc8cf1 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/api/platform.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/platform.rs @@ -12,8 +12,8 @@ use super::{ }, errors::NativeError, interaction::{ - InteractionScope, MutationDeadline, NativeEvidence, PostureResult, ScopePlan, - ScopeRequirements, TargetCursorHandle, + InteractionScope, MutationDeadline, NativeEvidence, NativeSideEffectBoundary, + PostureResult, ScopePlan, ScopeRequirements, TargetCursorHandle, }, menu::NativeMenuEvidence, observation::{ @@ -116,7 +116,7 @@ pub enum ResolvedAction { point: ResolvedPoint, spec: ClickSpec, }, - Drag(ResolvedDrag), + Drag(Box), ElementScroll { element: ResolvedElement, spec: ElementScrollSpec, @@ -242,6 +242,17 @@ pub trait ObservationProvider: Send + Sync { pub trait SemanticActionProvider: Send + Sync { type PreparedAction: Send; + /// Determines whether an element click has an exact, currently usable + /// semantic primitive. A false result lets core resolve the element's + /// current observation-owned point and query the captured-point route. + /// Stale identity failures must be returned, never converted to fallback. + async fn element_click_candidate( + &self, + target: &mut TargetState, + element: &ResolvedElement, + spec: &ClickSpec, + ) -> Result; + async fn prepare( &self, target: &mut TargetState, @@ -253,52 +264,58 @@ pub trait SemanticActionProvider: Send + Sync { &self, target: &mut TargetState, scope: &mut InteractionScope, + boundary: &mut NativeSideEffectBoundary<'_>, action: Self::PreparedAction, ) -> Result; } -/// Dispatch-only targeted pointer operations. +/// Two-phase targeted pointer operations under the locked native target state. /// -/// Method entry is the controller-owned dispatch boundary. Implementations do -/// not perform a second preflight and must treat cancellation as a potentially -/// partial dispatch. +/// `prepare` is side-effect free and retains the exact surface/capture/native +/// route proof before core consumes the observation. `dispatch` is the +/// controller-owned dispatch boundary and may only execute that retained plan. #[async_trait] -pub trait PointerActionProvider: Send + Sync { - async fn click( - &self, - scope: &mut InteractionScope, - point: ResolvedPoint, - click: ClickSpec, - ) -> Result; - async fn drag( +pub trait PointerActionProvider: Send + Sync { + type PreparedAction: Send; + + async fn prepare( &self, + target: &mut TargetState, scope: &mut InteractionScope, - drag: ResolvedDrag, - ) -> Result; - async fn scroll( + action: &ResolvedAction, + ) -> Result; + + async fn dispatch( &self, + target: &mut TargetState, scope: &mut InteractionScope, - scroll: ResolvedScroll, + boundary: &mut NativeSideEffectBoundary<'_>, + action: Self::PreparedAction, ) -> Result; } -/// Dispatch-only keyboard operations against a core-resolved focus token. +/// Two-phase keyboard operations against a core-resolved focus token. /// -/// Method entry occurs after observation consumption and dirty marking. Any -/// fallible focus or route preparation belongs in `InteractionProvider`. +/// Normalization and exact native focus/route proof complete in `prepare` +/// before observation consumption. `dispatch` may only execute the retained +/// plan and must release any partially posted modifier sequence on failure. #[async_trait] -pub trait KeyboardActionProvider: Send + Sync { - async fn press_key( +pub trait KeyboardActionProvider: Send + Sync { + type PreparedAction: Send; + + async fn prepare( &self, + target: &mut TargetState, scope: &mut InteractionScope, - focus: &ResolvedFocus, - stroke: KeyStroke, - ) -> Result; - async fn type_text( + action: &ResolvedAction, + ) -> Result; + + async fn dispatch( &self, + target: &mut TargetState, scope: &mut InteractionScope, - focus: &ResolvedFocus, - text: &str, + boundary: &mut NativeSideEffectBoundary<'_>, + action: Self::PreparedAction, ) -> Result; } @@ -347,8 +364,8 @@ pub trait PlatformDriver: Send + Sync + 'static { type Windows: WindowProvider; type Observation: ObservationProvider; type Semantic: SemanticActionProvider; - type Pointer: PointerActionProvider; - type Keyboard: KeyboardActionProvider; + type Pointer: PointerActionProvider; + type Keyboard: KeyboardActionProvider; type Interaction: InteractionProvider; type Invalidations: InvalidationSubscription; diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/settlement.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/settlement.rs index e61f718279..0e003cf357 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/api/settlement.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/settlement.rs @@ -11,6 +11,7 @@ use super::{contracts::ActionId, errors::NativeError}; pub enum SettlementSignal { DispatchStarted, DispatchComplete, + PointerSequenceComplete, AxAction, AxValueChanged, FocusChanged, diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/api/target.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/api/target.rs index e9220d42b2..0bc52c3d79 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/api/target.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/api/target.rs @@ -10,6 +10,7 @@ use std::{ }; use tokio::sync::Mutex as AsyncMutex; +use uuid::Uuid; use super::{ contracts::{AppId, ClientId, WindowGeneration, WindowId}, @@ -90,6 +91,9 @@ pub struct TargetControllerState { } pub struct TargetController { + /// Opaque, process-local correlation id for content-free diagnostics. + /// It is not a stable public handle and never crosses the v2 wire. + pub instance_id: String, pub key: TargetKey, pub process: NativeProcessHandle, pub mutation_lock: SharedProcessMutationLock, @@ -108,6 +112,7 @@ impl TargetController

{ mutation_lock: SharedProcessMutationLock, ) -> Self { Self { + instance_id: Uuid::new_v4().to_string(), key, process: window.process.clone(), mutation_lock, diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs index 174d9e1a7c..5f3c2fe41b 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/protocol.rs @@ -29,7 +29,10 @@ impl Request { } pub fn tool_call(&self) -> anyhow::Result { - let params = self.params.as_ref().ok_or_else(|| anyhow::anyhow!("missing params"))?; + let params = self + .params + .as_ref() + .ok_or_else(|| anyhow::anyhow!("missing params"))?; let name = params .get("name") .and_then(|v| v.as_str()) @@ -59,6 +62,26 @@ pub struct V2ProtocolVersion { pub const V2_PROTOCOL_VERSION: V2ProtocolVersion = V2ProtocolVersion { major: 2, minor: 0 }; +/// Canonical native method inventory. Keep this list as the single schema +/// comparison boundary for the Rust daemon and typed clients. +pub const V2_METHODS: &[&str] = &[ + "driver.v2.check_readiness", + "driver.v2.get_capabilities", + "driver.v2.list_apps", + "driver.v2.launch_app", + "driver.v2.list_windows", + "driver.v2.get_window", + "driver.v2.get_window_state", + "driver.v2.click", + "driver.v2.drag", + "driver.v2.scroll", + "driver.v2.press_key", + "driver.v2.type_text", + "driver.v2.set_value", + "driver.v2.select_text", + "driver.v2.perform_secondary_action", +]; + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(tag = "method", content = "params", deny_unknown_fields)] pub enum V2Command { @@ -94,6 +117,47 @@ pub enum V2Command { PerformSecondaryAction(SecondaryActionCommand), } +impl V2Command { + pub fn method(&self) -> &'static str { + match self { + Self::CheckReadiness(_) => "driver.v2.check_readiness", + Self::GetCapabilities(_) => "driver.v2.get_capabilities", + Self::ListApps(_) => "driver.v2.list_apps", + Self::LaunchApp(_) => "driver.v2.launch_app", + Self::ListWindows(_) => "driver.v2.list_windows", + Self::GetWindow(_) => "driver.v2.get_window", + Self::GetWindowState(_) => "driver.v2.get_window_state", + Self::Click(_) => "driver.v2.click", + Self::Drag(_) => "driver.v2.drag", + Self::Scroll(_) => "driver.v2.scroll", + Self::PressKey(_) => "driver.v2.press_key", + Self::TypeText(_) => "driver.v2.type_text", + Self::SetValue(_) => "driver.v2.set_value", + Self::SelectText(_) => "driver.v2.select_text", + Self::PerformSecondaryAction(_) => "driver.v2.perform_secondary_action", + } + } + + /// Returns whether dispatch may change externally visible application + /// state. These requests share one connection-wide lane so launch and + /// action ordering stays deterministic while discovery and observation + /// requests remain free to overlap. + pub fn requires_serial_dispatch(&self) -> bool { + matches!( + self, + Self::LaunchApp(_) + | Self::Click(_) + | Self::Drag(_) + | Self::Scroll(_) + | Self::PressKey(_) + | Self::TypeText(_) + | Self::SetValue(_) + | Self::SelectText(_) + | Self::PerformSecondaryAction(_) + ) + } +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct V2RequestEnvelope { @@ -140,10 +204,10 @@ pub struct V2HandshakeResponse { pub driver_name: String, pub driver_version: String, pub build: String, + pub methods: Vec, } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] pub struct V2ResponseEnvelope { pub request_id: String, pub protocol_version: V2ProtocolVersion, @@ -195,14 +259,23 @@ pub struct RpcError { impl Response { pub fn ok(id: Value, result: Value) -> Self { - Self { jsonrpc: "2.0", id, body: ResponseBody::Result { result } } + Self { + jsonrpc: "2.0", + id, + body: ResponseBody::Result { result }, + } } pub fn error(id: Value, code: i64, message: impl Into) -> Self { Self { jsonrpc: "2.0", id, - body: ResponseBody::Error { error: RpcError { code, message: message.into() } }, + body: ResponseBody::Error { + error: RpcError { + code, + message: message.into(), + }, + }, } } @@ -227,7 +300,7 @@ pub enum Content { annotations: Option, }, Image { - data: String, // base64-encoded PNG + data: String, // base64-encoded PNG #[serde(rename = "mimeType")] mime_type: String, #[serde(skip_serializing_if = "Option::is_none")] @@ -237,15 +310,26 @@ pub enum Content { impl Content { pub fn text(text: impl Into) -> Self { - Content::Text { text: text.into(), annotations: None } + Content::Text { + text: text.into(), + annotations: None, + } } pub fn image_png(data_base64: String) -> Self { - Content::Image { data: data_base64, mime_type: "image/png".into(), annotations: None } + Content::Image { + data: data_base64, + mime_type: "image/png".into(), + annotations: None, + } } pub fn image_jpeg(data_base64: String) -> Self { - Content::Image { data: data_base64, mime_type: "image/jpeg".into(), annotations: None } + Content::Image { + data: data_base64, + mime_type: "image/jpeg".into(), + annotations: None, + } } } @@ -261,11 +345,18 @@ pub struct ToolResult { impl ToolResult { pub fn text(msg: impl Into) -> Self { - Self { content: vec![Content::text(msg)], ..Default::default() } + Self { + content: vec![Content::text(msg)], + ..Default::default() + } } pub fn error(msg: impl Into) -> Self { - Self { content: vec![Content::text(msg)], is_error: Some(true), ..Default::default() } + Self { + content: vec![Content::text(msg)], + is_error: Some(true), + ..Default::default() + } } pub fn with_structured(mut self, v: Value) -> Self { @@ -346,7 +437,10 @@ mod image_mime_type_tests { let c = Content::image_png("ZmFrZQ==".into()); let v = serde_json::to_value(&c).expect("serialize"); assert_eq!(v.get("type").and_then(|t| t.as_str()), Some("image")); - assert_eq!(v.get("mimeType").and_then(|t| t.as_str()), Some("image/png")); + assert_eq!( + v.get("mimeType").and_then(|t| t.as_str()), + Some("image/png") + ); assert_eq!(v.get("data").and_then(|t| t.as_str()), Some("ZmFrZQ==")); } @@ -355,7 +449,10 @@ mod image_mime_type_tests { let c = Content::image_jpeg("ZmFrZQ==".into()); let v = serde_json::to_value(&c).expect("serialize"); assert_eq!(v.get("type").and_then(|t| t.as_str()), Some("image")); - assert_eq!(v.get("mimeType").and_then(|t| t.as_str()), Some("image/jpeg")); + assert_eq!( + v.get("mimeType").and_then(|t| t.as_str()), + Some("image/jpeg") + ); } /// Text parts must not grow a mimeType field — the field belongs to @@ -366,6 +463,9 @@ mod image_mime_type_tests { let c = Content::text("hi"); let v = serde_json::to_value(&c).expect("serialize"); assert_eq!(v.get("type").and_then(|t| t.as_str()), Some("text")); - assert!(v.get("mimeType").is_none(), "text content must not carry mimeType"); + assert!( + v.get("mimeType").is_none(), + "text content must not carry mimeType" + ); } } diff --git a/libs/cua-driver/rust/crates/cua-driver-core/src/server.rs b/libs/cua-driver/rust/crates/cua-driver-core/src/server.rs index 97a0be020d..7da18fca18 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/src/server.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/src/server.rs @@ -1,11 +1,42 @@ //! Async MCP stdio server loop. use std::sync::Arc; -use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; +use tokio::io::{AsyncBufRead, AsyncBufReadExt, AsyncRead, AsyncWrite, AsyncWriteExt, BufReader}; +use tokio::sync::{mpsc, Semaphore}; +use tokio::task::{JoinError, JoinSet}; use tracing::{debug, error, warn}; -use crate::protocol::{initialize_result, Request, Response}; use crate::tool::ToolRegistry; +use crate::{ + api::{ClientId, DriverController, ErrorCode, ErrorPhase, NativeError, PlatformDriver}, + protocol::{ + initialize_result, Request, Response, V2Command, V2Failure, V2HandshakeRequest, + V2HandshakeResponse, V2ProtocolVersion, V2RequestEnvelope, V2ResponseBody, + V2ResponseEnvelope, V2Success, V2_METHODS, V2_PROTOCOL_VERSION, + }, +}; + +const V2_MAX_LINE_BYTES: usize = 32 * 1024 * 1024; +const V2_MAX_IN_FLIGHT_REQUESTS: usize = 128; +const V2_RESPONSE_QUEUE_CAPACITY: usize = 128; +const V2_IDLE_SWEEP_INTERVAL: std::time::Duration = std::time::Duration::from_secs(60); + +#[derive(Debug, Clone)] +pub struct V2ServerMetadata { + pub driver_name: String, + pub driver_version: String, + pub build: String, +} + +impl V2ServerMetadata { + pub fn current(driver_name: impl Into, build: impl Into) -> Self { + Self { + driver_name: driver_name.into(), + driver_version: env!("CARGO_PKG_VERSION").to_owned(), + build: build.into(), + } + } +} /// Run the MCP server, reading JSON-RPC lines from stdin and writing /// responses to stdout. Exits when stdin reaches EOF or a fatal I/O @@ -57,11 +88,527 @@ pub async fn run(registry: Arc) -> anyhow::Result<()> { Ok(()) } +/// Run the native, background-only v2 protocol over stdio. +/// +/// This is intentionally separate from MCP and the legacy tool registry. The +/// first non-empty line must be the version-range handshake; every later line +/// is one strict [`V2RequestEnvelope`]. One opaque client id is created for the +/// transport lifetime and all target state is destroyed before this function +/// returns, regardless of whether the stream ended cleanly or with an error. +pub async fn run_v2( + controller: Arc>, + metadata: V2ServerMetadata, +) -> anyhow::Result<()> { + run_v2_io( + tokio::io::stdin(), + tokio::io::stdout(), + controller, + metadata, + ) + .await +} + +/// I/O-generic v2 server used by the production stdio entrypoint and protocol +/// boundary tests. It has no access to `ToolRegistry` by construction. +pub async fn run_v2_io( + input: R, + output: W, + controller: Arc>, + metadata: V2ServerMetadata, +) -> anyhow::Result<()> +where + R: AsyncRead + Unpin, + W: AsyncWrite + Unpin + Send + 'static, + P: PlatformDriver, +{ + ensure_v2_method_inventory()?; + let client_id = ClientId::new(); + let invalidation_task = controller.start_invalidation_loop(); + let idle_controller = Arc::clone(&controller); + let idle_task = tokio::spawn(async move { + let mut interval = tokio::time::interval(V2_IDLE_SWEEP_INTERVAL); + interval.tick().await; + loop { + interval.tick().await; + if let Err(error) = idle_controller.expire_idle_targets().await { + tracing::error!( + code = ?error.code, + phase = ?error.phase, + "v2 target idle-expiry teardown failed" + ); + } + } + }); + + let serve_result = serve_v2_io(input, output, &controller, &client_id, &metadata).await; + idle_task.abort(); + invalidation_task.abort(); + let _ = idle_task.await; + let _ = invalidation_task.await; + let cleanup_result = controller.close_connection(&client_id).await; + + match (serve_result, cleanup_result) { + (Ok(()), Ok(count)) => { + debug!(client_id = %client_id, targets_destroyed = count, "v2 connection closed"); + Ok(()) + } + (Err(error), Ok(count)) => { + debug!(client_id = %client_id, targets_destroyed = count, "v2 connection failed and closed"); + Err(error) + } + (Ok(()), Err(cleanup)) => Err(anyhow::anyhow!( + "v2 connection cleanup failed (code={:?}, phase={:?}): {}", + cleanup.code, + cleanup.phase, + cleanup.message + )), + (Err(error), Err(cleanup)) => { + tracing::error!( + client_id = %client_id, + code = ?cleanup.code, + phase = ?cleanup.phase, + "v2 connection cleanup also failed" + ); + Err(error) + } + } +} + +async fn serve_v2_io( + input: R, + output: W, + controller: &Arc>, + client_id: &ClientId, + metadata: &V2ServerMetadata, +) -> anyhow::Result<()> +where + R: AsyncRead + Unpin, + W: AsyncWrite + Unpin + Send + 'static, + P: PlatformDriver, +{ + let mut reader = BufReader::new(input); + let (response_tx, response_rx) = mpsc::channel(V2_RESPONSE_QUEUE_CAPACITY); + let mut writer_task = tokio::spawn(write_v2_responses(output, response_rx)); + let mut dispatch_tasks = JoinSet::new(); + let effectful_dispatch = Arc::new(Semaphore::new(1)); + + let handshake_line = match read_v2_line(&mut reader).await { + Ok(Some(line)) => line, + Ok(None) => { + return finish_v2_writer(response_tx, writer_task, Ok(())).await; + } + Err(error) => { + return finish_v2_writer(response_tx, writer_task, Err(error)).await; + } + }; + let handshake = match serde_json::from_str::(&handshake_line) { + Ok(handshake) => handshake, + Err(error) => { + let request_id = request_id_from_invalid_line(&handshake_line); + let queued = queue_v2_failure( + &response_tx, + request_id, + NativeError::invalid(format!("invalid v2 handshake: {error}")), + ) + .await; + return finish_v2_writer(response_tx, writer_task, queued).await; + } + }; + if let Err(protocol_error) = validate_handshake_range(&handshake) { + let queued = queue_v2_failure(&response_tx, handshake.request_id, protocol_error).await; + return finish_v2_writer(response_tx, writer_task, queued).await; + } + if let Err(error) = queue_v2_json( + &response_tx, + &V2HandshakeResponse { + request_id: handshake.request_id, + minimum_version: V2_PROTOCOL_VERSION, + maximum_version: V2_PROTOCOL_VERSION, + driver_name: metadata.driver_name.clone(), + driver_version: metadata.driver_version.clone(), + build: metadata.build.clone(), + methods: V2_METHODS + .iter() + .map(|method| (*method).to_owned()) + .collect(), + }, + ) + .await + { + return finish_v2_writer(response_tx, writer_task, Err(error)).await; + } + + let mut serve_result = Ok(()); + let mut completed_writer_result = None; + loop { + tokio::select! { + writer_result = &mut writer_task => { + completed_writer_result = Some(flatten_writer_result(writer_result)); + break; + } + completed = dispatch_tasks.join_next(), if !dispatch_tasks.is_empty() => { + if let Some(completed) = completed { + if let Err(error) = flatten_dispatch_result(completed) { + serve_result = Err(error); + break; + } + } + } + line = read_v2_line(&mut reader) => { + let line = match line { + Ok(Some(line)) => line, + Ok(None) => break, + Err(error) => { + serve_result = Err(error); + break; + } + }; + let request = match serde_json::from_str::(&line) { + Ok(request) => request, + Err(error) => { + if let Err(write_error) = queue_v2_failure( + &response_tx, + request_id_from_invalid_line(&line), + NativeError::invalid(format!("invalid v2 request envelope: {error}")), + ) + .await + { + serve_result = Err(write_error); + break; + } + continue; + } + }; + let request_id = request.request_id.clone(); + let method = request.command.method(); + if let Err(version_error) = request.validate_version() { + if let Err(write_error) = + queue_v2_failure(&response_tx, request_id, version_error).await + { + serve_result = Err(write_error); + break; + } + continue; + } + if dispatch_tasks.len() >= V2_MAX_IN_FLIGHT_REQUESTS { + let overload = NativeError::new( + ErrorCode::TargetBusy, + ErrorPhase::Preflight, + true, + "native v2 connection reached its in-flight request limit", + ) + .with_detail("limit", V2_MAX_IN_FLIGHT_REQUESTS); + if let Err(write_error) = + queue_v2_failure(&response_tx, request_id, overload).await + { + serve_result = Err(write_error); + break; + } + continue; + } + let effectful_permit = if request.command.requires_serial_dispatch() { + match Arc::clone(&effectful_dispatch).try_acquire_owned() { + Ok(permit) => Some(permit), + Err(_) => { + let busy = NativeError::new( + ErrorCode::TargetBusy, + ErrorPhase::Preflight, + true, + "another launch or mutation is already active on this v2 connection", + ) + .with_detail("method", method); + if let Err(write_error) = + queue_v2_failure(&response_tx, request_id, busy).await + { + serve_result = Err(write_error); + break; + } + continue; + } + } + } else { + None + }; + debug!(request_id, method, client_id = %client_id, "v2 request started"); + let task_controller = Arc::clone(controller); + let task_client_id = client_id.clone(); + let task_response_tx = response_tx.clone(); + dispatch_tasks.spawn(async move { + // The owned permit covers the complete native dispatch and + // is dropped only after its typed result is constructed. + let _effectful_permit = effectful_permit; + let result = + dispatch_v2(&task_controller, &task_client_id, request.command).await; + match result { + Ok(result) => { + debug!(request_id, method, "v2 request completed"); + queue_v2_json( + &task_response_tx, + &V2ResponseEnvelope { + request_id, + protocol_version: V2_PROTOCOL_VERSION, + body: V2ResponseBody::Result(V2Success { result }), + }, + ) + .await + } + Err(error) => { + warn!( + request_id, + method, + code = ?error.code, + phase = ?error.phase, + retryable = error.retryable, + "v2 request failed" + ); + queue_v2_failure(&task_response_tx, request_id, error).await + } + } + }); + } + } + } + + // EOF ends intake, not deadline-owned native work. In particular, a + // launch posture witness deliberately keeps containment alive until its + // own deadline. Drain every bounded request before connection teardown so + // dropping the transport cannot release target state underneath it. + while let Some(completed) = dispatch_tasks.join_next().await { + if let Err(error) = flatten_dispatch_result(completed) { + if serve_result.is_ok() { + serve_result = Err(error); + } else { + tracing::error!("v2 request task also failed while draining"); + } + } + } + + drop(response_tx); + let writer_result = match completed_writer_result { + Some(result) => result, + None => flatten_writer_result(writer_task.await), + }; + combine_v2_serve_results(serve_result, writer_result) +} + +async fn write_v2_responses( + output: W, + mut responses: mpsc::Receiver>, +) -> anyhow::Result<()> { + let mut writer = tokio::io::BufWriter::new(output); + while let Some(serialized) = responses.recv().await { + writer.write_all(&serialized).await?; + writer.write_all(b"\n").await?; + writer.flush().await?; + } + Ok(()) +} + +fn flatten_dispatch_result(result: Result, JoinError>) -> anyhow::Result<()> { + match result { + Ok(result) => result, + Err(error) => Err(anyhow::anyhow!("v2 request task failed: {error}")), + } +} + +fn flatten_writer_result(result: Result, JoinError>) -> anyhow::Result<()> { + match result { + Ok(result) => result, + Err(error) => Err(anyhow::anyhow!("v2 response writer task failed: {error}")), + } +} + +fn combine_v2_serve_results( + serve_result: anyhow::Result<()>, + writer_result: anyhow::Result<()>, +) -> anyhow::Result<()> { + match (serve_result, writer_result) { + (Ok(()), Ok(())) => Ok(()), + (Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error), + (Err(serve_error), Err(_writer_error)) => { + tracing::error!("v2 response writer also failed after the serve loop failed"); + Err(serve_error) + } + } +} + +async fn finish_v2_writer( + response_tx: mpsc::Sender>, + writer_task: tokio::task::JoinHandle>, + serve_result: anyhow::Result<()>, +) -> anyhow::Result<()> { + drop(response_tx); + combine_v2_serve_results(serve_result, flatten_writer_result(writer_task.await)) +} + +async fn dispatch_v2( + controller: &DriverController

, + client_id: &ClientId, + command: V2Command, +) -> Result { + let result = match command { + V2Command::CheckReadiness(_) => serde_json::to_value(controller.check_readiness().await?), + V2Command::GetCapabilities(_) => serde_json::to_value(controller.get_capabilities().await?), + V2Command::ListApps(request) => serde_json::to_value(controller.list_apps(request).await?), + V2Command::LaunchApp(request) => { + serde_json::to_value(controller.launch_app(request).await?) + } + V2Command::ListWindows(request) => { + serde_json::to_value(controller.list_windows(request).await?) + } + V2Command::GetWindow(request) => { + serde_json::to_value(controller.get_window(request).await?) + } + V2Command::GetWindowState(request) => { + serde_json::to_value(controller.get_window_state(client_id, request).await?) + } + V2Command::Click(command) => { + serde_json::to_value(controller.click(client_id, command).await?) + } + V2Command::Drag(command) => { + serde_json::to_value(controller.drag(client_id, command).await?) + } + V2Command::Scroll(command) => { + serde_json::to_value(controller.scroll(client_id, command).await?) + } + V2Command::PressKey(command) => { + serde_json::to_value(controller.press_key(client_id, command).await?) + } + V2Command::TypeText(command) => { + serde_json::to_value(controller.type_text(client_id, command).await?) + } + V2Command::SetValue(command) => { + serde_json::to_value(controller.set_value(client_id, command).await?) + } + V2Command::SelectText(command) => { + serde_json::to_value(controller.select_text(client_id, command).await?) + } + V2Command::PerformSecondaryAction(command) => serde_json::to_value( + controller + .perform_secondary_action(client_id, command) + .await?, + ), + }; + result.map_err(|error| { + NativeError::new( + ErrorCode::Internal, + ErrorPhase::Verify, + false, + format!("failed to serialize typed v2 result: {error}"), + ) + }) +} + +fn validate_handshake_range(request: &V2HandshakeRequest) -> Result<(), NativeError> { + let ordered = request.minimum_version.major < request.maximum_version.major + || (request.minimum_version.major == request.maximum_version.major + && request.minimum_version.minor <= request.maximum_version.minor); + let supported = version_at_least(V2_PROTOCOL_VERSION, request.minimum_version) + && version_at_least(request.maximum_version, V2_PROTOCOL_VERSION); + if ordered && supported { + return Ok(()); + } + Err(NativeError::new( + ErrorCode::ProtocolMismatch, + ErrorPhase::Validate, + false, + format!( + "client protocol range {}.{} through {}.{} does not include driver protocol {}.{}", + request.minimum_version.major, + request.minimum_version.minor, + request.maximum_version.major, + request.maximum_version.minor, + V2_PROTOCOL_VERSION.major, + V2_PROTOCOL_VERSION.minor, + ), + )) +} + +fn version_at_least(left: V2ProtocolVersion, right: V2ProtocolVersion) -> bool { + (left.major, left.minor) >= (right.major, right.minor) +} + +fn ensure_v2_method_inventory() -> anyhow::Result<()> { + let mut methods = V2_METHODS.to_vec(); + methods.sort_unstable(); + if methods.windows(2).any(|pair| pair[0] == pair[1]) { + anyhow::bail!("duplicate native v2 method in startup inventory"); + } + if methods.len() != 15 { + anyhow::bail!( + "native v2 startup inventory has {} methods; expected 15", + methods.len() + ); + } + Ok(()) +} + +async fn read_v2_line(reader: &mut R) -> anyhow::Result> { + loop { + let mut line = String::new(); + let bytes = reader.read_line(&mut line).await?; + if bytes == 0 { + return Ok(None); + } + if bytes > V2_MAX_LINE_BYTES { + anyhow::bail!("v2 protocol line exceeded {V2_MAX_LINE_BYTES} bytes"); + } + let trimmed = line.trim(); + if !trimmed.is_empty() { + return Ok(Some(trimmed.to_owned())); + } + } +} + +fn request_id_from_invalid_line(line: &str) -> String { + serde_json::from_str::(line) + .ok() + .and_then(|value| { + value + .get("request_id") + .and_then(|id| id.as_str()) + .map(str::to_owned) + }) + .filter(|id| !id.trim().is_empty()) + .unwrap_or_else(|| "unknown".to_owned()) +} + +async fn queue_v2_failure( + responses: &mpsc::Sender>, + request_id: String, + error: NativeError, +) -> anyhow::Result<()> { + queue_v2_json( + responses, + &V2ResponseEnvelope:: { + request_id, + protocol_version: V2_PROTOCOL_VERSION, + body: V2ResponseBody::Error(V2Failure { error }), + }, + ) + .await +} + +async fn queue_v2_json( + responses: &mpsc::Sender>, + value: &T, +) -> anyhow::Result<()> { + let serialized = serde_json::to_vec(value)?; + responses + .send(serialized) + .await + .map_err(|_| anyhow::anyhow!("v2 response writer stopped before accepting a response")) +} + /// Dispatch one MCP JSON-RPC request against the registry (initialize / /// tools/list / tools/call). Shared by the stdio loop above and the /// daemon's HTTP transport (`cua-driver`'s `mcp_http`) so both speak the /// exact same MCP semantics. -pub async fn handle_request(req: Request, id: serde_json::Value, registry: &Arc) -> Response { +pub async fn handle_request( + req: Request, + id: serde_json::Value, + registry: &Arc, +) -> Response { match req.method.as_str() { "initialize" => Response::ok(id, initialize_result()), @@ -84,3 +631,70 @@ pub async fn handle_request(req: Request, id: serde_json::Value, registry: &Arc< } } } + +#[cfg(test)] +mod v2_tests { + use super::*; + + fn handshake(minimum: V2ProtocolVersion, maximum: V2ProtocolVersion) -> V2HandshakeRequest { + V2HandshakeRequest { + request_id: "handshake-1".to_owned(), + minimum_version: minimum, + maximum_version: maximum, + } + } + + #[test] + fn native_method_inventory_is_complete_and_unique() { + ensure_v2_method_inventory().expect("canonical v2 inventory"); + assert_eq!( + V2_METHODS, + [ + "driver.v2.check_readiness", + "driver.v2.get_capabilities", + "driver.v2.list_apps", + "driver.v2.launch_app", + "driver.v2.list_windows", + "driver.v2.get_window", + "driver.v2.get_window_state", + "driver.v2.click", + "driver.v2.drag", + "driver.v2.scroll", + "driver.v2.press_key", + "driver.v2.type_text", + "driver.v2.set_value", + "driver.v2.select_text", + "driver.v2.perform_secondary_action", + ] + ); + } + + #[test] + fn handshake_accepts_only_a_range_containing_the_native_version() { + assert!(validate_handshake_range(&handshake( + V2ProtocolVersion { major: 2, minor: 0 }, + V2ProtocolVersion { major: 2, minor: 0 }, + )) + .is_ok()); + + for request in [ + handshake( + V2ProtocolVersion { major: 3, minor: 0 }, + V2ProtocolVersion { major: 3, minor: 1 }, + ), + handshake( + V2ProtocolVersion { major: 1, minor: 0 }, + V2ProtocolVersion { major: 1, minor: 9 }, + ), + handshake( + V2ProtocolVersion { major: 2, minor: 1 }, + V2ProtocolVersion { major: 2, minor: 0 }, + ), + ] { + let error = validate_handshake_range(&request).expect_err("incompatible range"); + assert_eq!(error.code, ErrorCode::ProtocolMismatch); + assert_eq!(error.phase, ErrorPhase::Validate); + assert!(!error.retryable); + } + } +} diff --git a/libs/cua-driver/rust/crates/cua-driver-core/tests/api_v2_contracts.rs b/libs/cua-driver/rust/crates/cua-driver-core/tests/api_v2_contracts.rs index 717c408722..32f9acf200 100644 --- a/libs/cua-driver/rust/crates/cua-driver-core/tests/api_v2_contracts.rs +++ b/libs/cua-driver/rust/crates/cua-driver-core/tests/api_v2_contracts.rs @@ -393,7 +393,10 @@ struct FakePlatform { preflight_action_ids: Arc>>, acquired_action_ids: Arc>>, block_dispatch: std::sync::atomic::AtomicBool, + block_before_boundary: std::sync::atomic::AtomicBool, dispatch_fail: std::sync::atomic::AtomicBool, + dispatch_poison: std::sync::atomic::AtomicBool, + semantic_candidate_unusable: std::sync::atomic::AtomicBool, semantic_prepare_fail: std::sync::atomic::AtomicBool, semantic_verification_fail: std::sync::atomic::AtomicBool, observation_role: Mutex>, @@ -401,6 +404,8 @@ struct FakePlatform { cleanup_posture_violated: std::sync::atomic::AtomicBool, settle_pending: std::sync::atomic::AtomicBool, launch_fail: std::sync::atomic::AtomicBool, + launch_posture_unverifiable: std::sync::atomic::AtomicBool, + launch_posture_violation: std::sync::atomic::AtomicBool, refresh_geometry_during_observe: std::sync::atomic::AtomicBool, dispatch_entered: Notify, } @@ -484,7 +489,10 @@ impl LifecycleProvider for FakePlatform { }) } - async fn list_apps(&self, _query: AppQuery) -> Result, NativeError> { + async fn list_apps(&self, query: AppQuery) -> Result, NativeError> { + if query.name_contains.as_deref() == Some("slow-protocol-read") { + tokio::time::sleep(Duration::from_millis(50)).await; + } Ok(vec![app_ref()]) } @@ -495,7 +503,9 @@ impl LifecycleProvider for FakePlatform { ) -> Result { posture.begin_launch(); posture.record_partial_result(app_ref(), vec![window_ref()]); - posture.posture.held = true; + posture.posture.held = !self.launch_posture_unverifiable.load(Ordering::SeqCst) + && !self.launch_posture_violation.load(Ordering::SeqCst); + posture.posture.frontmost_changed = self.launch_posture_violation.load(Ordering::SeqCst); posture.pending_settlement = Some(PendingSettlementEvidence { state: PendingSettlementState::Pending, trigger_action_id: posture @@ -556,6 +566,37 @@ async fn post_dispatch_launch_failure_keeps_exact_partial_and_pending_evidence() assert_eq!(pending.trigger_action_id, *action_id); } +#[tokio::test] +async fn launch_posture_distinguishes_unverifiable_from_observed_violation() { + for (unverifiable, violation, expected) in [ + (true, false, ErrorCode::PostureUnverifiable), + (false, true, ErrorCode::PostureViolated), + ] { + let platform = Arc::new(FakePlatform::default()); + platform + .launch_posture_unverifiable + .store(unverifiable, Ordering::SeqCst); + platform + .launch_posture_violation + .store(violation, Ordering::SeqCst); + let controller = DriverController::new(platform, PlatformName::Macos, "test-os"); + let error = controller + .launch_app(LaunchAppRequest { + app: AppSelector::BundleId { + bundle_id: "com.example.fixture".to_owned(), + }, + }) + .await + .expect_err("non-held launch posture must fail"); + assert_eq!(error.code, expected); + assert_eq!(error.phase, ErrorPhase::Verify); + assert!(matches!( + error.partial_evidence.as_deref(), + Some(PartialEvidence::Launch { posture, .. }) if !posture.held + )); + } +} + #[async_trait] impl WindowProvider for FakePlatform { async fn list_windows(&self, _app: Option<&AppRef>) -> Result, NativeError> { @@ -667,6 +708,7 @@ impl ObservationProvider for FakePlatform { &format!("capture-{observation_suffix}"), CaptureRevision::parse, ), + observation_epoch: None, transform: SurfaceToWindowTransform { scale_x: 1.0, scale_y: 1.0, @@ -719,6 +761,19 @@ impl ObservationProvider for FakePlatform { impl SemanticActionProvider for FakePlatform { type PreparedAction = FakePreparedSemantic; + async fn element_click_candidate( + &self, + _target: &mut FakeTargetState, + element: &ResolvedElement, + spec: &ClickSpec, + ) -> Result { + Ok(!self.semantic_candidate_unusable.load(Ordering::SeqCst) + && spec.button == MouseButton::Left + && spec.click_count == 1 + && spec.modifiers.is_empty() + && element.actions.iter().any(|action| action == "AXPress")) + } + async fn prepare( &self, _target: &mut FakeTargetState, @@ -761,8 +816,10 @@ impl SemanticActionProvider for FakePlatform { &self, target: &mut FakeTargetState, _scope: &mut InteractionScope, + boundary: &mut NativeSideEffectBoundary<'_>, action: Self::PreparedAction, ) -> Result { + boundary.begin()?; target.semantic_dispatches += 1; self.dispatch_count.fetch_add(1, Ordering::SeqCst); self.record("dispatch"); @@ -787,13 +844,32 @@ impl SemanticActionProvider for FakePlatform { } #[async_trait] -impl PointerActionProvider for FakePlatform { - async fn click( +impl PointerActionProvider for FakePlatform { + type PreparedAction = ResolvedAction; + + async fn prepare( + &self, + _target: &mut FakeTargetState, + _scope: &mut InteractionScope, + action: &ResolvedAction, + ) -> Result { + self.record("pointer_prepare"); + Ok(action.clone()) + } + + async fn dispatch( &self, + _target: &mut FakeTargetState, _scope: &mut InteractionScope, - _point: ResolvedPoint, - _click: ClickSpec, + boundary: &mut NativeSideEffectBoundary<'_>, + _action: Self::PreparedAction, ) -> Result { + if self.block_before_boundary.load(Ordering::SeqCst) { + self.dispatch_entered.notify_waiters(); + std::future::pending::<()>().await; + } + boundary.begin()?; + boundary.begin()?; self.dispatch_count.fetch_add(1, Ordering::SeqCst); self.record("dispatch"); if self.block_dispatch.load(Ordering::SeqCst) { @@ -808,42 +884,39 @@ impl PointerActionProvider for FakePlatform { "fake dispatch failure", )); } + if self.dispatch_poison.load(Ordering::SeqCst) { + return Err(NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "fake native cleanup could not be proved", + ) + .with_target_invalidated()); + } Ok(NativeDispatch::dispatch_verified()) } - - async fn drag( - &self, - _scope: &mut InteractionScope, - _drag: ResolvedDrag, - ) -> Result { - Err(NativeError::unsupported("not used by this fixture")) - } - - async fn scroll( - &self, - _scope: &mut InteractionScope, - _scroll: ResolvedScroll, - ) -> Result { - Err(NativeError::unsupported("not used by this fixture")) - } } #[async_trait] -impl KeyboardActionProvider for FakePlatform { - async fn press_key( +impl KeyboardActionProvider for FakePlatform { + type PreparedAction = ResolvedAction; + + async fn prepare( &self, + _target: &mut FakeTargetState, _scope: &mut InteractionScope, - _focus: &ResolvedFocus, - _stroke: KeyStroke, - ) -> Result { - Err(NativeError::unsupported("not used by this fixture")) + action: &ResolvedAction, + ) -> Result { + self.record("keyboard_prepare"); + Ok(action.clone()) } - async fn type_text( + async fn dispatch( &self, + _target: &mut FakeTargetState, _scope: &mut InteractionScope, - _focus: &ResolvedFocus, - _text: &str, + _boundary: &mut NativeSideEffectBoundary<'_>, + _action: Self::PreparedAction, ) -> Result { Err(NativeError::unsupported("not used by this fixture")) } @@ -1051,6 +1124,61 @@ impl PlatformDriver for FakePlatform { } } +#[tokio::test] +async fn keyboard_actions_reject_unknown_observations_before_provider_prepare() { + let platform = Arc::new(FakePlatform::default()); + let controller = DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os"); + let client = id("keyboard-stale-client", ClientId::parse); + controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: false, + ax_tree_mode: AxTreeMode::Full, + }, + ) + .await + .unwrap(); + + let missing = id("missing-observation", ObservationId::parse); + let press_error = controller + .press_key( + &client, + PressKeyCommand { + window: window_ref(), + request: PressKeyRequest { + observation_id: missing.clone(), + stroke: KeyStroke { + key: "a".to_owned(), + modifiers: Vec::new(), + }, + }, + }, + ) + .await + .unwrap_err(); + let type_error = controller + .type_text( + &client, + TypeTextCommand { + window: window_ref(), + request: TypeTextRequest { + observation_id: missing, + text: "content-never-reaches-provider".to_owned(), + }, + }, + ) + .await + .unwrap_err(); + + for error in [press_error, type_error] { + assert_eq!(error.code, ErrorCode::ObservationStale); + } + assert_eq!(platform.ordering.lock().unwrap().as_slice(), ["observe"]); +} + #[tokio::test] async fn controller_preserves_target_state_and_consumes_at_dispatch_boundary() { let platform = Arc::new(FakePlatform::default()); @@ -1175,6 +1303,7 @@ async fn controller_preserves_target_state_and_consumes_at_dispatch_boundary() { "observe", "preflight", "scope_acquired", + "pointer_prepare", "dispatch", "settle", "scope_released" @@ -1342,6 +1471,67 @@ async fn semantic_element_dispatch_uses_the_locked_target_and_interaction_scope( ); } +#[tokio::test] +async fn element_click_without_exact_semantic_candidate_uses_current_observation_point_route() { + let platform = Arc::new(FakePlatform::default()); + platform + .semantic_candidate_unusable + .store(true, Ordering::SeqCst); + let controller = DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os"); + let client = id("element-point-fallback-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::Full, + }, + ) + .await + .unwrap(); + let element = observed.accessibility.as_ref().unwrap().elements[0] + .element_ref + .clone(); + + let receipt = controller + .click( + &client, + ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Element { element }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }, + ) + .await + .unwrap(); + + assert_eq!(receipt.route, Route::TargetedPointer); + assert!(platform + .ordering + .lock() + .unwrap() + .contains(&"pointer_prepare")); + assert_eq!( + controller + .targets + .get(&TargetKey::from_window(client, &resolved_window())) + .await + .unwrap() + .state + .lock() + .await + .platform + .semantic_dispatches, + 0 + ); +} + #[tokio::test] async fn semantic_prepare_refusal_preserves_the_observation_and_never_enters_dispatch() { let platform = Arc::new(FakePlatform::default()); @@ -1783,6 +1973,73 @@ async fn posture_violation_remains_primary_and_keeps_every_cleanup_failure() { assert_eq!(platform.shutdowns.load(Ordering::SeqCst), 1); } +#[tokio::test] +async fn cancellation_before_first_native_side_effect_preserves_current_settled_observation() { + let platform = Arc::new(FakePlatform::default()); + let controller = Arc::new(DriverController::new( + Arc::clone(&platform), + PlatformName::Macos, + "test-os", + )); + let client = id("cancel-before-boundary-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::Full, + }, + ) + .await + .unwrap(); + let command = ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Point { + observation_id: observed.observation_id.clone(), + surface_id: observed.surfaces[0].id.clone(), + point: Point { x: 20.0, y: 20.0 }, + }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }; + + platform.block_before_boundary.store(true, Ordering::SeqCst); + let dispatch_entered = platform.dispatch_entered.notified(); + let action = tokio::spawn({ + let controller = Arc::clone(&controller); + let client = client.clone(); + let command = command.clone(); + async move { controller.click(&client, command).await } + }); + dispatch_entered.await; + action.abort(); + assert!(action.await.unwrap_err().is_cancelled()); + assert_eq!(platform.cleanup_count.load(Ordering::SeqCst), 1); + + let target = controller + .targets + .get(&TargetKey::from_window(client.clone(), &resolved_window())) + .await + .unwrap(); + let mut state = target.state.lock().await; + assert!(state.settlement.settled_evidence().is_some()); + state + .observations + .current(&observed.observation_id, &resolved_window()) + .expect("pre-boundary cancellation must preserve the current observation"); + drop(state); + + platform + .block_before_boundary + .store(false, Ordering::SeqCst); + controller.click(&client, command).await.unwrap(); +} + #[tokio::test] async fn cancellation_inside_provider_leaves_observation_consumed_and_target_dirty() { let platform = Arc::new(FakePlatform::default()); @@ -1924,6 +2181,67 @@ async fn cancellation_cleanup_failure_poison_is_rebuilt_by_the_next_observation( assert_eq!(platform.shutdowns.load(Ordering::SeqCst), 1); } +#[tokio::test] +async fn typed_native_cleanup_poison_removes_target_and_forces_rebuild() { + let platform = Arc::new(FakePlatform::default()); + platform.dispatch_poison.store(true, Ordering::SeqCst); + let controller = DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os"); + let client = id("native-cleanup-poison-client", ClientId::parse); + let observed = controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::Full, + }, + ) + .await + .unwrap(); + let error = controller + .click( + &client, + ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Point { + observation_id: observed.observation_id, + surface_id: observed.surfaces[0].id.clone(), + point: Point { x: 20.0, y: 20.0 }, + }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }, + ) + .await + .unwrap_err(); + assert_eq!(error.code, ErrorCode::DispatchFailed); + assert!(controller + .targets + .get(&TargetKey::from_window(client.clone(), &resolved_window())) + .await + .is_err()); + + platform.dispatch_poison.store(false, Ordering::SeqCst); + controller + .get_window_state( + &client, + GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: true, + ax_tree_mode: AxTreeMode::Full, + }, + ) + .await + .unwrap(); + assert_eq!(platform.target_creations.load(Ordering::SeqCst), 2); + assert_eq!(platform.shutdowns.load(Ordering::SeqCst), 1); +} + #[tokio::test] async fn wedged_dispatch_deadline_releases_scope_without_reusing_observation() { let platform = Arc::new(FakePlatform::default()); @@ -2164,5 +2482,202 @@ fn menu_state_only_publishes_stable_lifecycle_states() { )); } +#[tokio::test] +async fn native_v2_stdio_drains_deadline_owned_work_before_eof_cleanup() { + use cua_driver_core::{ + protocol::{V2HandshakeRequest, V2HandshakeResponse, V2_METHODS}, + server::{run_v2_io, V2ServerMetadata}, + }; + use tokio::io::{AsyncBufReadExt, AsyncWriteExt, BufReader}; + + async fn write_json( + writer: &mut tokio::io::WriteHalf, + value: &impl serde::Serialize, + ) { + writer + .write_all(serde_json::to_string(value).unwrap().as_bytes()) + .await + .unwrap(); + writer.write_all(b"\n").await.unwrap(); + writer.flush().await.unwrap(); + } + + async fn read_json( + reader: &mut BufReader>, + ) -> T { + let mut line = String::new(); + reader.read_line(&mut line).await.unwrap(); + serde_json::from_str(&line).unwrap() + } + + let platform = Arc::new(FakePlatform::default()); + let controller = Arc::new( + DriverController::new(Arc::clone(&platform), PlatformName::Macos, "test-os") + .with_mutation_timeouts(Duration::from_millis(40), Duration::from_millis(100)), + ); + let targets = Arc::clone(&controller.targets); + let (client_stream, server_stream) = tokio::io::duplex(1024 * 1024); + let (server_read, server_write) = tokio::io::split(server_stream); + let server = tokio::spawn(run_v2_io( + server_read, + server_write, + controller, + V2ServerMetadata { + driver_name: "fake-macos".to_owned(), + driver_version: "test".to_owned(), + build: "test-build".to_owned(), + }, + )); + let (client_read, mut client_write) = tokio::io::split(client_stream); + let mut client_read = BufReader::new(client_read); + + write_json( + &mut client_write, + &V2HandshakeRequest { + request_id: "handshake".to_owned(), + minimum_version: V2_PROTOCOL_VERSION, + maximum_version: V2_PROTOCOL_VERSION, + }, + ) + .await; + let handshake: V2HandshakeResponse = read_json(&mut client_read).await; + assert_eq!(handshake.request_id, "handshake"); + assert_eq!(handshake.minimum_version, V2_PROTOCOL_VERSION); + assert_eq!(handshake.maximum_version, V2_PROTOCOL_VERSION); + assert_eq!( + handshake.methods, + V2_METHODS + .iter() + .map(|method| (*method).to_owned()) + .collect::>() + ); + + for (request_id, name_contains) in [ + ("apps-slow", "slow-protocol-read"), + ("apps-fast", "fast-protocol-read"), + ] { + write_json( + &mut client_write, + &V2RequestEnvelope { + request_id: request_id.to_owned(), + protocol_version: V2_PROTOCOL_VERSION, + command: V2Command::ListApps(ListAppsRequest { + query: Some(AppQuery { + name_contains: Some(name_contains.to_owned()), + running: None, + }), + }), + }, + ) + .await; + } + let fast: V2ResponseEnvelope = read_json(&mut client_read).await; + let slow: V2ResponseEnvelope = read_json(&mut client_read).await; + assert_eq!(fast.request_id, "apps-fast"); + assert_eq!(slow.request_id, "apps-slow"); + + let mut latest_state = None; + for (request_id, mode) in [ + ("state-full", AxTreeMode::Full), + ("state-diff", AxTreeMode::DiffIfAvailable), + ] { + write_json( + &mut client_write, + &V2RequestEnvelope { + request_id: request_id.to_owned(), + protocol_version: V2_PROTOCOL_VERSION, + command: V2Command::GetWindowState(GetWindowStateRequest { + window: window_ref(), + include_text: true, + include_screenshots: false, + ax_tree_mode: mode, + }), + }, + ) + .await; + let response: V2ResponseEnvelope = read_json(&mut client_read).await; + let V2ResponseBody::Result(result) = response.body else { + panic!("fake observation should succeed"); + }; + let state = result.result; + let accessibility = state.accessibility.as_ref().expect("fake AX state"); + match (mode, &accessibility.tree_update) { + (AxTreeMode::Full, AxTreeUpdate::Full { .. }) + | (AxTreeMode::DiffIfAvailable, AxTreeUpdate::Diff { .. }) => {} + other => panic!("unexpected revision mode: {other:?}"), + } + latest_state = Some(state); + } + + assert_eq!(platform.target_creations.load(Ordering::SeqCst), 1); + assert_eq!(targets.len().await, 1); + + let state = latest_state.expect("latest observation"); + platform.block_dispatch.store(true, Ordering::SeqCst); + let click_command = ClickCommand { + window: window_ref(), + request: ClickRequest { + target: ClickTarget::Point { + observation_id: state.observation_id, + surface_id: state.surfaces[0].id.clone(), + point: Point { x: 20.0, y: 20.0 }, + }, + button: MouseButton::Left, + click_count: 1, + modifiers: Vec::new(), + }, + }; + write_json( + &mut client_write, + &V2RequestEnvelope { + request_id: "click-drained-after-eof".to_owned(), + protocol_version: V2_PROTOCOL_VERSION, + command: V2Command::Click(click_command.clone()), + }, + ) + .await; + tokio::time::timeout(Duration::from_secs(1), async { + while platform.dispatch_count.load(Ordering::SeqCst) == 0 { + tokio::task::yield_now().await; + } + }) + .await + .expect("click dispatch entered before EOF"); + + write_json( + &mut client_write, + &V2RequestEnvelope { + request_id: "click-concurrent-refused".to_owned(), + protocol_version: V2_PROTOCOL_VERSION, + command: V2Command::Click(click_command), + }, + ) + .await; + let concurrent: V2ResponseEnvelope = read_json(&mut client_read).await; + assert_eq!(concurrent.request_id, "click-concurrent-refused"); + let V2ResponseBody::Error(failure) = concurrent.body else { + panic!("a second effectful request must not queue outside native deadlines"); + }; + assert_eq!(failure.error.code, ErrorCode::TargetBusy); + assert!(failure.error.retryable); + + client_write.shutdown().await.unwrap(); + drop(client_write); + tokio::time::timeout(Duration::from_secs(1), server) + .await + .expect("EOF drain honors the native mutation deadline") + .unwrap() + .unwrap(); + let response: V2ResponseEnvelope = read_json(&mut client_read).await; + assert_eq!(response.request_id, "click-drained-after-eof"); + let V2ResponseBody::Error(failure) = response.body else { + panic!("wedged dispatch must complete with its typed native deadline failure"); + }; + assert_eq!(failure.error.code, ErrorCode::DispatchFailed); + assert_eq!(platform.cleanup_count.load(Ordering::SeqCst), 1); + assert!(targets.is_empty().await); + assert_eq!(platform.shutdowns.load(Ordering::SeqCst), 1); +} + #[allow(dead_code)] fn _settlement_type_assertion(_: BTreeSet) {} diff --git a/libs/cua-driver/rust/crates/cua-driver/src/cli.rs b/libs/cua-driver/rust/crates/cua-driver/src/cli.rs index ffc9501013..d12a0f0782 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/cli.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/cli.rs @@ -17,6 +17,10 @@ use cua_driver_core::{protocol::Content, tool::ToolRegistry}; /// Which CLI command was requested. pub enum Command { + /// Native, typed background driver v2 protocol over newline-delimited + /// stdio. This is deliberately not MCP and cannot reach the legacy tool + /// registry. + DriverV2, Mcp { /// Force in-process MCP execution — skip the TCC auto-relaunch /// path that would spawn a daemon via `open -n -g -a CuaDriver @@ -162,7 +166,7 @@ pub fn parse_command() -> Command { if args.iter().any(|a| a == "--help" || a == "-h") { println!("cua-driver {} — cross-platform computer-use automation driver", env!("CARGO_PKG_VERSION")); println!("Usage: cua-driver [SUBCOMMAND] [OPTIONS]"); - println!("Subcommands: mcp, list-tools, describe, call, serve, stop, status, config, recording, update, check-update, doctor, diagnose, permissions, autostart, skills, manifest"); + println!("Subcommands: driver-v2, mcp, list-tools, describe, call, serve, stop, status, config, recording, update, check-update, doctor, diagnose, permissions, autostart, skills, manifest"); println!(); println!("permissions options (macOS):"); println!(" cua-driver permissions status Report Accessibility + Screen Recording status. Read-only (no prompt)."); @@ -322,6 +326,7 @@ pub fn parse_command() -> Command { socket: socket.clone(), claude_code_compat, }, + Some("driver-v2") => Command::DriverV2, Some("list-tools") => Command::ListTools, Some("mcp-config") => Command::McpConfig { client: mcp_client }, Some("serve") => Command::Serve { @@ -861,11 +866,19 @@ pub fn build_manifest() -> serde_json::Value { "command": binary, "args": ["mcp"] }, + "native_v2_invocation": { + "command": binary, + "args": ["driver-v2"], + "protocol_version": { "major": 2, "minor": 0 } + }, // Subcommand catalog — keep in sync with `parse_command` above. // `args` is a hint shape for consumers; the canonical source is // `--help` text. Each entry follows the same JSON shape so the // consumer can render uniformly. "subcommands": [ + { "name": "driver-v2", + "description": "Run the typed background-only native v2 protocol over newline-delimited stdio (macOS).", + "args": [] }, { "name": "mcp", "description": "Run the MCP JSON-RPC server over stdio (the default invocation).", "args": [ @@ -2872,6 +2885,7 @@ mod stdin_bom_tests { pub fn telemetry_entry_event(cmd: &Command) -> Option { use crate::telemetry::event; let name = match cmd { + Command::DriverV2 => "cua_driver_v2".to_owned(), Command::Mcp { .. } => event::MCP.to_owned(), Command::Serve { .. } => event::SERVE.to_owned(), Command::Stop { .. } => event::STOP.to_owned(), diff --git a/libs/cua-driver/rust/crates/cua-driver/src/main.rs b/libs/cua-driver/rust/crates/cua-driver/src/main.rs index 54d5c56329..68795e7a26 100644 --- a/libs/cua-driver/rust/crates/cua-driver/src/main.rs +++ b/libs/cua-driver/rust/crates/cua-driver/src/main.rs @@ -48,6 +48,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; /// they take `compat: bool` directly, but the binary crate decides what /// to pass without making every Command variant carry the flag. static CLAUDE_CODE_COMPAT: AtomicBool = AtomicBool::new(false); +#[cfg(target_os = "macos")] +static DRIVER_V2_MODE: AtomicBool = AtomicBool::new(false); fn init_logging() { use tracing_subscriber::EnvFilter; @@ -150,18 +152,31 @@ fn maybe_init_pip() { /// free of the `ureq` + rustls + ring deps that the check tool needs. #[cfg(target_os = "macos")] fn build_macos_registry() -> cua_driver_core::tool::ToolRegistry { - let mut r = platform_macos::register_tools(); + let mut r = platform_macos::register_legacy_tools_with_compat(false); check_update_tool::register_into(&mut r); r } #[cfg(target_os = "macos")] fn build_macos_registry_with_compat(compat: bool) -> cua_driver_core::tool::ToolRegistry { - let mut r = platform_macos::register_tools_with_compat(compat); + let mut r = platform_macos::register_legacy_tools_with_compat(compat); check_update_tool::register_into(&mut r); r } +#[cfg(target_os = "macos")] +fn current_macos_version() -> String { + std::process::Command::new("/usr/bin/sw_vers") + .arg("-productVersion") + .output() + .ok() + .filter(|output| output.status.success()) + .and_then(|output| String::from_utf8(output.stdout).ok()) + .map(|version| version.trim().to_owned()) + .filter(|version| !version.is_empty()) + .unwrap_or_else(|| "unknown".to_owned()) +} + // ── macOS entry-point ───────────────────────────────────────────────────── #[cfg(target_os = "macos")] @@ -174,6 +189,9 @@ fn main() { let command = cli::parse_command(); emit_entry_telemetry(&command); match command { + cli::Command::DriverV2 => { + DRIVER_V2_MODE.store(true, Ordering::SeqCst); + } cli::Command::TelemetryInstallEvent => { // Synchronous install ping (see `telemetry::capture_install`). // Blocks on the POST so the `.installation_recorded` marker @@ -452,7 +470,13 @@ fn main() { } } - let cursor_cfg = cursor_overlay::CursorConfig::from_args(); + let driver_v2 = DRIVER_V2_MODE.load(Ordering::SeqCst); + let mut cursor_cfg = cursor_overlay::CursorConfig::from_args(); + if driver_v2 { + // The typed v2 surface owns a per-target logical cursor but exposes no + // user-visible cursor/overlay API. + cursor_cfg.enabled = false; + } tracing::info!( version = env!("CARGO_PKG_VERSION"), @@ -508,24 +532,49 @@ fn main() { .build() .expect("tokio runtime"); let compat = CLAUDE_CODE_COMPAT.load(Ordering::SeqCst); - rt.block_on(async move { - // Register tools; overlay init has already happened above. - let registry = Arc::new(build_macos_registry_with_compat(compat)); - // Wire up replay tool's back-reference to the registry. - registry.init_self_weak(); - if let Err(e) = cua_driver_core::server::run(registry).await { - tracing::error!("MCP server error: {e}"); + let server_result = rt.block_on(async move { + if driver_v2 { + let platform = Arc::new(platform_macos::driver::MacDriver::new()); + let controller = Arc::new(cua_driver_core::api::DriverController::new( + platform, + cua_driver_core::api::PlatformName::Macos, + current_macos_version(), + )); + let metadata = cua_driver_core::server::V2ServerMetadata::current( + "cua-driver-macos", + option_env!("CUA_DRIVER_BUILD_ID").unwrap_or("dev"), + ); + cua_driver_core::server::run_v2(controller, metadata).await + } else { + // Register tools; overlay init has already happened above. + let registry = Arc::new(build_macos_registry_with_compat(compat)); + // Wire up replay tool's back-reference to the registry. + registry.init_self_weak(); + cua_driver_core::server::run(registry).await } }); + let exit_code = match server_result { + Ok(()) => 0, + Err(error) => { + if driver_v2 { + tracing::error!("native v2 server error: {error}"); + } else { + tracing::error!("MCP server error: {error}"); + } + 1 + } + }; // MCP server exited (stdin closed / client disconnected). // The main thread is blocked in NSApplication.run() and won't // exit on its own — force-exit the process cleanly. - std::process::exit(0); + std::process::exit(exit_code); }) .expect("spawn mcp thread"); // Main thread: AppKit overlay (blocks until the process exits). - if enabled { + if driver_v2 && platform_macos::session::has_graphic_access() { + platform_macos::pip::run_appkit_main_loop(); + } else if enabled { platform_macos::cursor::overlay::run_on_main_thread(); } // Overlay disabled: park the main thread while the MCP background thread @@ -546,6 +595,10 @@ fn main() -> anyhow::Result<()> { let command = cli::parse_command(); emit_entry_telemetry(&command); match command { + cli::Command::DriverV2 => { + eprintln!("cua-driver: driver-v2 is currently implemented only on macOS"); + return Ok(()); + } cli::Command::TelemetryInstallEvent => { // Synchronous install ping (see `telemetry::capture_install`). // Blocks on the POST so the `.installation_recorded` marker diff --git a/libs/cua-driver/rust/crates/platform-macos/src/browser/electron_js.rs b/libs/cua-driver/rust/crates/platform-macos/src/browser/electron_js.rs index c4683eebae..620bab49b2 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/browser/electron_js.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/browser/electron_js.rs @@ -125,4 +125,3 @@ fn find_bundle_path_for_app(name: &str) -> Option { } None } - diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/keyboard.rs new file mode 100644 index 0000000000..8ad04d7771 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/keyboard.rs @@ -0,0 +1,1486 @@ +//! Exact focused-control preparation and background-only macOS keyboard routes. + +use std::{ + collections::{BTreeMap, BTreeSet}, + sync::Arc, + time::{Duration, Instant}, +}; + +use async_trait::async_trait; +use core_graphics::event::CGEventFlags; +use cua_driver_core::api::{ + capabilities::{ + ActionKind, AddressingMode, CapabilityCell, CapabilityKey, Framework, PlatformName, + RouteDecision, WindowStateKind, + }, + contracts::{Modifier, Route, VerificationLevel}, + errors::{ErrorCode, ErrorPhase, NativeError}, + interaction::{InteractionScope, LeaseDecision, NativeEvidence, NativeSideEffectBoundary}, + observation::ResolvedFocus, + platform::{KeyboardActionProvider, NativeDispatch, ResolvedAction}, + settlement::SettlementSignal, +}; +use serde_json::{json, Value}; + +use crate::{ + ax::bindings::{ + self, copy_cf_range_attr, copy_string_attr_exact, focused_element_of_pid, + is_attribute_settable, kAXErrorSuccess, AxCfRange, + }, + input::keyboard::{ + chord_events, normalize_chord, post_prepared_targeted_event, prepare_targeted_event, + unicode_events, NormalizedChord, NormalizedModifier, TargetedKeyEvent, + TargetedKeyEventKind, TargetedPostPrimitive, + }, +}; + +use super::super::{ + observation::{RegisteredElementSnapshot, RetainedAxElement}, + target::MacTargetState, + windows::{MacWindowFacts, MacWindowRegistry}, +}; + +const EVENT_GAP: Duration = Duration::from_millis(8); + +trait KeyboardEventPoster: Send + Sync { + fn post( + &self, + pid: i32, + event: &TargetedKeyEvent, + boundary: &mut NativeSideEffectBoundary<'_>, + ) -> Result; +} + +#[derive(Default)] +struct SystemKeyboardEventPoster; + +impl KeyboardEventPoster for SystemKeyboardEventPoster { + fn post( + &self, + pid: i32, + event: &TargetedKeyEvent, + boundary: &mut NativeSideEffectBoundary<'_>, + ) -> Result { + let event = prepare_targeted_event(event)?; + boundary.begin()?; + Ok(post_prepared_targeted_event(pid, &event)) + } +} + +#[derive(Clone)] +pub struct MacKeyboardActions { + windows: MacWindowRegistry, + poster: Arc, + event_gap: Duration, +} + +pub struct MacPreparedKeyboardAction(PreparedKind); + +enum PreparedKind { + Chord { + pid: i32, + chord: NormalizedChord, + focus: PreparedFocus, + }, + SemanticInsertion { + focus: PreparedFocus, + text: String, + expected_value: String, + expected_selection: AxCfRange, + }, + UnicodeInsertion { + pid: i32, + focus: PreparedFocus, + text: String, + expected: Option, + }, +} + +struct PreparedFocus { + element: RetainedAxElement, + snapshot: RegisteredElementSnapshot, + ax_revision: String, + observation_id: String, + signal_epoch: u64, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct ExpectedInsertion { + value: String, + selection: AxCfRange, +} + +impl MacKeyboardActions { + pub fn new(windows: MacWindowRegistry) -> Self { + Self { + windows, + poster: Arc::new(SystemKeyboardEventPoster), + event_gap: EVENT_GAP, + } + } + + pub(crate) async fn prepare_action( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + action: &ResolvedAction, + ) -> Result { + let facts = self.windows.facts_for_stamp(&target.window).await?; + let prepared = match action { + ResolvedAction::PressKey { focus, stroke } => { + require_scope(target, scope, Route::TargetedKeyboard, true)?; + // Repeated here after the no-lease InteractionProvider check so + // the retained plan never depends on mutable side state. + let chord = normalize_chord(stroke)?; + let (focus, _) = prepare_focus(target, focus, &facts, FocusProof::IdentityOnly)?; + PreparedKind::Chord { + pid: facts.pid, + chord, + focus, + } + } + ResolvedAction::TypeText { focus, text } => match scope.route { + Route::Semantic => { + require_scope(target, scope, Route::Semantic, false)?; + let (focus, snapshot) = + prepare_focus(target, focus, &facts, FocusProof::TextState)?; + if snapshot.selected_text_settable != Some(true) { + return Err(NativeError::unsupported( + "focused macOS control has no exact writable AXSelectedText insertion route", + )); + } + let before = snapshot.string_value.as_deref().ok_or_else(|| { + NativeError::unsupported( + "semantic text insertion requires an exact CFString AXValue snapshot", + ) + })?; + let selection = snapshot.selected_text_range.ok_or_else(|| { + NativeError::unsupported( + "semantic text insertion requires an exact AXSelectedTextRange snapshot", + ) + })?; + let expected = apply_insertion(before, selection, text)?; + PreparedKind::SemanticInsertion { + focus, + text: text.clone(), + expected_value: expected.value, + expected_selection: expected.selection, + } + } + Route::TargetedKeyboard => { + require_scope(target, scope, Route::TargetedKeyboard, true)?; + let (focus, snapshot) = + prepare_focus(target, focus, &facts, FocusProof::TextState)?; + let expected = match ( + snapshot.string_value.as_deref(), + snapshot.selected_text_range, + ) { + (Some(before), Some(selection)) => { + Some(apply_insertion(before, selection, text)?) + } + _ => None, + }; + PreparedKind::UnicodeInsertion { + pid: facts.pid, + focus, + text: text.clone(), + expected, + } + } + Route::TargetedPointer => { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "text action reached the targeted-pointer route", + )) + } + }, + _ => { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "non-keyboard action reached macOS keyboard preparation", + )) + } + }; + Ok(MacPreparedKeyboardAction(prepared)) + } + + pub(crate) async fn dispatch_action( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + boundary: &mut NativeSideEffectBoundary<'_>, + action: MacPreparedKeyboardAction, + ) -> Result { + match action.0 { + PreparedKind::Chord { pid, chord, focus } => { + self.final_focus_validation( + target, + scope.owner.clone(), + scope.window.stamp(), + pid, + &focus, + FocusProof::IdentityOnly, + ) + .await?; + let evidence = + dispatch_chord(self.poster.as_ref(), self.event_gap, pid, &chord, boundary) + .map_err(|error| poison_unproved_cleanup(target, scope, error))?; + Ok(dispatch_unverified( + "macos_targeted_key_chord", + focus, + evidence, + )) + } + PreparedKind::UnicodeInsertion { + pid, + focus, + text, + expected, + } => { + self.final_focus_validation( + target, + scope.owner.clone(), + scope.window.stamp(), + pid, + &focus, + FocusProof::TextState, + ) + .await?; + let mut evidence = dispatch_unicode( + self.poster.as_ref(), + self.event_gap, + pid, + &text, + scope.deadline.work, + boundary, + ) + .map_err(|error| poison_unproved_cleanup(target, scope, error))?; + let readback = expected + .as_ref() + .map(|expected| exact_insertion_readback(focus.element.as_ptr(), expected)); + let readback_available = readback.is_some(); + let readback_matched = matches!(&readback, Some(Ok(true))); + if let Some(Err(error)) = &readback { + evidence.insert( + "exact_readback_error".to_owned(), + Value::String(error.message.clone()), + ); + } + evidence.insert( + "exact_readback_available".to_owned(), + readback_available.into(), + ); + evidence.insert("exact_readback_matched".to_owned(), readback_matched.into()); + if matches!(&readback, Some(Ok(_))) { + target + .signals + .record(SettlementSignal::VerificationReadbackComplete); + } + // A same-thread AX readback is useful positive evidence when it + // matches, but a targeted pid post has no delivery ack. Keep + // the contract honest and let notification settlement finish. + Ok(dispatch_unverified( + "macos_targeted_unicode_events", + focus, + evidence, + )) + } + PreparedKind::SemanticInsertion { + focus, + text, + expected_value, + expected_selection, + } => { + let pid = self.windows.facts_for_stamp(&target.window).await?.pid; + self.final_focus_validation( + target, + scope.owner.clone(), + scope.window.stamp(), + pid, + &focus, + FocusProof::TextState, + ) + .await?; + boundary.begin()?; + let result = unsafe { + bindings::set_string_attr(focus.element.as_ptr(), "AXSelectedText", &text) + }; + if result != kAXErrorSuccess { + return Err(NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "AXSelectedText insertion failed after entering the dispatch boundary", + ) + .with_detail("ax_error", result) + .with_detail("possibly_partial_delivery", true)); + } + let expected = ExpectedInsertion { + value: expected_value, + selection: expected_selection, + }; + let readback = exact_insertion_readback(focus.element.as_ptr(), &expected)?; + target + .signals + .record(SettlementSignal::VerificationReadbackComplete); + if !readback { + return Err(NativeError::new( + ErrorCode::VerificationFailed, + ErrorPhase::Verify, + true, + "AXSelectedText insertion did not produce the exact value and caret readback", + ) + .with_detail("possibly_partial_delivery", true)); + } + let mut evidence = focus_evidence(&focus); + evidence.insert( + "primitive".to_owned(), + Value::String("macos_ax_selected_text_insertion".to_owned()), + ); + evidence.insert("clipboard_used".to_owned(), false.into()); + evidence.insert("exact_typed_readback".to_owned(), true.into()); + evidence.insert( + "inserted_utf16_units".to_owned(), + text.encode_utf16().count().into(), + ); + Ok(NativeDispatch { + verification: VerificationLevel::EffectVerified, + evidence: NativeEvidence { + fields: evidence, + interaction_scope: None, + }, + warnings: Vec::new(), + menu: None, + }) + } + } + } + + async fn final_focus_validation( + &self, + target: &mut MacTargetState, + scope_owner: cua_driver_core::api::observation::ResolvedWindowStamp, + scope_window: cua_driver_core::api::observation::ResolvedWindowStamp, + pid: i32, + focus: &PreparedFocus, + proof: FocusProof, + ) -> Result<(), NativeError> { + if target.invalidated() || scope_owner != target.window || scope_window != target.window { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "keyboard target window changed after action preparation", + )); + } + let facts = self.windows.facts_for_stamp(&target.window).await?; + if facts.pid != pid || facts.cg_window_id != focus.snapshot.owner_window_id { + return Err(focus_drift( + "keyboard pid/window identity changed before native dispatch", + )); + } + if target.signals.epoch() != focus.signal_epoch { + return Err(NativeError::stale( + ErrorCode::ObservationRaced, + "focus/content notification raced keyboard action preparation", + )); + } + let live = unsafe { focused_element_of_pid(pid) } + .map(|element| unsafe { RetainedAxElement::from_owned(element) }) + .ok_or_else(|| focus_drift("target process lost its exact focused AX element"))?; + validate_focus_identity_proof( + focus.snapshot.owner == target.window, + focus.snapshot.owner_window_id, + live.same_identity(&focus.element), + unsafe { bindings::ax_get_window_id(live.as_ptr()) }, + facts.cg_window_id, + )?; + let live_role = unsafe { copy_string_attr_exact(live.as_ptr(), "AXRole") } + .map_err(|error| ax_focus_error("AXRole", error))?; + if !focus.snapshot.role_proven || live_role.as_deref() != Some(focus.snapshot.role.as_str()) + { + return Err(focus_drift( + "focused-control role changed before native dispatch", + )); + } + if matches!(proof, FocusProof::TextState) { + revalidate_text_state(&live, &focus.snapshot)?; + } + if target.signals.epoch() != focus.signal_epoch { + return Err(NativeError::stale( + ErrorCode::ObservationRaced, + "focus/content notification raced final keyboard validation", + )); + } + Ok(()) + } +} + +#[async_trait] +impl KeyboardActionProvider for MacKeyboardActions { + type PreparedAction = MacPreparedKeyboardAction; + + async fn prepare( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + action: &ResolvedAction, + ) -> Result { + self.prepare_action(target, scope, action).await + } + + async fn dispatch( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + boundary: &mut NativeSideEffectBoundary<'_>, + action: Self::PreparedAction, + ) -> Result { + self.dispatch_action(target, scope, boundary, action).await + } +} + +#[derive(Clone, Copy)] +enum FocusProof { + IdentityOnly, + TextState, +} + +fn prepare_focus( + target: &MacTargetState, + focus: &ResolvedFocus, + facts: &MacWindowFacts, + proof: FocusProof, +) -> Result<(PreparedFocus, RegisteredElementSnapshot), NativeError> { + let epoch = target.signals.epoch(); + let element_id = focus_element_id(focus_contract(target, focus)?)?; + let snapshot = target + .elements + .registered_by_id(element_id) + .ok_or_else(missing_focus_identity)?; + let live = unsafe { focused_element_of_pid(facts.pid) } + .map(|element| unsafe { RetainedAxElement::from_owned(element) }) + .ok_or_else(|| focus_drift("target process has no exact focused AX element"))?; + validate_focus_identity_proof( + snapshot.owner == target.window, + snapshot.owner_window_id, + live.same_identity(&snapshot.element), + unsafe { bindings::ax_get_window_id(live.as_ptr()) }, + facts.cg_window_id, + )?; + let live_role = unsafe { copy_string_attr_exact(live.as_ptr(), "AXRole") } + .map_err(|error| ax_focus_error("AXRole", error))?; + if !snapshot.role_proven || live_role.as_deref() != Some(snapshot.role.as_str()) { + return Err(focus_drift( + "live focused-control role changed since the supplied observation", + )); + } + if matches!(proof, FocusProof::TextState) { + revalidate_text_state(&live, &snapshot)?; + } + if target.signals.epoch() != epoch { + return Err(NativeError::stale( + ErrorCode::ObservationRaced, + "target focus/content notification raced keyboard preparation", + )); + } + Ok(( + PreparedFocus { + element: live, + snapshot: snapshot.clone(), + ax_revision: focus + .ax_revision + .as_ref() + .expect("focus contract requires AX revision") + .as_str() + .to_owned(), + observation_id: focus.observation_id.to_string(), + signal_epoch: epoch, + }, + snapshot, + )) +} + +fn validate_focus_identity_proof( + observed_owner_matches: bool, + observed_window_id: u32, + live_identity_matches: bool, + live_window_id: Option, + target_window_id: u32, +) -> Result<(), NativeError> { + if !observed_owner_matches || observed_window_id != target_window_id { + return Err(focus_drift( + "observed focused control no longer belongs to the exact target window", + )); + } + if !live_identity_matches || live_window_id != Some(target_window_id) { + return Err(focus_drift( + "live AX focus identity changed since the supplied observation", + )); + } + Ok(()) +} + +fn revalidate_text_state( + live: &RetainedAxElement, + snapshot: &RegisteredElementSnapshot, +) -> Result<(), NativeError> { + if snapshot.value_query_proven { + let live_value = unsafe { copy_string_attr_exact(live.as_ptr(), "AXValue") } + .map_err(|error| ax_focus_error("AXValue", error))?; + if live_value != snapshot.string_value { + return Err(focus_drift( + "focused-control value changed since the supplied observation", + )); + } + } + if let Some(expected_range) = snapshot.selected_text_range { + let live_range = unsafe { copy_cf_range_attr(live.as_ptr(), "AXSelectedTextRange") } + .map_err(|error| ax_focus_error("AXSelectedTextRange", error))?; + if live_range != Some(expected_range) { + return Err(focus_drift( + "focused-control selection changed since the supplied observation", + )); + } + } + if let Some(expected_settable) = snapshot.selected_text_settable { + let live_settable = unsafe { is_attribute_settable(live.as_ptr(), "AXSelectedText") } + .map_err(|error| ax_focus_error("AXSelectedText settable", error))?; + if live_settable != expected_settable { + return Err(focus_drift( + "focused-control insertion capability changed since observation", + )); + } + } + Ok(()) +} + +fn focus_contract<'a>( + target: &MacTargetState, + focus: &'a ResolvedFocus, +) -> Result<&'a ResolvedFocus, NativeError> { + if target.invalidated() || focus.window.stamp() != target.window { + return Err(focus_drift( + "resolved focus target no longer matches the locked macOS target", + )); + } + if focus.ax_revision.is_none() { + return Err(focus_drift( + "keyboard action requires an exact observed AX focus revision", + )); + } + Ok(focus) +} + +fn focus_element_id( + focus: &ResolvedFocus, +) -> Result<&cua_driver_core::api::contracts::ElementId, NativeError> { + focus.focused_element.as_ref().ok_or_else(|| { + NativeError::unsupported( + "keyboard action requires an exact focused element in the current observation", + ) + }) +} + +fn require_scope( + target: &MacTargetState, + scope: &InteractionScope, + route: Route, + target_belief: bool, +) -> Result<(), NativeError> { + let belief_ok = if target_belief { + scope.leases.target_belief == LeaseDecision::Acquired + } else { + scope.leases.target_belief == LeaseDecision::NotApplicable + }; + if scope.route != route + || scope.owner != target.window + || scope.window.stamp() != target.window + || scope.leases.posture_witness != LeaseDecision::Acquired + || !belief_ok + { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "keyboard action entered without its exact locked interaction scope", + )); + } + Ok(()) +} + +/// Deliberately contains no await point. Once the first modifier-down is +/// posted, controller cancellation cannot drop this future before every +/// modifier-up attempt has completed. +fn dispatch_chord( + poster: &dyn KeyboardEventPoster, + event_gap: Duration, + pid: i32, + chord: &NormalizedChord, + boundary: &mut NativeSideEffectBoundary<'_>, +) -> Result, NativeError> { + let events = chord_events(chord); + let first_release = chord.modifiers.len() + 2; + let mut attempted = 0_usize; + let mut delivered = 0_usize; + let mut possibly_down = Vec::new(); + let mut main_key_possibly_down = false; + let mut primitives = BTreeSet::new(); + + for event in &events[..first_release] { + attempted += 1; + if let TargetedKeyEventKind::ModifierDown(modifier) = event.kind { + possibly_down.push(normalized_modifier_for(chord, modifier)); + } + if event.kind == TargetedKeyEventKind::KeyDown { + // A void post can return an error after native delivery, so key + // cleanup ownership starts before the down attempt. + main_key_possibly_down = true; + } + match poster.post(pid, event, boundary) { + Ok(primitive) => { + delivered += 1; + primitives.insert(primitive_name(primitive)); + if event.kind == TargetedKeyEventKind::KeyUp { + main_key_possibly_down = false; + } + wait_gap_blocking(event_gap); + } + Err(error) => { + let main_cleanup = release_main_key( + poster, + event_gap, + pid, + chord.key_code, + event.flags, + main_key_possibly_down, + boundary, + ); + let cleanup = release_modifiers( + poster, + event_gap, + pid, + &possibly_down, + event.flags, + boundary, + ); + attempted += usize::from(main_cleanup.attempted) + cleanup.attempted.len(); + delivered += usize::from(main_cleanup.succeeded) + cleanup.succeeded.len(); + return Err(chord_failure( + error, + attempted, + delivered, + main_cleanup, + cleanup, + )); + } + } + } + + let cleanup = release_modifiers( + poster, + event_gap, + pid, + &possibly_down, + events + .get(first_release.saturating_sub(1)) + .map_or(CGEventFlags::CGEventFlagNull, |event| event.flags), + boundary, + ); + attempted += cleanup.attempted.len(); + delivered += cleanup.succeeded.len(); + primitives.extend(cleanup.primitives.iter().copied()); + if !cleanup.failures.is_empty() { + let error = NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "one or more modifier-up posts failed", + ); + return Err(chord_failure( + error, + attempted, + delivered, + MainKeyCleanup::default(), + cleanup, + )); + } + + Ok(BTreeMap::from([ + ("event_count_attempted".to_owned(), attempted.into()), + ("event_count_posted".to_owned(), delivered.into()), + ( + "modifier_releases_attempted".to_owned(), + json!(modifier_names(&possibly_down)), + ), + ("main_key_cleanup_attempted".to_owned(), false.into()), + ( + "post_primitives".to_owned(), + json!(primitives.into_iter().collect::>()), + ), + ("clipboard_used".to_owned(), false.into()), + ])) +} + +#[derive(Default)] +struct MainKeyCleanup { + attempted: bool, + succeeded: bool, + failure: Option, + primitive: Option<&'static str>, +} + +fn release_main_key( + poster: &dyn KeyboardEventPoster, + event_gap: Duration, + pid: i32, + key_code: u16, + flags: CGEventFlags, + possibly_down: bool, + boundary: &mut NativeSideEffectBoundary<'_>, +) -> MainKeyCleanup { + if !possibly_down { + return MainKeyCleanup::default(); + } + let event = TargetedKeyEvent { + kind: TargetedKeyEventKind::KeyUp, + key_code, + key_down: false, + flags, + text: None, + }; + let result = poster.post(pid, &event, boundary); + wait_gap_blocking(event_gap); + match result { + Ok(primitive) => MainKeyCleanup { + attempted: true, + succeeded: true, + failure: None, + primitive: Some(primitive_name(primitive)), + }, + Err(error) => MainKeyCleanup { + attempted: true, + succeeded: false, + failure: Some(error), + primitive: None, + }, + } +} + +fn dispatch_unicode( + poster: &dyn KeyboardEventPoster, + event_gap: Duration, + pid: i32, + text: &str, + deadline: Instant, + boundary: &mut NativeSideEffectBoundary<'_>, +) -> Result, NativeError> { + let events = unicode_events(text); + let mut primitives = BTreeSet::new(); + let mut attempted = 0_usize; + let mut posted = 0_usize; + let mut completed_pairs = 0_usize; + for pair in events.chunks_exact(2) { + if Instant::now() >= deadline { + return Err(NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "targeted Unicode sequence reached its controller deadline between complete key pairs", + ) + .with_detail("deadline_stage", "unicode_pair_boundary") + .with_detail("completed_pairs", completed_pairs) + .with_detail("event_count_attempted", attempted) + .with_detail("event_count_posted", posted)); + } + let down = &pair[0]; + let up = &pair[1]; + attempted += 1; + let down_result = poster.post(pid, down, boundary); + if let Ok(primitive) = down_result.as_ref() { + posted += 1; + primitives.insert(primitive_name(*primitive)); + } + wait_gap_blocking(event_gap); + + // The pair is one non-cancellable synchronous region. Even when the + // down post reports failure, it may have reached the void native API, + // so up is always attempted before control returns. + attempted += 1; + let up_result = poster.post(pid, up, boundary); + if let Ok(primitive) = up_result.as_ref() { + posted += 1; + primitives.insert(primitive_name(*primitive)); + } + wait_gap_blocking(event_gap); + + if down_result.is_err() || up_result.is_err() { + let cleanup_unproved = up_result.is_err(); + let mut failure = NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "targeted Unicode pair failed after possible partial delivery; key-up was attempted", + ) + .with_detail("possibly_partial_delivery", boundary.started()) + .with_detail("event_count_attempted", attempted) + .with_detail("event_count_posted", posted) + .with_detail("unicode_key_up_attempted", true) + .with_detail("unicode_key_up_succeeded", up_result.is_ok()); + if let Err(error) = down_result { + failure = failure.with_related(&error); + } + if let Err(error) = up_result { + failure = failure.with_related(&error); + } + if cleanup_unproved { + failure = failure.with_target_invalidated(); + } + return Err(failure); + } + completed_pairs += 1; + } + Ok(BTreeMap::from([ + ("event_count_attempted".to_owned(), attempted.into()), + ("event_count_posted".to_owned(), posted.into()), + ("completed_pairs".to_owned(), completed_pairs.into()), + ( + "post_primitives".to_owned(), + json!(primitives.into_iter().collect::>()), + ), + ("clipboard_used".to_owned(), false.into()), + ])) +} + +#[derive(Default)] +struct ModifierCleanup { + attempted: Vec, + succeeded: Vec, + failures: Vec, + primitives: BTreeSet<&'static str>, +} + +fn release_modifiers( + poster: &dyn KeyboardEventPoster, + event_gap: Duration, + pid: i32, + modifiers: &[NormalizedModifier], + mut flags: core_graphics::event::CGEventFlags, + boundary: &mut NativeSideEffectBoundary<'_>, +) -> ModifierCleanup { + let mut cleanup = ModifierCleanup::default(); + for modifier in modifiers.iter().rev() { + flags.remove(modifier.flag); + let event = TargetedKeyEvent { + kind: TargetedKeyEventKind::ModifierUp(modifier.modifier), + key_code: modifier.key_code, + key_down: false, + flags, + text: None, + }; + cleanup.attempted.push(modifier.modifier); + match poster.post(pid, &event, boundary) { + Ok(primitive) => { + cleanup.succeeded.push(modifier.modifier); + cleanup.primitives.insert(primitive_name(primitive)); + } + Err(error) => cleanup.failures.push(error), + } + wait_gap_blocking(event_gap); + } + cleanup +} + +fn chord_failure( + error: NativeError, + attempted: usize, + delivered: usize, + main_cleanup: MainKeyCleanup, + cleanup: ModifierCleanup, +) -> NativeError { + let mut failure = NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "targeted key chord failed after possible partial delivery; every possible modifier release was attempted", + ) + .with_detail("possibly_partial_delivery", delivered > 0) + .with_detail("event_count_attempted", attempted) + .with_detail("event_count_posted", delivered) + .with_detail("main_key_cleanup_attempted", main_cleanup.attempted) + .with_detail("main_key_cleanup_succeeded", main_cleanup.succeeded) + .with_detail( + "modifier_releases_attempted", + json!(modifier_names_raw(&cleanup.attempted)), + ) + .with_detail( + "modifier_releases_succeeded", + json!(modifier_names_raw(&cleanup.succeeded)), + ) + .with_detail("modifier_release_failures", cleanup.failures.len()) + .with_related(&error); + let cleanup_unproved = + (main_cleanup.attempted && !main_cleanup.succeeded) || !cleanup.failures.is_empty(); + if let Some(primitive) = main_cleanup.primitive { + failure = failure.with_detail("main_key_cleanup_primitive", primitive); + } + if let Some(main_error) = main_cleanup.failure { + failure = failure.with_related(&main_error); + } + for release_error in cleanup.failures { + failure = failure.with_related(&release_error); + } + if cleanup_unproved { + failure = failure.with_target_invalidated(); + } + failure +} + +fn poison_unproved_cleanup( + target: &mut MacTargetState, + scope: &InteractionScope, + error: NativeError, +) -> NativeError { + if error.target_invalidated() { + target.invalidate(); + scope.invalidate_target(); + } + error +} + +fn apply_insertion( + before: &str, + selection: AxCfRange, + inserted: &str, +) -> Result { + let location = usize::try_from(selection.location) + .map_err(|_| invalid_selection("selection location is negative"))?; + let length = usize::try_from(selection.length) + .map_err(|_| invalid_selection("selection length is negative"))?; + let end = location + .checked_add(length) + .ok_or_else(|| invalid_selection("selection range overflowed"))?; + let start_byte = byte_index_at_utf16(before, location) + .ok_or_else(|| invalid_selection("selection start is not a UTF-16 boundary"))?; + let end_byte = byte_index_at_utf16(before, end) + .ok_or_else(|| invalid_selection("selection end is not a UTF-16 boundary"))?; + let mut value = String::with_capacity(before.len() + inserted.len()); + value.push_str(&before[..start_byte]); + value.push_str(inserted); + value.push_str(&before[end_byte..]); + let caret = location + .checked_add(inserted.encode_utf16().count()) + .ok_or_else(|| invalid_selection("inserted caret location overflowed"))?; + Ok(ExpectedInsertion { + value, + selection: AxCfRange::from_utf16(caret, 0) + .ok_or_else(|| invalid_selection("inserted caret exceeds CFRange"))?, + }) +} + +fn byte_index_at_utf16(value: &str, target: usize) -> Option { + if target == 0 { + return Some(0); + } + let mut utf16 = 0_usize; + for (byte, character) in value.char_indices() { + if utf16 == target { + return Some(byte); + } + utf16 = utf16.checked_add(character.len_utf16())?; + if utf16 > target { + return None; + } + } + (utf16 == target).then_some(value.len()) +} + +fn exact_insertion_readback( + element: bindings::AXUIElementRef, + expected: &ExpectedInsertion, +) -> Result { + let value = unsafe { copy_string_attr_exact(element, "AXValue") } + .map_err(|error| ax_readback_error("AXValue", error))?; + let selection = unsafe { copy_cf_range_attr(element, "AXSelectedTextRange") } + .map_err(|error| ax_readback_error("AXSelectedTextRange", error))?; + Ok(value.as_deref() == Some(expected.value.as_str()) && selection == Some(expected.selection)) +} + +fn dispatch_unverified( + primitive: &str, + focus: PreparedFocus, + mut fields: BTreeMap, +) -> NativeDispatch { + fields.extend(focus_evidence(&focus)); + fields.insert("primitive".to_owned(), Value::String(primitive.to_owned())); + NativeDispatch { + verification: VerificationLevel::DispatchUnverified, + evidence: NativeEvidence { + fields, + interaction_scope: None, + }, + warnings: Vec::new(), + menu: None, + } +} + +fn focus_evidence(focus: &PreparedFocus) -> BTreeMap { + BTreeMap::from([ + ( + "focus_ax_revision".to_owned(), + Value::String(focus.ax_revision.clone()), + ), + ( + "focus_observation_id".to_owned(), + Value::String(focus.observation_id.clone()), + ), + ("focus_identity_revalidated".to_owned(), true.into()), + ]) +} + +fn normalized_modifier_for(chord: &NormalizedChord, modifier: Modifier) -> NormalizedModifier { + chord + .modifiers + .iter() + .find(|candidate| candidate.modifier == modifier) + .copied() + .expect("modifier-down event came from normalized chord") +} + +fn modifier_names(modifiers: &[NormalizedModifier]) -> Vec<&'static str> { + modifiers + .iter() + .map(|modifier| modifier_name(modifier.modifier)) + .collect() +} + +fn modifier_names_raw(modifiers: &[Modifier]) -> Vec<&'static str> { + modifiers.iter().copied().map(modifier_name).collect() +} + +fn modifier_name(modifier: Modifier) -> &'static str { + match modifier { + Modifier::Shift => "shift", + Modifier::Control => "control", + Modifier::Alt => "alt", + Modifier::Meta => "meta", + } +} + +fn primitive_name(primitive: TargetedPostPrimitive) -> &'static str { + match primitive { + TargetedPostPrimitive::SkyLightAuthenticated => "skylight_authenticated_pid", + TargetedPostPrimitive::CoreGraphicsPid => "core_graphics_pid", + } +} + +fn wait_gap_blocking(gap: Duration) { + if !gap.is_zero() { + std::thread::sleep(gap); + } +} + +fn missing_focus_identity() -> NativeError { + focus_drift("observed focused element is missing from the retained AX registry") +} + +fn focus_drift(message: &str) -> NativeError { + NativeError::stale(ErrorCode::AxRevisionMismatch, message) +} + +fn invalid_selection(message: &str) -> NativeError { + NativeError::stale(ErrorCode::AxRevisionMismatch, message) +} + +fn ax_focus_error(attribute: &str, error: bindings::AXError) -> NativeError { + focus_drift(&format!( + "exact focused-control {attribute} revalidation failed with AX error {error}" + )) +} + +fn ax_readback_error(attribute: &str, error: bindings::AXError) -> NativeError { + NativeError::new( + ErrorCode::VerificationFailed, + ErrorPhase::Verify, + true, + format!("exact {attribute} insertion readback failed with AX error {error}"), + ) +} + +pub(crate) fn keyboard_capability_cells(os_version: &str) -> Vec { + let frameworks = [ + Framework::Unknown, + Framework::AppKit, + Framework::Chromium, + Framework::WebKit, + Framework::Electron, + Framework::Catalyst, + ]; + let states = [ + WindowStateKind::Visible, + WindowStateKind::Occluded, + WindowStateKind::Minimized, + WindowStateKind::OffSpace, + WindowStateKind::Unknown, + ]; + let mut cells = Vec::with_capacity(frameworks.len() * states.len() * 2); + for framework in frameworks { + for state in &states { + let type_text = match state { + WindowStateKind::Visible | WindowStateKind::Occluded + if matches!( + framework, + Framework::AppKit | Framework::Chromium | Framework::WebKit + ) => + { + RouteDecision::Supported { + route: Route::Semantic, + } + } + WindowStateKind::Visible | WindowStateKind::Occluded => { + RouteDecision::Unsupported { + reason: "recipe_unproven: no exact background text recipe is published for this framework before manual host QA".to_owned(), + } + } + _ => RouteDecision::Unsupported { + reason: + "macOS keyboard/text actions require an exact current-Space non-minimized window state" + .to_owned(), + }, + }; + cells.push(CapabilityCell { + key: CapabilityKey { + platform: PlatformName::Macos, + os_version: os_version.to_owned(), + action: ActionKind::TypeText, + addressing: AddressingMode::ObservedFocus, + framework: framework.clone(), + window_state: state.clone(), + }, + decision: type_text, + }); + cells.push(CapabilityCell { + key: CapabilityKey { + platform: PlatformName::Macos, + os_version: os_version.to_owned(), + action: ActionKind::PressKey, + addressing: AddressingMode::ObservedFocus, + framework: framework.clone(), + window_state: state.clone(), + }, + decision: RouteDecision::Unsupported { + reason: "recipe_unproven: targeted key chords are implemented but no exact target-belief recipe is published before manual host QA".to_owned(), + }, + }); + } + } + cells +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use cua_driver_core::api::contracts::KeyStroke; + + use super::*; + + struct FailingPoster { + calls: Mutex>, + fail_at: usize, + } + + impl KeyboardEventPoster for FailingPoster { + fn post( + &self, + _pid: i32, + event: &TargetedKeyEvent, + _boundary: &mut NativeSideEffectBoundary<'_>, + ) -> Result { + let mut calls = self.calls.lock().unwrap(); + let index = calls.len(); + calls.push(event.kind); + if index == self.fail_at { + Err(NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "injected event-post failure", + )) + } else { + Ok(TargetedPostPrimitive::CoreGraphicsPid) + } + } + } + + struct FailingSetPoster { + calls: Mutex>, + fail_at: BTreeSet, + } + + impl KeyboardEventPoster for FailingSetPoster { + fn post( + &self, + _pid: i32, + event: &TargetedKeyEvent, + _boundary: &mut NativeSideEffectBoundary<'_>, + ) -> Result { + let mut calls = self.calls.lock().unwrap(); + let index = calls.len(); + calls.push(event.kind); + if self.fail_at.contains(&index) { + Err(NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "injected event-post failure", + )) + } else { + Ok(TargetedPostPrimitive::CoreGraphicsPid) + } + } + } + + #[test] + fn every_partial_chord_failure_attempts_all_possible_modifier_releases() { + let chord = normalize_chord(&KeyStroke { + key: "k".to_owned(), + modifiers: vec![Modifier::Control, Modifier::Alt, Modifier::Meta], + }) + .unwrap(); + let pre_release_events = chord.modifiers.len() + 2; + for fail_at in 0..pre_release_events { + let poster = FailingPoster { + calls: Mutex::new(Vec::new()), + fail_at, + }; + let mut observations = cua_driver_core::api::observation::ObservationStore::default(); + let mut settlement = cua_driver_core::api::settlement::SettlementState::default(); + let mut boundary = NativeSideEffectBoundary::new( + &mut observations, + &mut settlement, + cua_driver_core::api::contracts::ObservationId::parse("unused-observation") + .unwrap(), + cua_driver_core::api::contracts::ActionId::parse("unused-action").unwrap(), + cua_driver_core::api::settlement::SettlementProfile::dispatch_only("test"), + ); + let error = + dispatch_chord(&poster, Duration::ZERO, 42, &chord, &mut boundary).unwrap_err(); + assert_eq!(error.code, ErrorCode::DispatchFailed); + let calls = poster.calls.lock().unwrap(); + let attempted_modifiers: Vec<_> = calls[..=fail_at] + .iter() + .filter_map(|kind| match kind { + TargetedKeyEventKind::ModifierDown(modifier) => Some(*modifier), + _ => None, + }) + .collect(); + let releases: Vec<_> = calls[fail_at + 1..] + .iter() + .filter_map(|kind| match kind { + TargetedKeyEventKind::ModifierUp(modifier) => Some(*modifier), + _ => None, + }) + .collect(); + assert_eq!( + releases, + attempted_modifiers.into_iter().rev().collect::>() + ); + } + + let poster = FailingPoster { + calls: Mutex::new(Vec::new()), + fail_at: pre_release_events, + }; + let mut observations = cua_driver_core::api::observation::ObservationStore::default(); + let mut settlement = cua_driver_core::api::settlement::SettlementState::default(); + let mut boundary = NativeSideEffectBoundary::new( + &mut observations, + &mut settlement, + cua_driver_core::api::contracts::ObservationId::parse("unused-observation").unwrap(), + cua_driver_core::api::contracts::ActionId::parse("unused-action").unwrap(), + cua_driver_core::api::settlement::SettlementProfile::dispatch_only("test"), + ); + let error = dispatch_chord(&poster, Duration::ZERO, 42, &chord, &mut boundary).unwrap_err(); + assert_eq!(error.code, ErrorCode::DispatchFailed); + let calls = poster.calls.lock().unwrap(); + assert_eq!(calls.len(), pre_release_events + chord.modifiers.len()); + assert_eq!( + calls + .iter() + .filter(|kind| matches!(kind, TargetedKeyEventKind::ModifierUp(_))) + .count(), + chord.modifiers.len() + ); + assert!(error.target_invalidated()); + } + + #[test] + fn unicode_pair_always_attempts_up_and_poison_depends_on_up_cleanup() { + for (fail_at, poisoned) in [(0, false), (1, true)] { + let poster = FailingPoster { + calls: Mutex::new(Vec::new()), + fail_at, + }; + let mut observations = cua_driver_core::api::observation::ObservationStore::default(); + let mut settlement = cua_driver_core::api::settlement::SettlementState::default(); + let mut boundary = NativeSideEffectBoundary::new( + &mut observations, + &mut settlement, + cua_driver_core::api::contracts::ObservationId::parse("unused-observation") + .unwrap(), + cua_driver_core::api::contracts::ActionId::parse("unused-action").unwrap(), + cua_driver_core::api::settlement::SettlementProfile::dispatch_only("test"), + ); + let error = dispatch_unicode( + &poster, + Duration::ZERO, + 42, + "x", + Instant::now() + Duration::from_secs(1), + &mut boundary, + ) + .unwrap_err(); + assert_eq!( + *poster.calls.lock().unwrap(), + vec![ + TargetedKeyEventKind::UnicodeDown, + TargetedKeyEventKind::UnicodeUp, + ] + ); + assert_eq!(error.target_invalidated(), poisoned); + } + } + + #[test] + fn failed_main_key_up_is_retried_before_modifiers_and_poisoned_if_unproved() { + let chord = normalize_chord(&KeyStroke { + key: "k".to_owned(), + modifiers: vec![Modifier::Control], + }) + .unwrap(); + for (fail_at, poisoned) in [([2].as_slice(), false), ([2, 3].as_slice(), true)] { + let poster = FailingSetPoster { + calls: Mutex::new(Vec::new()), + fail_at: fail_at.iter().copied().collect(), + }; + let mut observations = cua_driver_core::api::observation::ObservationStore::default(); + let mut settlement = cua_driver_core::api::settlement::SettlementState::default(); + let mut boundary = NativeSideEffectBoundary::new( + &mut observations, + &mut settlement, + cua_driver_core::api::contracts::ObservationId::parse("unused-observation") + .unwrap(), + cua_driver_core::api::contracts::ActionId::parse("unused-action").unwrap(), + cua_driver_core::api::settlement::SettlementProfile::dispatch_only("test"), + ); + let error = + dispatch_chord(&poster, Duration::ZERO, 42, &chord, &mut boundary).unwrap_err(); + assert_eq!( + *poster.calls.lock().unwrap(), + vec![ + TargetedKeyEventKind::ModifierDown(Modifier::Control), + TargetedKeyEventKind::KeyDown, + TargetedKeyEventKind::KeyUp, + TargetedKeyEventKind::KeyUp, + TargetedKeyEventKind::ModifierUp(Modifier::Control), + ] + ); + assert_eq!(error.target_invalidated(), poisoned); + } + } + + #[test] + fn unicode_deadline_is_checked_between_atomic_pairs() { + let poster = FailingPoster { + calls: Mutex::new(Vec::new()), + fail_at: usize::MAX, + }; + let mut observations = cua_driver_core::api::observation::ObservationStore::default(); + let mut settlement = cua_driver_core::api::settlement::SettlementState::default(); + let mut boundary = NativeSideEffectBoundary::new( + &mut observations, + &mut settlement, + cua_driver_core::api::contracts::ObservationId::parse("unused-observation").unwrap(), + cua_driver_core::api::contracts::ActionId::parse("unused-action").unwrap(), + cua_driver_core::api::settlement::SettlementProfile::dispatch_only("test"), + ); + let error = dispatch_unicode( + &poster, + Duration::from_millis(2), + 42, + "xy", + Instant::now() + Duration::from_millis(1), + &mut boundary, + ) + .unwrap_err(); + assert_eq!(error.details["completed_pairs"], 1); + assert_eq!( + *poster.calls.lock().unwrap(), + vec![ + TargetedKeyEventKind::UnicodeDown, + TargetedKeyEventKind::UnicodeUp, + ] + ); + } + + #[test] + fn insertion_replaces_only_the_observed_utf16_selection() { + let before = "ab😀cd"; + let inserted = apply_insertion(before, AxCfRange::from_utf16(2, 2).unwrap(), "Z").unwrap(); + assert_eq!(inserted.value, "abZcd"); + assert_eq!(inserted.selection, AxCfRange::from_utf16(3, 0).unwrap()); + + let inserted = apply_insertion(before, AxCfRange::from_utf16(4, 0).unwrap(), "!").unwrap(); + assert_eq!(inserted.value, "ab😀!cd"); + assert_eq!(inserted.selection, AxCfRange::from_utf16(5, 0).unwrap()); + + let error = apply_insertion(before, AxCfRange::from_utf16(3, 0).unwrap(), "x").unwrap_err(); + assert_eq!(error.code, ErrorCode::AxRevisionMismatch); + } + + #[test] + fn changed_focused_element_or_owner_window_refuses_before_dispatch() { + let changed_element = + validate_focus_identity_proof(true, 7, false, Some(7), 7).unwrap_err(); + assert_eq!(changed_element.code, ErrorCode::AxRevisionMismatch); + assert_eq!(changed_element.phase, ErrorPhase::Preflight); + + let changed_owner = validate_focus_identity_proof(true, 8, true, Some(8), 7).unwrap_err(); + assert_eq!(changed_owner.code, ErrorCode::AxRevisionMismatch); + assert_eq!(changed_owner.phase, ErrorPhase::Preflight); + } + + #[test] + fn capabilities_publish_only_semantic_insertion_before_keyboard_host_qa() { + let cells = keyboard_capability_cells("fixture"); + assert!(cells + .iter() + .filter(|cell| matches!(cell.decision, RouteDecision::Supported { .. })) + .all(|cell| { + cell.key.action == ActionKind::TypeText + && matches!( + cell.key.window_state, + WindowStateKind::Visible | WindowStateKind::Occluded + ) + && matches!( + cell.decision, + RouteDecision::Supported { + route: Route::Semantic + } + ) + })); + assert!(cells + .iter() + .filter(|cell| cell.key.action == ActionKind::PressKey) + .all(|cell| matches!(cell.decision, RouteDecision::Unsupported { .. }))); + assert!(cells + .iter() + .filter(|cell| { + cell.key.action == ActionKind::TypeText && cell.key.framework == Framework::Unknown + }) + .all(|cell| matches!(cell.decision, RouteDecision::Unsupported { .. }))); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/mod.rs index 2460fbbd9a..b6739a31fe 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/mod.rs @@ -1,8 +1,13 @@ //! Exact, background-safe macOS semantic action routes. +mod keyboard; +mod pointer; mod scroll; mod semantic; +pub(crate) use keyboard::keyboard_capability_cells; +pub use keyboard::MacKeyboardActions; +pub use pointer::MacPointerActions; pub use semantic::MacSemanticActions; use cua_driver_core::api::{ @@ -75,6 +80,64 @@ pub(crate) fn semantic_capability_cells(os_version: &str) -> Vec cells } +pub(crate) fn pointer_capability_cells(os_version: &str) -> Vec { + let frameworks = [ + Framework::Unknown, + Framework::AppKit, + Framework::Chromium, + Framework::Electron, + Framework::WebKit, + Framework::Catalyst, + ]; + let actions = [ActionKind::Click, ActionKind::Drag, ActionKind::Scroll]; + let states = [ + WindowStateKind::Visible, + WindowStateKind::Occluded, + WindowStateKind::Minimized, + WindowStateKind::OffSpace, + WindowStateKind::Unknown, + ]; + let architecture = match std::env::consts::ARCH { + "aarch64" => "arm64", + value => value, + }; + let mut cells = Vec::with_capacity(frameworks.len() * actions.len() * states.len()); + for framework in frameworks { + for action in &actions { + for state in &states { + let proven_click = os_version == "26.5.1" + && architecture == "arm64" + && framework == Framework::Chromium + && *action == ActionKind::Click + && *state == WindowStateKind::Visible; + let decision = if proven_click { + RouteDecision::Supported { + route: Route::TargetedPointer, + } + } else { + RouteDecision::Unsupported { + reason: format!( + "recipe_unproven: targeted pointer {action:?} is not manually proved for macOS {os_version} {architecture} {framework:?} {state:?}" + ), + } + }; + cells.push(CapabilityCell { + key: CapabilityKey { + platform: PlatformName::Macos, + os_version: os_version.to_owned(), + action: action.clone(), + addressing: AddressingMode::CapturedPoint, + framework: framework.clone(), + window_state: state.clone(), + }, + decision, + }); + } + } + } + cells +} + #[cfg(test)] mod tests { use super::*; @@ -109,4 +172,41 @@ mod tests { Framework::Electron | Framework::Catalyst ))); } + + #[test] + fn targeted_pointer_cells_publish_only_the_proven_chromium_click_posture() { + let cells = pointer_capability_cells("26.5.1"); + assert_eq!(cells.len(), 90); + let supported: Vec<_> = cells + .iter() + .filter(|cell| matches!(cell.decision, RouteDecision::Supported { .. })) + .collect(); + if std::env::consts::ARCH == "aarch64" { + assert_eq!(supported.len(), 1); + assert!(supported.iter().all(|cell| { + cell.key.action == ActionKind::Click + && cell.key.addressing == AddressingMode::CapturedPoint + && cell.key.framework == Framework::Chromium + && cell.key.window_state == WindowStateKind::Visible + && matches!( + cell.decision, + RouteDecision::Supported { + route: Route::TargetedPointer + } + ) + })); + assert!(cells.iter().any(|cell| { + cell.key.action == ActionKind::Click + && cell.key.framework == Framework::Chromium + && cell.key.window_state == WindowStateKind::Occluded + && matches!(cell.decision, RouteDecision::Unsupported { .. }) + })); + } else { + assert!(supported.is_empty()); + } + assert!(cells + .iter() + .filter(|cell| matches!(cell.key.action, ActionKind::Drag | ActionKind::Scroll)) + .all(|cell| matches!(cell.decision, RouteDecision::Unsupported { .. }))); + } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/pointer.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/pointer.rs new file mode 100644 index 0000000000..d545c2d356 --- /dev/null +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/pointer.rs @@ -0,0 +1,1649 @@ +//! Exact window-stamped targeted pointer sequences for the v2 driver. + +use std::{ + sync::{ + atomic::{AtomicI64, Ordering}, + Arc, + }, + thread, + time::{Duration, Instant}, +}; + +use async_trait::async_trait; +use core_graphics::{ + event::{CGEvent, CGEventFlags, CGEventType, CGMouseButton, EventField, ScrollEventUnit}, + event_source::{CGEventSource, CGEventSourceStateID}, + geometry::CGPoint, +}; +use cua_driver_core::api::{ + contracts::{Modifier, MouseButton, Point, VerificationLevel}, + errors::{ErrorCode, ErrorPhase, NativeError}, + interaction::{InteractionScope, NativeEvidence, NativeSideEffectBoundary, TargetCursorHandle}, + observation::{ResolvedPoint, ResolvedWindowStamp, SurfaceOwner}, + platform::{NativeDispatch, PointerActionProvider, ResolvedAction}, + settlement::SettlementSignal, + Route, +}; +use foreign_types::ForeignType; +use serde_json::Value; + +use crate::{ + driver::{ + settlement::MacSignalJournal, + target::MacTargetState, + windows::{MacRelatedWindowFacts, MacWindowFacts, MacWindowRegistry}, + }, + input::skylight, +}; + +const CLICK_PRIMER_MOVE_MS: u64 = 15; +const CLICK_PRIMER_UP_MS: u64 = 16; +const CLICK_TARGET_START_MS: u64 = 116; +const CLICK_UP_DELAY_MS: u64 = 1; +const CLICK_PAIR_GAP_MS: u64 = 80; +const MAX_DRAG_STEPS: usize = 60; +const MIN_DRAG_STEPS: usize = 2; +const DRAG_POINTS_PER_STEP: f64 = 16.0; +const FIXED_POINT_SCALE: f64 = 65_536.0; +const POINT_EPSILON: f64 = 0.001; + +#[derive(Clone)] +pub struct MacPointerActions { + windows: MacWindowRegistry, + sink: Arc, +} + +impl MacPointerActions { + pub fn new(windows: MacWindowRegistry) -> Self { + Self { + windows, + sink: Arc::new(SystemTargetedPointerSink), + } + } + + async fn prepare_point( + &self, + target_stamp: ResolvedWindowStamp, + scope_owner: ResolvedWindowStamp, + point: &ResolvedPoint, + ) -> Result { + ensure_point_matches_target(&target_stamp, &scope_owner, point)?; + let (pid, cg_window_id, owner_bounds) = match &point.surface_owner { + SurfaceOwner::Target(owner) => { + if owner != &target_stamp { + return Err(stale_surface( + point, + "captured target surface owner no longer matches the target controller", + )); + } + let facts = self.windows.facts_for_stamp(owner).await?; + target_point_facts(point, facts)? + } + SurfaceOwner::RelatedTransient { owner, parent } => { + if parent != &target_stamp { + return Err(stale_surface( + point, + "captured transient parent no longer matches the target controller", + )); + } + let facts = self.windows.facts_for_related_stamp(owner, parent).await?; + related_point_facts(point, facts)? + } + }; + let window_local = Point { + x: point.screen_point.x - owner_bounds.x, + y: point.screen_point.y - owner_bounds.y, + }; + if window_local.x < 0.0 + || window_local.y < 0.0 + || window_local.x >= owner_bounds.width + || window_local.y >= owner_bounds.height + { + return Err(stale_surface( + point, + "captured point lies outside its live native surface owner", + )); + } + let derived_screen = Point { + x: owner_bounds.x + window_local.x, + y: owner_bounds.y + window_local.y, + }; + if !same_point(derived_screen, point.screen_point) { + return Err(stale_surface( + point, + "captured point transform no longer reproduces its live screen point", + )); + } + let observation_epoch = point.observation_epoch.ok_or_else(|| { + stale_surface( + point, + "captured point has no exact native observation journal epoch", + ) + })?; + Ok(NativePoint { + screen: point.screen_point, + window_local, + logical: point.window_point, + pid, + cg_window_id, + owner_bounds, + owner: surface_owner_stamp(&point.surface_owner).clone(), + surface_id: point.surface_id.to_string(), + capture_revision: point.capture_revision.to_string(), + geometry_revision: point.geometry_revision.to_string(), + observation_epoch: observation_epoch.0, + }) + } + + async fn prepare_action( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + action: &ResolvedAction, + ) -> Result { + if target.invalidated() || scope.route != Route::TargetedPointer { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "targeted pointer prepare requires a live targeted-pointer scope", + )); + } + if !skylight::is_targeted_pointer_route_available() { + return Err(NativeError::unsupported( + "recipe_unproven: complete SkyLight targeted-pointer post/window-stamp symbols are unavailable", + ) + .with_detail("recipe_status", "recipe_unproven")); + } + + let (sequence, final_cursor, route_name) = match action { + ResolvedAction::PointClick { point, spec } => { + let point = self + .prepare_point(target.window.clone(), scope.owner.clone(), point) + .await?; + let sequence = build_gesture( + point.clone(), + None, + spec.button, + spec.click_count, + &spec.modifiers, + 0, + )?; + (sequence, point.logical, "macos_targeted_pointer_click") + } + ResolvedAction::Drag(drag) => { + let start = self + .prepare_point(target.window.clone(), scope.owner.clone(), &drag.start) + .await?; + let end = self + .prepare_point(target.window.clone(), scope.owner.clone(), &drag.end) + .await?; + ensure_same_surface(&start, &end)?; + if same_point(start.screen, end.screen) { + return Err(NativeError::invalid( + "drag endpoints must differ so the native sequence contains real motion", + )); + } + let sequence = build_gesture( + start, + Some(end.clone()), + drag.button, + 1, + &drag.modifiers, + drag.duration_ms, + )?; + (sequence, end.logical, "macos_targeted_pointer_drag") + } + ResolvedAction::DeltaScroll(scroll) => { + let point = self + .prepare_point(target.window.clone(), scope.owner.clone(), &scroll.point) + .await?; + let sequence = build_scroll(point.clone(), scroll.delta_x, scroll.delta_y)?; + (sequence, point.logical, "macos_targeted_pointer_scroll") + } + _ => { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "non-pointer action reached macOS pointer prepare", + )) + } + }; + ensure_sequence_fits_deadline(&sequence, scope.deadline.work)?; + Ok(MacPreparedPointerAction { + sequence, + final_cursor, + route_name, + }) + } + + async fn dispatch_action( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + boundary: &mut NativeSideEffectBoundary<'_>, + action: MacPreparedPointerAction, + ) -> Result { + if target.invalidated() || scope.owner != target.window { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "targeted pointer dispatch target changed after prepare", + )); + } + self.revalidate_sequence_target( + target.window.clone(), + scope.owner.clone(), + scope.window.stamp(), + &action.sequence, + ) + .await?; + if target.invalidated() + || scope.owner != target.window + || target.signals.epoch() != action.sequence.target.observation_epoch + { + return Err(NativeError::stale( + ErrorCode::ObservationRaced, + "pointer target or observation epoch changed during final live-facts validation", + )); + } + let epoch_witness = PointerEpochWitness { + journal: target.signals.clone(), + expected: action.sequence.target.observation_epoch, + }; + let mut evidence = PointerDispatchEvidence::new(&action); + let result = dispatch_sequence( + self.sink.as_ref(), + &action.sequence, + scope.deadline.work, + &scope.logical_cursor, + &mut evidence, + boundary, + &epoch_witness, + ); + evidence.merge_into(&mut scope.native_evidence); + if let Err(mut error) = result { + if evidence.cleanup_attempted && !evidence.cleanup_succeeded { + target.invalidate(); + scope.invalidate_target(); + } + error.details.insert( + "pointer_dispatch".to_owned(), + serde_json::to_value(&scope.native_evidence) + .expect("typed pointer dispatch evidence must serialize"), + ); + return Err(error); + } + scope.logical_cursor.update(action.final_cursor); + target + .signals + .record(SettlementSignal::PointerSequenceComplete); + + // Both SkyLight and public CGEvent process posting are void APIs. A + // completed call proves only that the complete targeted sequence was + // attempted; it is not a native delivery acknowledgement. + let verification = VerificationLevel::DispatchUnverified; + let mut native = NativeEvidence::default(); + native + .fields + .insert("pointer_route".to_owned(), action.route_name.into()); + native.fields.insert( + "pointer_event_count".to_owned(), + Value::from(action.sequence.events.len()), + ); + native.fields.insert( + "pointer_transport".to_owned(), + "skylight_then_core_graphics_post_to_pid".into(), + ); + native.fields.insert( + "surface_id".to_owned(), + action.sequence.target.surface_id.clone().into(), + ); + native.fields.insert( + "capture_revision".to_owned(), + action.sequence.target.capture_revision.clone().into(), + ); + native.fields.insert( + "geometry_revision".to_owned(), + action.sequence.target.geometry_revision.clone().into(), + ); + native.fields.insert( + "observation_epoch".to_owned(), + action.sequence.target.observation_epoch.into(), + ); + Ok(NativeDispatch { + verification, + evidence: native, + warnings: Vec::new(), + menu: None, + }) + } + + async fn revalidate_sequence_target( + &self, + target_window: ResolvedWindowStamp, + scope_owner: ResolvedWindowStamp, + scope_window: ResolvedWindowStamp, + sequence: &PreparedSequence, + ) -> Result<(), NativeError> { + if scope_owner != target_window + || scope_window != target_window + || sequence.target.geometry_revision != target_window.geometry_revision.as_str() + { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "pointer target/window geometry changed after native action preparation", + )); + } + if !skylight::is_targeted_pointer_route_available() { + return Err(NativeError::unsupported( + "recipe_unproven: targeted pointer symbols disappeared before dispatch", + )); + } + let (pid, cg_window_id, bounds, stamp) = if sequence.target.owner == target_window { + let facts = self.windows.facts_for_stamp(&target_window).await?; + (facts.pid, facts.cg_window_id, facts.bounds, facts.stamp) + } else { + let facts = self + .windows + .facts_for_related_stamp(&sequence.target.owner, &target_window) + .await?; + (facts.pid, facts.cg_window_id, facts.bounds, facts.stamp) + }; + if pid != sequence.target.pid + || cg_window_id != sequence.target.cg_window_id + || bounds != sequence.target.owner_bounds + || stamp != sequence.target.owner + { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "pointer surface owner facts or geometry changed before the first native post", + ) + .with_detail("surface_id", sequence.target.surface_id.clone()) + .with_detail("capture_revision", sequence.target.capture_revision.clone()) + .with_detail( + "geometry_revision", + sequence.target.geometry_revision.clone(), + )); + } + for event in &sequence.events { + if matches!( + event.kind, + PointerEventKind::PrimerDown | PointerEventKind::PrimerUp + ) { + continue; + } + let expected_screen = Point { + x: bounds.x + event.point.window_local.x, + y: bounds.y + event.point.window_local.y, + }; + if event.point.pid != pid + || event.point.cg_window_id != cg_window_id + || event.point.owner != stamp + || event.point.owner_bounds != bounds + || !same_point(expected_screen, event.point.screen) + { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "prepared pointer event no longer matches final live surface geometry", + )); + } + } + Ok(()) + } +} + +#[async_trait] +impl PointerActionProvider for MacPointerActions { + type PreparedAction = MacPreparedPointerAction; + + async fn prepare( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + action: &ResolvedAction, + ) -> Result { + self.prepare_action(target, scope, action).await + } + + async fn dispatch( + &self, + target: &mut MacTargetState, + scope: &mut InteractionScope, + boundary: &mut NativeSideEffectBoundary<'_>, + action: Self::PreparedAction, + ) -> Result { + // The complete sequence is intentionally synchronous. Core cannot + // cancel between a button-down and its paired cleanup button-up. + self.dispatch_action(target, scope, boundary, action).await + } +} + +pub struct MacPreparedPointerAction { + sequence: PreparedSequence, + final_cursor: Point, + route_name: &'static str, +} + +#[derive(Debug, Clone)] +struct PreparedSequence { + target: NativePoint, + events: Vec, +} + +#[derive(Debug, Clone)] +struct NativePoint { + screen: Point, + window_local: Point, + logical: Point, + pid: i32, + cg_window_id: u32, + owner_bounds: cua_driver_core::api::contracts::Rect, + owner: ResolvedWindowStamp, + surface_id: String, + capture_revision: String, + geometry_revision: String, + observation_epoch: u64, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum PointerEventKind { + Moved, + PrimerDown, + PrimerUp, + Down, + Dragged, + Up, + Scroll, +} + +#[derive(Debug, Clone)] +struct PreparedPointerEvent { + kind: PointerEventKind, + point: NativePoint, + button: MouseButton, + click_state: u8, + modifiers: Vec, + scheduled_after: Duration, + native_scroll_x: f64, + native_scroll_y: f64, +} + +fn build_gesture( + start: NativePoint, + drag_to: Option, + button: MouseButton, + click_count: u8, + modifiers: &[Modifier], + duration_ms: u32, +) -> Result { + if !(1..=3).contains(&click_count) { + return Err(NativeError::invalid( + "targeted pointer click_count must be between 1 and 3", + )); + } + if drag_to.is_some() && click_count != 1 { + return Err(NativeError::invalid( + "a targeted drag is one gesture and cannot carry multiple click pairs", + )); + } + let primer = NativePoint { + screen: Point { x: -1.0, y: -1.0 }, + window_local: Point { x: -1.0, y: -1.0 }, + logical: start.logical, + ..start.clone() + }; + let mut events = vec![ + pointer_event( + PointerEventKind::Moved, + start.clone(), + button, + 0, + modifiers, + 0, + ), + pointer_event( + PointerEventKind::PrimerDown, + primer.clone(), + MouseButton::Left, + 1, + &[], + CLICK_PRIMER_MOVE_MS, + ), + pointer_event( + PointerEventKind::PrimerUp, + primer, + MouseButton::Left, + 1, + &[], + CLICK_PRIMER_UP_MS, + ), + ]; + + if let Some(end) = drag_to { + events.push(pointer_event( + PointerEventKind::Down, + start.clone(), + button, + 1, + modifiers, + CLICK_TARGET_START_MS, + )); + let distance = ((end.screen.x - start.screen.x).powi(2) + + (end.screen.y - start.screen.y).powi(2)) + .sqrt(); + let steps = ((distance / DRAG_POINTS_PER_STEP).ceil() as usize) + .clamp(MIN_DRAG_STEPS, MAX_DRAG_STEPS); + for index in 1..=steps { + let fraction = index as f64 / (steps + 1) as f64; + let point = interpolate_point(&start, &end, fraction); + let delay = u64::from(duration_ms) * index as u64 / (steps + 1) as u64; + events.push(pointer_event( + PointerEventKind::Dragged, + point, + button, + 1, + modifiers, + CLICK_TARGET_START_MS + delay, + )); + } + events.push(pointer_event( + PointerEventKind::Up, + end, + button, + 1, + modifiers, + CLICK_TARGET_START_MS + u64::from(duration_ms), + )); + return Ok(PreparedSequence { + target: start, + events, + }); + } + + for pair_index in 0..click_count { + let down_at = + CLICK_TARGET_START_MS + u64::from(pair_index) * (CLICK_UP_DELAY_MS + CLICK_PAIR_GAP_MS); + let click_state = pair_index + 1; + events.push(pointer_event( + PointerEventKind::Down, + start.clone(), + button, + click_state, + modifiers, + down_at, + )); + events.push(pointer_event( + PointerEventKind::Up, + start.clone(), + button, + click_state, + modifiers, + down_at + CLICK_UP_DELAY_MS, + )); + } + Ok(PreparedSequence { + target: start, + events, + }) +} + +fn build_scroll( + point: NativePoint, + public_delta_x: f64, + public_delta_y: f64, +) -> Result { + if !public_delta_x.is_finite() || !public_delta_y.is_finite() { + return Err(NativeError::invalid( + "targeted pointer scroll deltas must be finite", + )); + } + if public_delta_x == 0.0 && public_delta_y == 0.0 { + return Err(NativeError::invalid( + "targeted pointer scroll deltas cannot both be zero", + )); + } + // Core is positive-right/positive-down. Native pixel wheel deltas are the + // opposite: positive reveals left/up content. + let native_scroll_x = exact_fixed_delta(-public_delta_x)?; + let native_scroll_y = exact_fixed_delta(-public_delta_y)?; + let event = PreparedPointerEvent { + kind: PointerEventKind::Scroll, + point: point.clone(), + button: MouseButton::Left, + click_state: 0, + modifiers: Vec::new(), + scheduled_after: Duration::ZERO, + native_scroll_x, + native_scroll_y, + }; + Ok(PreparedSequence { + target: point, + events: vec![event], + }) +} + +fn exact_fixed_delta(value: f64) -> Result { + let scaled = value * FIXED_POINT_SCALE; + if !scaled.is_finite() + || scaled < f64::from(i32::MIN) + || scaled > f64::from(i32::MAX) + || (scaled.round() - scaled).abs() > f64::EPSILON * scaled.abs().max(1.0) * 4.0 + { + return Err(NativeError::unsupported( + "exact macOS pixel scroll route requires a signed 16.16-representable logical delta", + ) + .with_detail("requested_native_delta", value)); + } + Ok(scaled.round() / FIXED_POINT_SCALE) +} + +fn pointer_event( + kind: PointerEventKind, + point: NativePoint, + button: MouseButton, + click_state: u8, + modifiers: &[Modifier], + scheduled_after_ms: u64, +) -> PreparedPointerEvent { + PreparedPointerEvent { + kind, + point, + button, + click_state, + modifiers: modifiers.to_vec(), + scheduled_after: Duration::from_millis(scheduled_after_ms), + native_scroll_x: 0.0, + native_scroll_y: 0.0, + } +} + +fn interpolate_point(start: &NativePoint, end: &NativePoint, fraction: f64) -> NativePoint { + let interpolate = |left: f64, right: f64| left + (right - left) * fraction; + NativePoint { + screen: Point { + x: interpolate(start.screen.x, end.screen.x), + y: interpolate(start.screen.y, end.screen.y), + }, + window_local: Point { + x: interpolate(start.window_local.x, end.window_local.x), + y: interpolate(start.window_local.y, end.window_local.y), + }, + logical: Point { + x: interpolate(start.logical.x, end.logical.x), + y: interpolate(start.logical.y, end.logical.y), + }, + ..start.clone() + } +} + +fn ensure_sequence_fits_deadline( + sequence: &PreparedSequence, + deadline: Instant, +) -> Result<(), NativeError> { + let duration = sequence + .events + .last() + .map_or(Duration::ZERO, |event| event.scheduled_after); + let completes_at = Instant::now().checked_add(duration).ok_or_else(|| { + NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Preflight, + false, + "pointer sequence duration exceeds the platform clock range", + ) + })?; + if completes_at > deadline { + return Err(NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Preflight, + true, + "pointer sequence cannot complete inside the controller-owned work deadline", + ) + .with_detail("sequence_duration_ms", duration.as_millis() as u64)); + } + Ok(()) +} + +fn ensure_point_matches_target( + target: &ResolvedWindowStamp, + scope_owner: &ResolvedWindowStamp, + point: &ResolvedPoint, +) -> Result<(), NativeError> { + if &point.window.stamp() != target + || scope_owner != target + || point.geometry_revision != target.geometry_revision + { + return Err(stale_surface( + point, + "point/window/geometry revision no longer matches the target controller", + )); + } + let expected_screen = Point { + x: point.window.geometry.bounds.x + point.window_point.x, + y: point.window.geometry.bounds.y + point.window_point.y, + }; + if !same_point(expected_screen, point.screen_point) { + return Err(stale_surface( + point, + "captured point no longer matches its observation-owned window transform", + )); + } + Ok(()) +} + +fn target_point_facts( + point: &ResolvedPoint, + facts: MacWindowFacts, +) -> Result<(i32, u32, cua_driver_core::api::contracts::Rect), NativeError> { + if facts.stamp != point.window.stamp() { + return Err(stale_surface( + point, + "live target window facts changed after point observation", + )); + } + Ok((facts.pid, facts.cg_window_id, facts.bounds)) +} + +fn related_point_facts( + point: &ResolvedPoint, + facts: MacRelatedWindowFacts, +) -> Result<(i32, u32, cua_driver_core::api::contracts::Rect), NativeError> { + if &facts.stamp != surface_owner_stamp(&point.surface_owner) { + return Err(stale_surface( + point, + "live transient window facts changed after point observation", + )); + } + Ok((facts.pid, facts.cg_window_id, facts.bounds)) +} + +fn ensure_same_surface(start: &NativePoint, end: &NativePoint) -> Result<(), NativeError> { + if start.pid != end.pid + || start.cg_window_id != end.cg_window_id + || start.owner != end.owner + || start.surface_id != end.surface_id + || start.capture_revision != end.capture_revision + || start.geometry_revision != end.geometry_revision + || start.observation_epoch != end.observation_epoch + { + return Err(NativeError::stale( + ErrorCode::SurfaceStale, + "drag endpoints do not share one current captured surface and native owner", + )); + } + Ok(()) +} + +fn surface_owner_stamp(owner: &SurfaceOwner) -> &ResolvedWindowStamp { + match owner { + SurfaceOwner::Target(owner) | SurfaceOwner::RelatedTransient { owner, .. } => owner, + } +} + +fn stale_surface(point: &ResolvedPoint, message: &'static str) -> NativeError { + NativeError::stale(ErrorCode::SurfaceStale, message) + .with_detail("surface_id", point.surface_id.to_string()) + .with_detail("capture_revision", point.capture_revision.to_string()) + .with_detail("geometry_revision", point.geometry_revision.to_string()) +} + +fn same_point(left: Point, right: Point) -> bool { + (left.x - right.x).abs() <= POINT_EPSILON && (left.y - right.y).abs() <= POINT_EPSILON +} + +trait TargetedPointerSink: Send + Sync { + fn post( + &self, + event: &PreparedPointerEvent, + gesture_id: i64, + boundary: &mut NativeSideEffectBoundary<'_>, + epoch: Option<&PointerEpochWitness>, + ) -> Result<(), NativeError>; +} + +#[derive(Clone)] +struct PointerEpochWitness { + journal: MacSignalJournal, + expected: u64, +} + +impl PointerEpochWitness { + fn commit_first_post( + &self, + post: impl FnOnce() -> Result, + ) -> Result { + match self.journal.commit_if_epoch(self.expected, post)? { + Some(result) => Ok(result), + None => Err(NativeError::stale( + ErrorCode::ObservationRaced, + "native signal journal advanced after the captured-point observation", + ) + .with_detail("observed_epoch", self.expected) + .with_detail("current_epoch", self.journal.epoch())), + } + } +} + +struct SystemTargetedPointerSink; + +impl TargetedPointerSink for SystemTargetedPointerSink { + fn post( + &self, + event: &PreparedPointerEvent, + gesture_id: i64, + boundary: &mut NativeSideEffectBoundary<'_>, + epoch: Option<&PointerEpochWitness>, + ) -> Result<(), NativeError> { + let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { + NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "CGEventSource creation failed for targeted pointer dispatch", + ) + })?; + let native = match event.kind { + PointerEventKind::Scroll => build_native_scroll(source, event)?, + _ => build_native_mouse(source, event)?, + }; + stamp_targeted_event(&native, event, gesture_id)?; + let pointer = native.as_ptr() as *mut std::ffi::c_void; + let mut post = || { + boundary.begin()?; + post_both_targeted_transports( + || skylight::post_to_pid(event.point.pid, pointer, false), + || { + native.post_to_pid(event.point.pid); + Ok(()) + }, + ) + }; + match epoch { + Some(epoch) => epoch.commit_first_post(post), + None => post(), + } + } +} + +fn post_both_targeted_transports( + skylight_post: impl FnOnce() -> bool, + core_graphics_post: impl FnOnce() -> Result<(), NativeError>, +) -> Result<(), NativeError> { + let skylight_succeeded = skylight_post(); + let core_graphics_result = core_graphics_post(); + let core_graphics_succeeded = core_graphics_result.is_ok(); + + let skylight_error = (!skylight_succeeded).then(|| { + NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "SLEventPostToPid became unavailable after pointer preflight", + ) + }); + let mut failures = Vec::new(); + if let Some(error) = skylight_error { + failures.push(error); + } + if let Err(error) = core_graphics_result { + failures.push(error); + } + let Some(error) = NativeError::primary(failures) else { + return Ok(()); + }; + Err(error + .with_detail("skylight_attempted", true) + .with_detail("skylight_succeeded", skylight_succeeded) + .with_detail("core_graphics_attempted", true) + .with_detail("core_graphics_call_completed", core_graphics_succeeded)) +} + +fn build_native_mouse( + source: CGEventSource, + event: &PreparedPointerEvent, +) -> Result { + let (event_type, button) = native_mouse_shape(event.kind, event.button)?; + let native = CGEvent::new_mouse_event( + source, + event_type, + CGPoint::new(event.point.screen.x, event.point.screen.y), + button, + ) + .map_err(|_| { + NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "CGEvent mouse event construction failed", + ) + })?; + let flags = modifier_flags(&event.modifiers); + if flags != CGEventFlags::CGEventFlagNull { + native.set_flags(flags); + } + Ok(native) +} + +fn build_native_scroll( + source: CGEventSource, + event: &PreparedPointerEvent, +) -> Result { + let native_y = event.native_scroll_y.round(); + let native_x = event.native_scroll_x.round(); + if native_y < f64::from(i32::MIN) + || native_y > f64::from(i32::MAX) + || native_x < f64::from(i32::MIN) + || native_x > f64::from(i32::MAX) + { + return Err(NativeError::unsupported( + "macOS pixel scroll delta exceeds the native event field range", + )); + } + let native = CGEvent::new_scroll_event( + source, + ScrollEventUnit::PIXEL, + 2, + native_y as i32, + native_x as i32, + 0, + ) + .map_err(|_| { + NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "CGEvent pixel scroll construction failed", + ) + })?; + native.set_double_value_field( + EventField::SCROLL_WHEEL_EVENT_FIXED_POINT_DELTA_AXIS_1, + event.native_scroll_y, + ); + native.set_double_value_field( + EventField::SCROLL_WHEEL_EVENT_FIXED_POINT_DELTA_AXIS_2, + event.native_scroll_x, + ); + native.set_integer_value_field(EventField::SCROLL_WHEEL_EVENT_IS_CONTINUOUS, 1); + unsafe { + CGEventSetLocation( + native.as_ptr() as *mut std::ffi::c_void, + event.point.screen.x, + event.point.screen.y, + ); + } + Ok(native) +} + +fn stamp_targeted_event( + event: &CGEvent, + spec: &PreparedPointerEvent, + gesture_id: i64, +) -> Result<(), NativeError> { + let pointer = event.as_ptr() as *mut std::ffi::c_void; + if !skylight::set_window_location( + pointer, + spec.point.window_local.x, + spec.point.window_local.y, + ) { + return Err(missing_stamp("CGEventSetWindowLocation")); + } + for (field, value) in [ + (0, phase(spec.kind)), + (1, i64::from(spec.click_state)), + (3, button_number(spec.button)), + (7, 3), + (40, i64::from(spec.point.pid)), + (51, i64::from(spec.point.cg_window_id)), + (58, gesture_id), + (91, i64::from(spec.point.cg_window_id)), + (92, i64::from(spec.point.cg_window_id)), + ] { + if !skylight::set_integer_field(pointer, field, value) { + return Err(missing_stamp("SLEventSetIntegerValueField")); + } + } + Ok(()) +} + +fn native_mouse_shape( + kind: PointerEventKind, + button: MouseButton, +) -> Result<(CGEventType, CGMouseButton), NativeError> { + let native_button = match button { + MouseButton::Left => CGMouseButton::Left, + MouseButton::Right => CGMouseButton::Right, + MouseButton::Middle => CGMouseButton::Center, + }; + let event_type = match (kind, button) { + (PointerEventKind::Moved, _) => CGEventType::MouseMoved, + (PointerEventKind::PrimerDown, _) | (PointerEventKind::Down, MouseButton::Left) => { + CGEventType::LeftMouseDown + } + (PointerEventKind::PrimerUp, _) | (PointerEventKind::Up, MouseButton::Left) => { + CGEventType::LeftMouseUp + } + (PointerEventKind::Down, MouseButton::Right) => CGEventType::RightMouseDown, + (PointerEventKind::Up, MouseButton::Right) => CGEventType::RightMouseUp, + (PointerEventKind::Down, MouseButton::Middle) => CGEventType::OtherMouseDown, + (PointerEventKind::Up, MouseButton::Middle) => CGEventType::OtherMouseUp, + (PointerEventKind::Dragged, MouseButton::Left) => CGEventType::LeftMouseDragged, + (PointerEventKind::Dragged, MouseButton::Right) => CGEventType::RightMouseDragged, + (PointerEventKind::Dragged, MouseButton::Middle) => CGEventType::OtherMouseDragged, + (PointerEventKind::Scroll, _) => { + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Dispatch, + false, + "scroll event reached mouse event construction", + )) + } + }; + Ok((event_type, native_button)) +} + +fn phase(kind: PointerEventKind) -> i64 { + match kind { + PointerEventKind::Moved | PointerEventKind::PrimerUp => 2, + PointerEventKind::PrimerDown => 1, + PointerEventKind::Down + | PointerEventKind::Dragged + | PointerEventKind::Up + | PointerEventKind::Scroll => 3, + } +} + +fn button_number(button: MouseButton) -> i64 { + match button { + MouseButton::Left => 0, + MouseButton::Right => 1, + MouseButton::Middle => 2, + } +} + +fn modifier_flags(modifiers: &[Modifier]) -> CGEventFlags { + modifiers + .iter() + .fold(CGEventFlags::CGEventFlagNull, |flags, modifier| { + flags + | match modifier { + Modifier::Shift => CGEventFlags::CGEventFlagShift, + Modifier::Control => CGEventFlags::CGEventFlagControl, + Modifier::Alt => CGEventFlags::CGEventFlagAlternate, + Modifier::Meta => CGEventFlags::CGEventFlagCommand, + } + }) +} + +fn missing_stamp(symbol: &'static str) -> NativeError { + NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "required targeted pointer event stamp became unavailable after preflight", + ) + .with_detail("symbol", symbol) +} + +#[link(name = "CoreGraphics", kind = "framework")] +extern "C" { + fn CGEventSetLocation(event: *mut std::ffi::c_void, x: f64, y: f64); +} + +fn dispatch_sequence( + sink: &dyn TargetedPointerSink, + sequence: &PreparedSequence, + deadline: Instant, + logical_cursor: &TargetCursorHandle, + evidence: &mut PointerDispatchEvidence, + boundary: &mut NativeSideEffectBoundary<'_>, + epoch: &PointerEpochWitness, +) -> Result<(), NativeError> { + let started = Instant::now(); + let gesture_id = gesture_id(sequence); + let mut pressed: Option<(MouseButton, NativePoint)> = None; + + for event in &sequence.events { + if let Err(error) = wait_until(started + event.scheduled_after, deadline) { + let dispatch_started = boundary.started() || evidence.completed_events > 0; + let cleanup = if dispatch_started { + cleanup_pressed(sink, pressed.take(), gesture_id, boundary) + } else { + CleanupResult::not_needed() + }; + evidence.cleanup_attempted = cleanup.attempted; + evidence.cleanup_succeeded = cleanup.succeeded; + evidence.modifiers_cleared = cleanup.succeeded; + evidence.may_have_partially_landed = dispatch_started; + return Err(error_with_cleanup( + error, + cleanup, + evidence.completed_events, + )); + } + if matches!( + event.kind, + PointerEventKind::PrimerDown | PointerEventKind::Down + ) { + // Posting APIs are void. A returned error may still follow partial + // native delivery, so cleanup must conservatively assume down. + pressed = Some((event.button, event.point.clone())); + } + let first_post_epoch = (evidence.completed_events == 0).then_some(epoch); + if let Err(error) = sink.post(event, gesture_id, boundary, first_post_epoch) { + let dispatch_started = boundary.started() || evidence.completed_events > 0; + let cleanup = if dispatch_started { + cleanup_pressed(sink, pressed.take(), gesture_id, boundary) + } else { + CleanupResult::not_needed() + }; + evidence.cleanup_attempted = cleanup.attempted; + evidence.cleanup_succeeded = cleanup.succeeded; + evidence.modifiers_cleared = cleanup.succeeded; + evidence.may_have_partially_landed = dispatch_started; + return Err(error_with_cleanup( + error, + cleanup, + evidence.completed_events, + )); + } + evidence.completed_events += 1; + evidence.may_have_partially_landed = true; + match event.kind { + PointerEventKind::PrimerDown | PointerEventKind::Down => {} + PointerEventKind::PrimerUp | PointerEventKind::Up => pressed = None, + PointerEventKind::Moved | PointerEventKind::Dragged => { + logical_cursor.update(event.point.logical); + } + PointerEventKind::Scroll => logical_cursor.update(event.point.logical), + } + } + evidence.may_have_partially_landed = false; + evidence.cleanup_succeeded = true; + evidence.modifiers_cleared = true; + Ok(()) +} + +fn wait_until(scheduled: Instant, deadline: Instant) -> Result<(), NativeError> { + let now = Instant::now(); + if now > deadline || scheduled > deadline { + return Err(NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "targeted pointer sequence exceeded the controller-owned work deadline", + )); + } + if scheduled > now { + thread::sleep(scheduled.saturating_duration_since(now)); + } + Ok(()) +} + +#[derive(Debug, Clone, Copy)] +struct CleanupResult { + attempted: bool, + succeeded: bool, +} + +impl CleanupResult { + fn not_needed() -> Self { + Self { + attempted: false, + succeeded: true, + } + } +} + +fn cleanup_pressed( + sink: &dyn TargetedPointerSink, + pressed: Option<(MouseButton, NativePoint)>, + gesture_id: i64, + boundary: &mut NativeSideEffectBoundary<'_>, +) -> CleanupResult { + let Some((button, point)) = pressed else { + return CleanupResult::not_needed(); + }; + let cleanup = pointer_event(PointerEventKind::Up, point, button, 1, &[], 0); + CleanupResult { + attempted: true, + succeeded: sink.post(&cleanup, gesture_id, boundary, None).is_ok(), + } +} + +fn error_with_cleanup( + mut error: NativeError, + cleanup: CleanupResult, + completed_events: usize, +) -> NativeError { + error + .details + .insert("completed_events".to_owned(), completed_events.into()); + error + .details + .insert("cleanup_attempted".to_owned(), cleanup.attempted.into()); + error + .details + .insert("cleanup_succeeded".to_owned(), cleanup.succeeded.into()); + if cleanup.attempted && !cleanup.succeeded { + error.retryable = false; + } + error +} + +fn gesture_id(sequence: &PreparedSequence) -> i64 { + static NEXT_GESTURE_ID: AtomicI64 = AtomicI64::new(1); + let sequence_id = NEXT_GESTURE_ID.fetch_add(1, Ordering::Relaxed); + sequence_id + ^ i64::from(sequence.target.cg_window_id) + ^ i64::from(sequence.target.pid).rotate_left(17) +} + +struct PointerDispatchEvidence { + completed_events: usize, + cleanup_attempted: bool, + cleanup_succeeded: bool, + modifiers_cleared: bool, + may_have_partially_landed: bool, + hardware_cursor_warp_attempted: bool, +} + +impl PointerDispatchEvidence { + fn new(_action: &MacPreparedPointerAction) -> Self { + Self { + completed_events: 0, + cleanup_attempted: false, + cleanup_succeeded: false, + modifiers_cleared: false, + may_have_partially_landed: false, + hardware_cursor_warp_attempted: false, + } + } + + fn merge_into(&self, evidence: &mut NativeEvidence) { + evidence.fields.insert( + "pointer_completed_events".to_owned(), + self.completed_events.into(), + ); + evidence.fields.insert( + "pointer_cleanup_attempted".to_owned(), + self.cleanup_attempted.into(), + ); + evidence.fields.insert( + "pointer_cleanup_succeeded".to_owned(), + self.cleanup_succeeded.into(), + ); + evidence.fields.insert( + "pointer_modifiers_cleared".to_owned(), + self.modifiers_cleared.into(), + ); + evidence.fields.insert( + "pointer_dispatch_may_have_partially_landed".to_owned(), + self.may_have_partially_landed.into(), + ); + evidence.fields.insert( + "hardware_cursor_warp_attempted".to_owned(), + self.hardware_cursor_warp_attempted.into(), + ); + } +} + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use cua_driver_core::api::{ + contracts::{ + AppId, AppRef, CaptureRevision, GeometryRevision, SurfaceId, WindowGeneration, + WindowId, WindowRef, + }, + observation::{NativeProcessHandle, NativeWindowHandle}, + }; + + use super::*; + + fn point(x: f64, y: f64) -> NativePoint { + let app = AppRef { + id: AppId::parse("app").unwrap(), + name: None, + pid: Some(10), + running: true, + }; + let public = WindowRef { + id: WindowId::parse("window").unwrap(), + app, + title: None, + }; + NativePoint { + screen: Point { x, y }, + window_local: Point { x, y }, + logical: Point { x, y }, + pid: 10, + cg_window_id: 20, + owner_bounds: cua_driver_core::api::contracts::Rect { + x: 0.0, + y: 0.0, + width: 1_000.0, + height: 1_000.0, + }, + owner: ResolvedWindowStamp { + app_id: public.app.id, + window_id: public.id, + generation: WindowGeneration(1), + geometry_revision: GeometryRevision::parse("geometry").unwrap(), + native_window: NativeWindowHandle::new("native").unwrap(), + process: NativeProcessHandle::new("process").unwrap(), + }, + surface_id: SurfaceId::parse("surface").unwrap().to_string(), + capture_revision: CaptureRevision::parse("capture").unwrap().to_string(), + geometry_revision: GeometryRevision::parse("geometry").unwrap().to_string(), + observation_epoch: 0, + } + } + + #[test] + fn one_builder_preserves_button_count_modifiers_and_drag_motion() { + for (button, count) in [ + (MouseButton::Left, 1), + (MouseButton::Right, 2), + (MouseButton::Middle, 3), + ] { + let sequence = build_gesture( + point(10.0, 20.0), + None, + button, + count, + &[Modifier::Shift, Modifier::Meta], + 0, + ) + .unwrap(); + let target: Vec<_> = sequence + .events + .iter() + .filter(|event| matches!(event.kind, PointerEventKind::Down | PointerEventKind::Up)) + .collect(); + assert_eq!(target.len(), usize::from(count) * 2); + assert!(target.iter().all(|event| { + event.button == button && event.modifiers == [Modifier::Shift, Modifier::Meta] + })); + assert_eq!( + target + .chunks_exact(2) + .map(|pair| pair[0].click_state) + .collect::>(), + (1..=count).collect::>() + ); + } + + let drag = build_gesture( + point(0.0, 0.0), + Some(point(64.0, 32.0)), + MouseButton::Right, + 1, + &[Modifier::Alt], + 400, + ) + .unwrap(); + assert!(matches!(drag.events[0].kind, PointerEventKind::Moved)); + let dragged: Vec<_> = drag + .events + .iter() + .filter(|event| event.kind == PointerEventKind::Dragged) + .collect(); + assert!(dragged.len() >= 2); + assert!(dragged.windows(2).all(|pair| { + pair[0].scheduled_after <= pair[1].scheduled_after + && pair[0].point.screen.x < pair[1].point.screen.x + })); + assert!(matches!( + drag.events.last().unwrap().kind, + PointerEventKind::Up + )); + } + + #[test] + fn scroll_converts_public_right_down_only_at_native_boundary() { + let sequence = build_scroll(point(5.0, 6.0), 12.5, 7.25).unwrap(); + let event = &sequence.events[0]; + assert_eq!(event.native_scroll_x, -12.5); + assert_eq!(event.native_scroll_y, -7.25); + assert!(build_scroll(point(0.0, 0.0), 0.1, 0.0).is_err()); + } + + #[test] + fn proved_pointer_transport_posts_skylight_then_core_graphics() { + let calls = Mutex::new(Vec::new()); + post_both_targeted_transports( + || { + calls.lock().unwrap().push("skylight"); + true + }, + || { + calls.lock().unwrap().push("core_graphics"); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(*calls.lock().unwrap(), ["skylight", "core_graphics"]); + } + + #[test] + fn failed_skylight_post_still_attempts_core_graphics_and_reports_both() { + let calls = Mutex::new(Vec::new()); + let error = post_both_targeted_transports( + || { + calls.lock().unwrap().push("skylight"); + false + }, + || { + calls.lock().unwrap().push("core_graphics"); + Ok(()) + }, + ) + .unwrap_err(); + + assert_eq!(*calls.lock().unwrap(), ["skylight", "core_graphics"]); + assert_eq!(error.code, ErrorCode::DispatchFailed); + assert_eq!(error.details["skylight_succeeded"], false); + assert_eq!(error.details["core_graphics_attempted"], true); + assert_eq!(error.details["core_graphics_call_completed"], true); + } + + struct FakeSink { + events: Mutex>, + fail_at: usize, + signal_after_first: Option, + } + + impl TargetedPointerSink for FakeSink { + fn post( + &self, + event: &PreparedPointerEvent, + _gesture_id: i64, + _boundary: &mut NativeSideEffectBoundary<'_>, + epoch: Option<&PointerEpochWitness>, + ) -> Result<(), NativeError> { + let post = || { + let mut events = self.events.lock().unwrap(); + let index = events.len(); + events.push(event.clone()); + if index == self.fail_at { + Err(NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "injected pointer post failure", + )) + } else { + Ok(index) + } + }; + let index = if let Some(epoch) = epoch { + epoch.commit_first_post(post)? + } else { + post()? + }; + if index == 0 { + if let Some(journal) = &self.signal_after_first { + journal.record(SettlementSignal::FocusChanged); + } + } + Ok(()) + } + } + + #[test] + fn partial_drag_posts_targeted_button_cleanup_without_cursor_warp() { + let sequence = build_gesture( + point(0.0, 0.0), + Some(point(64.0, 0.0)), + MouseButton::Left, + 1, + &[Modifier::Shift], + 0, + ) + .unwrap(); + let down_index = sequence + .events + .iter() + .position(|event| event.kind == PointerEventKind::Down) + .unwrap(); + let sink = FakeSink { + events: Mutex::new(Vec::new()), + fail_at: down_index + 1, + signal_after_first: None, + }; + let mut evidence = PointerDispatchEvidence { + completed_events: 0, + cleanup_attempted: false, + cleanup_succeeded: false, + modifiers_cleared: false, + may_have_partially_landed: false, + hardware_cursor_warp_attempted: false, + }; + let mut observations = cua_driver_core::api::observation::ObservationStore::default(); + let mut settlement = cua_driver_core::api::settlement::SettlementState::default(); + let mut boundary = NativeSideEffectBoundary::new( + &mut observations, + &mut settlement, + cua_driver_core::api::contracts::ObservationId::parse("unused-observation").unwrap(), + cua_driver_core::api::contracts::ActionId::parse("unused-action").unwrap(), + cua_driver_core::api::settlement::SettlementProfile::dispatch_only("test"), + ); + let journal = MacSignalJournal::default(); + let epoch = PointerEpochWitness { + expected: journal.epoch(), + journal, + }; + let error = dispatch_sequence( + &sink, + &sequence, + Instant::now() + Duration::from_secs(1), + &TargetCursorHandle::default(), + &mut evidence, + &mut boundary, + &epoch, + ) + .unwrap_err(); + assert_eq!(error.code, ErrorCode::DispatchFailed); + assert!(evidence.cleanup_attempted); + assert!(evidence.cleanup_succeeded); + assert!(evidence.modifiers_cleared); + assert!(!evidence.hardware_cursor_warp_attempted); + let events = sink.events.lock().unwrap(); + let cleanup = events.last().unwrap(); + assert_eq!(cleanup.kind, PointerEventKind::Up); + assert!(cleanup.modifiers.is_empty()); + assert_eq!(cleanup.point.pid, sequence.target.pid); + assert_eq!(cleanup.point.cg_window_id, sequence.target.cg_window_id); + } + + #[test] + fn advanced_observation_epoch_refuses_before_the_first_pointer_post() { + let sequence = build_scroll(point(5.0, 6.0), 1.0, 0.0).unwrap(); + let sink = FakeSink { + events: Mutex::new(Vec::new()), + fail_at: usize::MAX, + signal_after_first: None, + }; + let journal = MacSignalJournal::default(); + let epoch = PointerEpochWitness { + expected: journal.epoch(), + journal: journal.clone(), + }; + journal.record(SettlementSignal::FocusChanged); + let mut evidence = PointerDispatchEvidence::new(&MacPreparedPointerAction { + final_cursor: sequence.target.logical, + route_name: "test", + sequence: sequence.clone(), + }); + let mut observations = cua_driver_core::api::observation::ObservationStore::default(); + let mut settlement = cua_driver_core::api::settlement::SettlementState::default(); + let mut boundary = NativeSideEffectBoundary::new( + &mut observations, + &mut settlement, + cua_driver_core::api::contracts::ObservationId::parse("unused-observation").unwrap(), + cua_driver_core::api::contracts::ActionId::parse("unused-action").unwrap(), + cua_driver_core::api::settlement::SettlementProfile::dispatch_only("test"), + ); + let error = dispatch_sequence( + &sink, + &sequence, + Instant::now() + Duration::from_secs(1), + &TargetCursorHandle::default(), + &mut evidence, + &mut boundary, + &epoch, + ) + .unwrap_err(); + assert_eq!(error.code, ErrorCode::ObservationRaced); + assert!(error.retryable); + assert!(!boundary.started()); + assert!(!evidence.cleanup_attempted); + assert!(!evidence.may_have_partially_landed); + assert!(sink.events.lock().unwrap().is_empty()); + } + + #[test] + fn signal_from_first_post_does_not_abort_the_owned_pointer_sequence() { + let sequence = + build_gesture(point(10.0, 20.0), None, MouseButton::Left, 1, &[], 0).unwrap(); + let journal = MacSignalJournal::default(); + let sink = FakeSink { + events: Mutex::new(Vec::new()), + fail_at: usize::MAX, + signal_after_first: Some(journal.clone()), + }; + let epoch = PointerEpochWitness { + expected: journal.epoch(), + journal: journal.clone(), + }; + let mut evidence = PointerDispatchEvidence::new(&MacPreparedPointerAction { + final_cursor: sequence.target.logical, + route_name: "test", + sequence: sequence.clone(), + }); + let mut observations = cua_driver_core::api::observation::ObservationStore::default(); + let mut settlement = cua_driver_core::api::settlement::SettlementState::default(); + let mut boundary = NativeSideEffectBoundary::new( + &mut observations, + &mut settlement, + cua_driver_core::api::contracts::ObservationId::parse("unused-observation").unwrap(), + cua_driver_core::api::contracts::ActionId::parse("unused-action").unwrap(), + cua_driver_core::api::settlement::SettlementProfile::dispatch_only("test"), + ); + + dispatch_sequence( + &sink, + &sequence, + Instant::now() + Duration::from_secs(1), + &TargetCursorHandle::default(), + &mut evidence, + &mut boundary, + &epoch, + ) + .unwrap(); + + assert_eq!(journal.epoch(), epoch.expected + 1); + assert_eq!(sink.events.lock().unwrap().len(), sequence.events.len()); + assert_eq!(evidence.completed_events, sequence.events.len()); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/semantic.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/semantic.rs index 013a66504b..4e11edad01 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/semantic.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/actions/semantic.rs @@ -5,7 +5,7 @@ use core_foundation::base::{CFRelease, CFTypeRef}; use cua_driver_core::api::{ contracts::{MouseButton, Route, ScrollDirection, SelectionType, VerificationLevel}, errors::{ErrorCode, ErrorPhase, NativeError}, - interaction::{InteractionScope, LeaseDecision, NativeEvidence}, + interaction::{InteractionScope, LeaseDecision, NativeEvidence, NativeSideEffectBoundary}, observation::ResolvedElement, platform::{ClickSpec, ElementScrollSpec, NativeDispatch, ResolvedAction, SelectionSpec}, settlement::SettlementSignal, @@ -38,7 +38,10 @@ pub struct MacSemanticActions { /// Retained, side-effect-free semantic recipe produced before core consumes /// the observation. Its internals stay platform-owned and cannot be rebuilt /// from loose metadata at the dispatch boundary. -pub struct MacPreparedSemanticAction(PreparedKind); +pub struct MacPreparedSemanticAction { + kind: PreparedKind, + signal_epoch: u64, +} enum PreparedKind { AxAction { @@ -67,6 +70,25 @@ impl MacSemanticActions { Self { windows } } + async fn element_click_candidate( + &self, + target: &mut MacTargetState, + element: &ResolvedElement, + spec: &ClickSpec, + ) -> Result { + let live = self.refetch_exact(target, element).await?; + let Ok(action) = click_action(&live.snapshot.role, live.snapshot.subrole.as_deref(), spec) + else { + return Ok(false); + }; + Ok(action == AX_PRESS + && live + .snapshot + .actions + .iter() + .any(|candidate| candidate == AX_PRESS)) + } + async fn prepare( &self, target: &mut MacTargetState, @@ -184,7 +206,10 @@ impl MacSemanticActions { )) } }; - Ok(MacPreparedSemanticAction(prepared)) + Ok(MacPreparedSemanticAction { + kind: prepared, + signal_epoch: target.signals.epoch(), + }) } async fn prepare_scroll_route( @@ -236,14 +261,18 @@ impl MacSemanticActions { &self, target: &mut MacTargetState, scope: &mut InteractionScope, + boundary: &mut NativeSideEffectBoundary<'_>, action: MacPreparedSemanticAction, ) -> Result { - match action.0 { + self.revalidate_prepared(target, scope.owner.clone(), scope.window.stamp(), &action) + .await?; + match action.kind { PreparedKind::AxAction { element, action, primitive, } => { + boundary.begin()?; perform_exact(element.as_ptr(), &action)?; target.signals.record(SettlementSignal::AxAction); Ok(dispatch( @@ -285,6 +314,7 @@ impl MacSemanticActions { ); let mut completed_pages = 0_u16; for _ in 0..pages { + boundary.begin()?; let result = unsafe { bindings::perform_action(native, action) }; if result != kAXErrorSuccess { return Err(NativeError::new( @@ -315,6 +345,7 @@ impl MacSemanticActions { )) } PreparedKind::SetValue { element, value } => { + boundary.begin()?; let result = unsafe { bindings::set_string_attr(element.as_ptr(), "AXValue", &value) }; if result != kAXErrorSuccess { @@ -343,6 +374,7 @@ impl MacSemanticActions { range, expected_text, } => { + boundary.begin()?; let result = unsafe { bindings::set_cf_range_attr(element.as_ptr(), "AXSelectedTextRange", range) }; @@ -382,6 +414,49 @@ impl MacSemanticActions { } } + async fn revalidate_prepared( + &self, + target: &mut MacTargetState, + scope_owner: cua_driver_core::api::observation::ResolvedWindowStamp, + scope_window: cua_driver_core::api::observation::ResolvedWindowStamp, + action: &MacPreparedSemanticAction, + ) -> Result<(), NativeError> { + if target.invalidated() || scope_owner != target.window || scope_window != target.window { + return Err(NativeError::stale( + ErrorCode::WindowIdentityChanged, + "semantic target identity changed after native action preparation", + )); + } + let epoch = target.signals.epoch(); + if epoch != action.signal_epoch { + return Err(NativeError::stale( + ErrorCode::ObservationRaced, + "semantic AX/content notification raced the final native validation", + )); + } + let snapshots: Vec<_> = match &action.kind { + PreparedKind::AxAction { element, .. } + | PreparedKind::SetValue { element, .. } + | PreparedKind::SelectText { element, .. } => vec![element.snapshot.clone()], + PreparedKind::PageScroll { route, .. } => match route { + PageRoute::Direct { element, .. } + | PageRoute::ScrollbarPageChild { element, .. } => { + vec![element.snapshot.clone()] + } + }, + }; + for snapshot in snapshots { + self.refetch_registered_exact(target, snapshot).await?; + } + if target.signals.epoch() != epoch { + return Err(NativeError::stale( + ErrorCode::ObservationRaced, + "semantic AX/content notification raced the exact dispatch refetch", + )); + } + Ok(()) + } + async fn refetch_exact( &self, target: &mut MacTargetState, @@ -938,6 +1013,15 @@ fn verification_error(message: impl Into) -> NativeError { impl cua_driver_core::api::platform::SemanticActionProvider for MacSemanticActions { type PreparedAction = MacPreparedSemanticAction; + async fn element_click_candidate( + &self, + target: &mut MacTargetState, + element: &ResolvedElement, + spec: &ClickSpec, + ) -> Result { + MacSemanticActions::element_click_candidate(self, target, element, spec).await + } + async fn prepare( &self, target: &mut MacTargetState, @@ -951,9 +1035,10 @@ impl cua_driver_core::api::platform::SemanticActionProvider for &self, target: &mut MacTargetState, scope: &mut InteractionScope, + boundary: &mut NativeSideEffectBoundary<'_>, action: Self::PreparedAction, ) -> Result { - MacSemanticActions::dispatch(self, target, scope, action).await + MacSemanticActions::dispatch(self, target, scope, boundary, action).await } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/focus.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/focus.rs index 7d2e5c6f61..6cfb14ee73 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/focus.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/focus.rs @@ -16,6 +16,11 @@ use super::target::{MacActiveBelief, MacFocusState}; const PROVEN_OS_VERSION: &str = "26.5.1"; const PROVEN_ARCHITECTURE: &str = "arm64"; +// The live F1b recipe posted KeyFocusReturned followed by NewFront. Route B's +// recovered SLPS record has no subtype field, so those two grants are the same +// make-key byte shape but must still be posted twice, in order. +const PROVEN_CHROMIUM_GRANT_RECORDS: [SlpsMakeKeyState; 2] = + [SlpsMakeKeyState::MakeKey, SlpsMakeKeyState::MakeKey]; #[derive(Debug, Clone, PartialEq, Eq)] pub struct HostRecipeContext { @@ -75,7 +80,7 @@ impl MacScopeRecipe { match self.target_belief { TargetBeliefRecipe::NotApplicable => "semantic_no_target_belief", TargetBeliefRecipe::ChromiumPointClickSlpsMakeKeyHost26_5_1Arm64 => { - "chromium_point_click_slps_make_key_macos_26_5_1_arm64" + "chromium_point_click_two_slps_make_key_records_macos_26_5_1_arm64" } } } @@ -255,12 +260,31 @@ impl TargetBeliefLease { }); } - if let Err(error) = poster.post(pid, window_id, SlpsMakeKeyState::MakeKey) { - state - .lock() - .expect("macOS focus coordinator poisoned") - .active_belief = None; - return Err(error); + let mut grants_posted = 0usize; + for grant in PROVEN_CHROMIUM_GRANT_RECORDS { + if let Err(error) = poster.post(pid, window_id, grant) { + let mut error = error + .with_detail("grant_records_posted", grants_posted) + .with_detail( + "grant_records_expected", + PROVEN_CHROMIUM_GRANT_RECORDS.len(), + ); + let mut rollback_complete = true; + if grants_posted > 0 { + if let Err(cleanup) = poster.post(pid, window_id, SlpsMakeKeyState::RemoveKey) { + rollback_complete = false; + error = error.with_related(&cleanup); + } + } + if rollback_complete { + state + .lock() + .expect("macOS focus coordinator poisoned") + .active_belief = None; + } + return Err(error); + } + grants_posted += 1; } Ok(Self { @@ -324,7 +348,7 @@ mod tests { struct FakePoster { signals: Mutex>, - fail: Option, + fail_at: Vec, } impl TargetBeliefPoster for FakePoster { @@ -334,8 +358,10 @@ mod tests { _window_id: u32, signal: SlpsMakeKeyState, ) -> Result<(), NativeError> { - self.signals.lock().unwrap().push(signal); - if self.fail == Some(signal) { + let mut signals = self.signals.lock().unwrap(); + let index = signals.len(); + signals.push(signal); + if self.fail_at.contains(&index) { Err(NativeError::new( ErrorCode::Internal, ErrorPhase::Preflight, @@ -384,6 +410,9 @@ mod tests { surface_id: SurfaceId::parse("surface").unwrap(), surface_owner: SurfaceOwner::Target(window.stamp()), capture_revision: CaptureRevision::parse("capture").unwrap(), + observation_epoch: Some(cua_driver_core::api::observation::NativeObservationEpoch( + 0, + )), surface_point: Point { x: 1.0, y: 1.0 }, window_point: Point { x: 1.0, y: 1.0 }, screen_point: Point { x: 1.0, y: 1.0 }, @@ -463,7 +492,7 @@ mod tests { let state = Arc::new(Mutex::new(MacFocusState::default())); let poster = Arc::new(FakePoster { signals: Mutex::new(Vec::new()), - fail: None, + fail_at: Vec::new(), }); let mut lease = TargetBeliefLease::acquire_with( ActionId::parse("action").unwrap(), @@ -475,13 +504,17 @@ mod tests { .unwrap(); assert_eq!( poster.signals.lock().unwrap().as_slice(), - [SlpsMakeKeyState::MakeKey] + [SlpsMakeKeyState::MakeKey, SlpsMakeKeyState::MakeKey] ); assert!(state.lock().unwrap().active_belief.is_some()); lease.release().unwrap(); assert_eq!( poster.signals.lock().unwrap().as_slice(), - [SlpsMakeKeyState::MakeKey, SlpsMakeKeyState::RemoveKey] + [ + SlpsMakeKeyState::MakeKey, + SlpsMakeKeyState::MakeKey, + SlpsMakeKeyState::RemoveKey + ] ); assert!(state.lock().unwrap().active_belief.is_none()); } @@ -491,7 +524,7 @@ mod tests { let state = Arc::new(Mutex::new(MacFocusState::default())); let poster = Arc::new(FakePoster { signals: Mutex::new(Vec::new()), - fail: Some(SlpsMakeKeyState::MakeKey), + fail_at: vec![0], }); let error = TargetBeliefLease::acquire_with( ActionId::parse("action").unwrap(), @@ -509,4 +542,61 @@ mod tests { ); assert!(state.lock().unwrap().active_belief.is_none()); } + + #[test] + fn second_grant_failure_posts_paired_revoke_and_clears_ownership() { + let state = Arc::new(Mutex::new(MacFocusState::default())); + let poster = Arc::new(FakePoster { + signals: Mutex::new(Vec::new()), + fail_at: vec![1], + }); + let error = TargetBeliefLease::acquire_with( + ActionId::parse("action").unwrap(), + 44, + 99, + Arc::clone(&state), + poster.clone(), + ) + .err() + .expect("second grant failure must refuse the lease"); + assert_eq!(error.phase, ErrorPhase::Preflight); + assert_eq!(error.details["grant_records_posted"], 1); + assert_eq!( + poster.signals.lock().unwrap().as_slice(), + [ + SlpsMakeKeyState::MakeKey, + SlpsMakeKeyState::MakeKey, + SlpsMakeKeyState::RemoveKey + ] + ); + assert!(state.lock().unwrap().active_belief.is_none()); + } + + #[test] + fn failed_partial_grant_rollback_keeps_ownership_for_poison_and_shutdown_retry() { + let state = Arc::new(Mutex::new(MacFocusState::default())); + let poster = Arc::new(FakePoster { + signals: Mutex::new(Vec::new()), + fail_at: vec![1, 2], + }); + let error = TargetBeliefLease::acquire_with( + ActionId::parse("action").unwrap(), + 44, + 99, + Arc::clone(&state), + poster.clone(), + ) + .err() + .expect("grant and rollback failure must refuse the lease"); + assert_eq!(error.related_failures.len(), 1); + assert_eq!( + poster.signals.lock().unwrap().as_slice(), + [ + SlpsMakeKeyState::MakeKey, + SlpsMakeKeyState::MakeKey, + SlpsMakeKeyState::RemoveKey + ] + ); + assert!(state.lock().unwrap().active_belief.is_some()); + } } diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/interaction.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/interaction.rs index e6555e3777..eb21801dde 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/interaction.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/interaction.rs @@ -25,7 +25,7 @@ use crate::{ apps::nsworkspace::WorkspaceEventHub, ax::enablement::AxEnablementLease, focus_steal::{SuppressionLease, SuppressionOutcome}, - input::slps_make_key, + input::{keyboard::normalize_chord, slps_make_key}, }; use super::{ @@ -39,6 +39,9 @@ use super::{ windows::{MacWindowFacts, MacWindowRegistry}, }; +#[cfg(test)] +use super::target::MacActiveBelief; + const CONTAINMENT_BARRIER_TIMEOUT: Duration = Duration::from_millis(250); #[derive(Debug)] @@ -361,6 +364,11 @@ impl InteractionProvider for MacInter ensure_target_live(target, focus, window)?; let facts = self.windows.facts_for_stamp(&window.stamp()).await?; ensure_native_facts_match(&facts, target, window)?; + if let ResolvedAction::PressKey { stroke, .. } = action { + // Platform key vocabulary is validated before any posture, + // accessibility, containment, or target-belief lease is acquired. + normalize_chord(stroke)?; + } let recipe = select_scope_recipe(&self.host, &window.framework, route, action, &requirements)?; if recipe.target_belief != TargetBeliefRecipe::NotApplicable && !slps_make_key::available() @@ -505,7 +513,9 @@ fn acquire_resources( failures.extend(outcome.failures); let mut combined = NativeError::primary(failures).expect("acquisition failure is nonempty"); if belief_still_active { - combined = combined.with_detail("target_poisoned", true); + combined = combined + .with_detail("target_poisoned", true) + .with_target_invalidated(); } return Err(combined); } @@ -782,6 +792,7 @@ mod tests { log: Arc>>, containment_deadlines: Arc>>, fail_at: Option<&'static str>, + poison_belief: bool, } impl LoggingHooks { @@ -849,8 +860,24 @@ mod tests { _action_id: &ActionId, _pid: i32, _cg_window_id: u32, - _focus_state: Arc>, + focus_state: Arc>, ) -> Result>, NativeError> { + if self.poison_belief { + focus_state + .lock() + .expect("macOS focus coordinator poisoned") + .active_belief = Some(MacActiveBelief { + action_id: ActionId::parse("partial-belief").unwrap(), + pid: 44, + cg_window_id: 99, + }); + return Err(NativeError::new( + ErrorCode::Internal, + ErrorPhase::Preflight, + false, + "injected partial belief rollback failure", + )); + } self.acquire("belief+", "belief-") } } @@ -934,6 +961,7 @@ mod tests { log: Arc::clone(&log), containment_deadlines: Arc::new(Mutex::new(Vec::new())), fail_at: None, + poison_belief: false, }; let poison = Arc::new(AtomicBool::new(false)); let plan = scope_plan(); @@ -1007,6 +1035,7 @@ mod tests { log: Arc::clone(&log), containment_deadlines: Arc::new(Mutex::new(Vec::new())), fail_at: Some(fail_at), + poison_belief: false, }; let error = acquire_resources( &hooks, @@ -1021,6 +1050,36 @@ mod tests { } } + #[test] + fn partial_target_belief_rollback_emits_typed_core_invalidation_signal() { + let log = Arc::new(Mutex::new(Vec::new())); + let hooks = LoggingHooks { + log, + containment_deadlines: Arc::new(Mutex::new(Vec::new())), + fail_at: None, + poison_belief: true, + }; + let poison = Arc::new(AtomicBool::new(false)); + let focus_state = Arc::new(Mutex::new(MacFocusState::default())); + let error = acquire_resources( + &hooks, + &scope_plan(), + Arc::clone(&poison), + Arc::clone(&focus_state), + ) + .err() + .expect("partial belief rollback must refuse acquisition"); + + assert!(error.target_invalidated()); + assert_eq!(error.details["target_poisoned"], true); + assert!(poison.load(Ordering::Acquire)); + assert!(focus_state + .lock() + .expect("macOS focus coordinator poisoned") + .active_belief + .is_some()); + } + #[test] fn cleanup_is_reverse_order_and_restored_violation_poison_is_sticky() { let log = Arc::new(Mutex::new(Vec::new())); diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/mod.rs index 511b18d0f6..2c86ee2c65 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/mod.rs @@ -1,16 +1,10 @@ //! Composition root for the background-only macOS v2 platform driver. -use async_trait::async_trait; use cua_driver_core::api::{ capabilities::CapabilityCell, - contracts::KeyStroke, errors::NativeError, - interaction::InteractionScope, - observation::{ResolvedDrag, ResolvedFocus, ResolvedPoint, ResolvedScroll, ResolvedWindow}, - platform::{ - ClickSpec, KeyboardActionProvider, NativeDispatch, PlatformDriver, PointerActionProvider, - TargetFocusCoordinator, - }, + observation::ResolvedWindow, + platform::{PlatformDriver, TargetFocusCoordinator}, }; pub mod actions; @@ -24,7 +18,10 @@ pub mod settlement; pub mod target; pub mod windows; -use actions::{semantic_capability_cells, MacSemanticActions}; +use actions::{ + keyboard_capability_cells, pointer_capability_cells, semantic_capability_cells, + MacKeyboardActions, MacPointerActions, MacSemanticActions, +}; use interaction::MacInteractionProvider; use lifecycle::MacLifecycle; use observation::MacObservationProvider; @@ -43,7 +40,8 @@ pub struct MacDriver { observation: MacObservationProvider, interaction: MacInteractionProvider, semantic: MacSemanticActions, - unavailable: UnavailableProvider, + pointer: MacPointerActions, + keyboard: MacKeyboardActions, invalidations: MacInvalidationHub, } @@ -55,13 +53,16 @@ impl MacDriver { let observation = MacObservationProvider::new(windows.clone()); let interaction = MacInteractionProvider::new(windows.clone()); let semantic = MacSemanticActions::new(windows.clone()); + let pointer = MacPointerActions::new(windows.clone()); + let keyboard = MacKeyboardActions::new(windows.clone()); Self { lifecycle, windows, observation, interaction, semantic, - unavailable: UnavailableProvider, + pointer, + keyboard, invalidations, } } @@ -73,65 +74,7 @@ impl Default for MacDriver { } } -#[derive(Debug, Clone, Copy, Default)] -pub struct UnavailableProvider; - -fn later_plan(capability: &str) -> NativeError { - NativeError::unsupported(format!( - "macOS v2 {capability} is not implemented by the lifecycle/window provider" - )) -} - -#[async_trait] -impl PointerActionProvider for UnavailableProvider { - async fn click( - &self, - _scope: &mut InteractionScope, - _point: ResolvedPoint, - _click: ClickSpec, - ) -> Result { - Err(later_plan("pointer click")) - } - - async fn drag( - &self, - _scope: &mut InteractionScope, - _drag: ResolvedDrag, - ) -> Result { - Err(later_plan("pointer drag")) - } - - async fn scroll( - &self, - _scope: &mut InteractionScope, - _scroll: ResolvedScroll, - ) -> Result { - Err(later_plan("pointer scroll")) - } -} - -#[async_trait] -impl KeyboardActionProvider for UnavailableProvider { - async fn press_key( - &self, - _scope: &mut InteractionScope, - _focus: &ResolvedFocus, - _stroke: KeyStroke, - ) -> Result { - Err(later_plan("keyboard input")) - } - - async fn type_text( - &self, - _scope: &mut InteractionScope, - _focus: &ResolvedFocus, - _text: &str, - ) -> Result { - Err(later_plan("text input")) - } -} - -#[async_trait] +#[async_trait::async_trait] impl PlatformDriver for MacDriver { type TargetState = MacTargetState; type TargetFocusCoordinator = MacTargetFocusCoordinator; @@ -139,8 +82,8 @@ impl PlatformDriver for MacDriver { type Windows = MacWindowRegistry; type Observation = MacObservationProvider; type Semantic = MacSemanticActions; - type Pointer = UnavailableProvider; - type Keyboard = UnavailableProvider; + type Pointer = MacPointerActions; + type Keyboard = MacKeyboardActions; type Interaction = MacInteractionProvider; type Invalidations = MacInvalidationSubscription; @@ -186,11 +129,11 @@ impl PlatformDriver for MacDriver { } fn pointer(&self) -> &Self::Pointer { - &self.unavailable + &self.pointer } fn keyboard(&self) -> &Self::Keyboard { - &self.unavailable + &self.keyboard } fn interaction(&self) -> &Self::Interaction { @@ -198,7 +141,10 @@ impl PlatformDriver for MacDriver { } fn capability_cells(&self, os_version: &str) -> Vec { - semantic_capability_cells(os_version) + let mut cells = semantic_capability_cells(os_version); + cells.extend(pointer_capability_cells(os_version)); + cells.extend(keyboard_capability_cells(os_version)); + cells } fn subscribe_invalidations(&self) -> Self::Invalidations { diff --git a/libs/cua-driver/rust/crates/platform-macos/src/driver/observation.rs b/libs/cua-driver/rust/crates/platform-macos/src/driver/observation.rs index ca27e2d256..7193882274 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/driver/observation.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/driver/observation.rs @@ -290,6 +290,9 @@ impl MacObservationProvider { }, window_bounds: Some(capture.bounds), capture_revision: CaptureRevision::new(), + observation_epoch: Some(cua_driver_core::api::observation::NativeObservationEpoch( + attempt.epoch, + )), transform: capture.transform, freshness, owner, @@ -807,6 +810,7 @@ struct RawAxNode { value_settable: Option, selected_text_range: Option, selected_text_range_settable: Option, + selected_text_settable: Option, bounds: Option, actions: Vec, actions_proven: bool, @@ -1001,6 +1005,9 @@ unsafe fn walk_ax( let selected_text_range_settable = selected_text_range .as_ref() .and_then(|_| bindings::is_attribute_settable(element, "AXSelectedTextRange").ok()); + let selected_text_settable = selected_text_range + .as_ref() + .and_then(|_| bindings::is_attribute_settable(element, "AXSelectedText").ok()); let bounds = element_screen_rect(element).map(|frame| Rect { x: frame[0], y: frame[1], @@ -1040,6 +1047,7 @@ unsafe fn walk_ax( value_settable, selected_text_range, selected_text_range_settable, + selected_text_settable, bounds, actions, actions_proven, @@ -1090,10 +1098,35 @@ struct RegisteredElement { value_settable: Option, selected_text_range: Option, selected_text_range_settable: Option, + selected_text_settable: Option, actions: Vec, actions_proven: bool, } +impl RegisteredElement { + fn snapshot(&self) -> RegisteredElementSnapshot { + RegisteredElementSnapshot { + element: self.element.clone(), + parent: self.parent.clone(), + owner: self.owner.clone(), + owner_window_id: self.owner_window_id, + role: self.role.clone(), + role_proven: self.role_proven, + subrole: self.subrole.clone(), + orientation: self.orientation.clone(), + value_proof: self.value_proof.clone(), + value_query_proven: self.value_query_proven, + string_value: self.string_value.clone(), + value_settable: self.value_settable, + selected_text_range: self.selected_text_range, + selected_text_range_settable: self.selected_text_range_settable, + selected_text_settable: self.selected_text_settable, + actions: self.actions.clone(), + actions_proven: self.actions_proven, + } + } +} + #[derive(Clone)] pub(crate) struct RegisteredElementSnapshot { pub element: RetainedAxElement, @@ -1110,6 +1143,7 @@ pub(crate) struct RegisteredElementSnapshot { pub value_settable: Option, pub selected_text_range: Option, pub selected_text_range_settable: Option, + pub selected_text_settable: Option, pub actions: Vec, pub actions_proven: bool, } @@ -1214,6 +1248,7 @@ impl MacElementRegistry { value_settable: node.value_settable, selected_text_range: node.selected_text_range, selected_text_range_settable: node.selected_text_range_settable, + selected_text_settable: node.selected_text_settable, actions: node.actions, actions_proven: node.actions_proven, }); @@ -1240,47 +1275,22 @@ impl MacElementRegistry { self.elements .iter() .find(|element| &element.native == native && &element.id == id) - .map(|element| RegisteredElementSnapshot { - element: element.element.clone(), - parent: element.parent.clone(), - owner: element.owner.clone(), - owner_window_id: element.owner_window_id, - role: element.role.clone(), - role_proven: element.role_proven, - subrole: element.subrole.clone(), - orientation: element.orientation.clone(), - value_proof: element.value_proof.clone(), - value_query_proven: element.value_query_proven, - string_value: element.string_value.clone(), - value_settable: element.value_settable, - selected_text_range: element.selected_text_range, - selected_text_range_settable: element.selected_text_range_settable, - actions: element.actions.clone(), - actions_proven: element.actions_proven, - }) + .map(RegisteredElement::snapshot) + } + + pub(crate) fn registered_by_id(&self, id: &ElementId) -> Option { + let mut matches = self.elements.iter().filter(|element| &element.id == id); + let element = matches.next()?; + if matches.next().is_some() { + return None; + } + Some(element.snapshot()) } pub(crate) fn registered_snapshots(&self) -> Vec { self.elements .iter() - .map(|element| RegisteredElementSnapshot { - element: element.element.clone(), - parent: element.parent.clone(), - owner: element.owner.clone(), - owner_window_id: element.owner_window_id, - role: element.role.clone(), - role_proven: element.role_proven, - subrole: element.subrole.clone(), - orientation: element.orientation.clone(), - value_proof: element.value_proof.clone(), - value_query_proven: element.value_query_proven, - string_value: element.string_value.clone(), - value_settable: element.value_settable, - selected_text_range: element.selected_text_range, - selected_text_range_settable: element.selected_text_range_settable, - actions: element.actions.clone(), - actions_proven: element.actions_proven, - }) + .map(RegisteredElement::snapshot) .collect() } } @@ -1572,6 +1582,7 @@ mod tests { value_settable: Some(false), selected_text_range: None, selected_text_range_settable: Some(false), + selected_text_settable: Some(false), bounds: None, actions: vec!["AXPress".to_owned()], actions_proven: true, diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs index 0c465a8004..12251b7fbe 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/input/keyboard.rs @@ -9,15 +9,378 @@ use core_graphics::{ event::{CGEvent, CGEventFlags}, event_source::{CGEventSource, CGEventSourceStateID}, }; +use cua_driver_core::api::{ + contracts::{KeyStroke, Modifier}, + errors::{ErrorCode, ErrorPhase, NativeError}, +}; use foreign_types::ForeignType; +/// A normalized physical modifier key. V2 keeps this typed through cleanup so +/// a partial chord can release every modifier whose down event may have +/// reached the target. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct NormalizedModifier { + pub modifier: Modifier, + pub key_code: u16, + pub flag: CGEventFlags, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct NormalizedChord { + pub key_code: u16, + pub modifiers: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TargetedKeyEventKind { + ModifierDown(Modifier), + KeyDown, + KeyUp, + ModifierUp(Modifier), + UnicodeDown, + UnicodeUp, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct TargetedKeyEvent { + pub kind: TargetedKeyEventKind, + pub key_code: u16, + pub key_down: bool, + pub flags: CGEventFlags, + pub text: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum TargetedPostPrimitive { + SkyLightAuthenticated, + CoreGraphicsPid, +} + +pub(crate) struct PreparedTargetedKeyEvent(CGEvent); + +/// Normalize the complete chord before any interaction lease is acquired. +/// The mapping is the physical US ANSI key vocabulary used by the existing +/// macOS tools. Shifted printable glyphs add Shift explicitly instead of +/// depending on ambient keyboard state. +pub(crate) fn normalize_chord(stroke: &KeyStroke) -> Result { + let (key_code, implied_shift) = normalized_key_code(&stroke.key).ok_or_else(|| { + NativeError::new( + ErrorCode::InvalidRequest, + ErrorPhase::Preflight, + false, + format!("unknown macOS key name: {}", stroke.key), + ) + .with_detail("key", stroke.key.clone()) + })?; + + let mut requested = stroke.modifiers.clone(); + if implied_shift && !requested.contains(&Modifier::Shift) { + requested.push(Modifier::Shift); + } + requested.sort(); + requested.dedup(); + + let modifiers: Vec<_> = requested.into_iter().map(normalized_modifier).collect(); + if modifiers + .iter() + .any(|modifier| modifier.key_code == key_code) + { + return Err(NativeError::new( + ErrorCode::InvalidRequest, + ErrorPhase::Preflight, + false, + "the main key cannot also be present in the chord modifiers", + ) + .with_detail("key", stroke.key.clone())); + } + Ok(NormalizedChord { + key_code, + modifiers, + }) +} + +pub(crate) fn chord_events(chord: &NormalizedChord) -> Vec { + let mut events = Vec::with_capacity(chord.modifiers.len() * 2 + 2); + let mut flags = CGEventFlags::CGEventFlagNull; + for modifier in &chord.modifiers { + flags |= modifier.flag; + events.push(TargetedKeyEvent { + kind: TargetedKeyEventKind::ModifierDown(modifier.modifier), + key_code: modifier.key_code, + key_down: true, + flags, + text: None, + }); + } + events.push(TargetedKeyEvent { + kind: TargetedKeyEventKind::KeyDown, + key_code: chord.key_code, + key_down: true, + flags, + text: None, + }); + events.push(TargetedKeyEvent { + kind: TargetedKeyEventKind::KeyUp, + key_code: chord.key_code, + key_down: false, + flags, + text: None, + }); + for modifier in chord.modifiers.iter().rev() { + flags.remove(modifier.flag); + events.push(TargetedKeyEvent { + kind: TargetedKeyEventKind::ModifierUp(modifier.modifier), + key_code: modifier.key_code, + key_down: false, + flags, + text: None, + }); + } + events +} + +pub(crate) fn unicode_events(text: &str) -> Vec { + text.chars() + .flat_map(|character| { + let text = character.to_string(); + [ + TargetedKeyEvent { + kind: TargetedKeyEventKind::UnicodeDown, + key_code: 0, + key_down: true, + flags: CGEventFlags::CGEventFlagNull, + text: Some(text.clone()), + }, + TargetedKeyEvent { + kind: TargetedKeyEventKind::UnicodeUp, + key_code: 0, + key_down: false, + flags: CGEventFlags::CGEventFlagNull, + text: Some(text), + }, + ] + }) + .collect() +} + +/// Construct the complete native keyboard event without posting it. V2 uses +/// this split so all fallible construction precedes the controller's explicit +/// first-native-side-effect boundary. +pub(crate) fn prepare_targeted_event( + event: &TargetedKeyEvent, +) -> Result { + let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState).map_err(|_| { + NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "CGEventSource creation failed for targeted keyboard dispatch", + ) + })?; + let native = + CGEvent::new_keyboard_event(source, event.key_code, event.key_down).map_err(|_| { + NativeError::new( + ErrorCode::DispatchFailed, + ErrorPhase::Dispatch, + false, + "CGEvent keyboard event creation failed", + ) + })?; + native.set_flags(event.flags); + if let Some(text) = &event.text { + native.set_string(text); + } + Ok(PreparedTargetedKeyEvent(native)) +} + +/// Post an already-constructed event to one pid. No fallible operation occurs +/// before the posting primitive is attempted. +pub(crate) fn post_prepared_targeted_event( + pid: i32, + event: &PreparedTargetedKeyEvent, +) -> TargetedPostPrimitive { + let event_ptr = event.0.as_ptr() as *mut std::ffi::c_void; + if crate::input::skylight::post_to_pid(pid as libc::pid_t, event_ptr, true) { + TargetedPostPrimitive::SkyLightAuthenticated + } else { + event.0.post_to_pid(pid as libc::pid_t); + TargetedPostPrimitive::CoreGraphicsPid + } +} + +fn normalized_modifier(modifier: Modifier) -> NormalizedModifier { + let (key_code, flag) = match modifier { + Modifier::Shift => (56, CGEventFlags::CGEventFlagShift), + Modifier::Control => (59, CGEventFlags::CGEventFlagControl), + Modifier::Alt => (58, CGEventFlags::CGEventFlagAlternate), + Modifier::Meta => (55, CGEventFlags::CGEventFlagCommand), + }; + NormalizedModifier { + modifier, + key_code, + flag, + } +} + +fn normalized_key_code(key: &str) -> Option<(u16, bool)> { + let key = key.trim(); + if key.is_empty() { + return None; + } + if key.eq_ignore_ascii_case("plus") { + return Some((24, true)); + } + let named = match key.to_ascii_lowercase().as_str() { + "return" | "enter" => Some(36), + "tab" => Some(48), + "space" => Some(49), + "delete" | "backspace" => Some(51), + "escape" | "esc" => Some(53), + "command" | "cmd" | "meta" => Some(55), + "shift" => Some(56), + "capslock" | "caps_lock" => Some(57), + "option" | "alt" => Some(58), + "control" | "ctrl" => Some(59), + "fn" => Some(63), + "home" => Some(115), + "pageup" | "page_up" => Some(116), + "del" | "forward_delete" => Some(117), + "end" => Some(119), + "pagedown" | "page_down" => Some(121), + "left" | "left_arrow" => Some(123), + "right" | "right_arrow" => Some(124), + "down" | "down_arrow" => Some(125), + "up" | "up_arrow" => Some(126), + "f1" => Some(122), + "f2" => Some(120), + "f3" => Some(99), + "f4" => Some(118), + "f5" => Some(96), + "f6" => Some(97), + "f7" => Some(98), + "f8" => Some(100), + "f9" => Some(101), + "f10" => Some(109), + "f11" => Some(103), + "f12" => Some(111), + "f13" => Some(105), + "f14" => Some(107), + "f15" => Some(113), + "f16" => Some(106), + "f17" => Some(64), + "f18" => Some(79), + "f19" => Some(80), + "f20" => Some(90), + _ => None, + }; + if let Some(code) = named { + return Some((code, false)); + } + + let mut characters = key.chars(); + let character = characters.next()?; + if characters.next().is_some() { + return None; + } + printable_key_code(character) +} + +fn printable_key_code(character: char) -> Option<(u16, bool)> { + let base = match character.to_ascii_lowercase() { + 'a' => 0, + 's' => 1, + 'd' => 2, + 'f' => 3, + 'h' => 4, + 'g' => 5, + 'z' => 6, + 'x' => 7, + 'c' => 8, + 'v' => 9, + 'b' => 11, + 'q' => 12, + 'w' => 13, + 'e' => 14, + 'r' => 15, + 'y' => 16, + 't' => 17, + '1' => 18, + '2' => 19, + '3' => 20, + '4' => 21, + '6' => 22, + '5' => 23, + '=' | '+' => 24, + '9' => 25, + '7' => 26, + '-' | '_' => 27, + '8' => 28, + '0' => 29, + ']' | '}' => 30, + 'o' => 31, + 'u' => 32, + '[' | '{' => 33, + 'i' => 34, + 'p' => 35, + 'l' => 37, + 'j' => 38, + '\'' | '"' => 39, + 'k' => 40, + ';' | ':' => 41, + '\\' | '|' => 42, + ',' | '<' => 43, + '/' | '?' => 44, + 'n' => 45, + 'm' => 46, + '.' | '>' => 47, + '`' | '~' => 50, + '!' => 18, + '@' => 19, + '#' => 20, + '$' => 21, + '^' => 22, + '%' => 23, + '(' => 25, + '&' => 26, + '*' => 28, + ')' => 29, + ' ' => 49, + _ => return None, + }; + let implied_shift = character.is_ascii_uppercase() + || matches!( + character, + '!' | '@' + | '#' + | '$' + | '%' + | '^' + | '&' + | '*' + | '(' + | ')' + | '_' + | '+' + | '{' + | '}' + | '|' + | ':' + | '"' + | '<' + | '>' + | '?' + | '~' + ); + Some((base, implied_shift)) +} + /// Press and release a single key, delivered to `pid` without stealing focus. pub fn press_key(pid: i32, key: &str, modifiers: &[&str]) -> anyhow::Result<()> { // Handle "+" / "plus" → Shift+= (US keyboard layout). if key == "+" || key.to_lowercase() == "plus" { let flags = modifier_flags(&["shift"]); let eq_code = key_name_to_code("=")?; - post_key(pid, eq_code, true, modifier_flags(modifiers) | flags)?; + post_key(pid, eq_code, true, modifier_flags(modifiers) | flags)?; std::thread::sleep(std::time::Duration::from_millis(8)); post_key(pid, eq_code, false, modifier_flags(modifiers) | flags)?; return Ok(()); @@ -143,7 +506,12 @@ fn post_key(pid: i32, key_code: u16, key_down: bool, flags: CGEventFlags) -> any Ok(()) } -fn post_key_no_auth(pid: i32, key_code: u16, key_down: bool, flags: CGEventFlags) -> anyhow::Result<()> { +fn post_key_no_auth( + pid: i32, + key_code: u16, + key_down: bool, + flags: CGEventFlags, +) -> anyhow::Result<()> { let source = CGEventSource::new(CGEventSourceStateID::HIDSystemState) .map_err(|_| anyhow::anyhow!("CGEventSource::new failed"))?; let event = CGEvent::new_keyboard_event(source, key_code, key_down) @@ -196,16 +564,142 @@ fn key_name_to_code(key: &str) -> anyhow::Result { "right" | "right_arrow" => 124, "down" | "down_arrow" => 125, "up" | "up_arrow" => 126, - "f1" => 122, "f2" => 120, "f3" => 99, "f4" => 118, "f5" => 96, - "f6" => 97, "f7" => 98, "f8" => 100, "f9" => 101, "f10" => 109, - "f11" => 103, "f12" => 111, - "a" => 0, "s" => 1, "d" => 2, "f" => 3, "h" => 4, "g" => 5, "z" => 6, "x" => 7, - "c" => 8, "v" => 9, "b" => 11, "q" => 12, "w" => 13, "e" => 14, "r" => 15, "y" => 16, - "t" => 17, "1" => 18, "2" => 19, "3" => 20, "4" => 21, "6" => 22, "5" => 23, "=" => 24, - "9" => 25, "7" => 26, "-" => 27, "8" => 28, "0" => 29, "]" => 30, "o" => 31, "u" => 32, - "[" => 33, "i" => 34, "p" => 35, "l" => 37, "j" => 38, "'" => 39, "k" => 40, ";" => 41, - "\\" => 42, "," => 43, "/" => 44, "n" => 45, "m" => 46, "." => 47, "`" => 50, + "f1" => 122, + "f2" => 120, + "f3" => 99, + "f4" => 118, + "f5" => 96, + "f6" => 97, + "f7" => 98, + "f8" => 100, + "f9" => 101, + "f10" => 109, + "f11" => 103, + "f12" => 111, + "a" => 0, + "s" => 1, + "d" => 2, + "f" => 3, + "h" => 4, + "g" => 5, + "z" => 6, + "x" => 7, + "c" => 8, + "v" => 9, + "b" => 11, + "q" => 12, + "w" => 13, + "e" => 14, + "r" => 15, + "y" => 16, + "t" => 17, + "1" => 18, + "2" => 19, + "3" => 20, + "4" => 21, + "6" => 22, + "5" => 23, + "=" => 24, + "9" => 25, + "7" => 26, + "-" => 27, + "8" => 28, + "0" => 29, + "]" => 30, + "o" => 31, + "u" => 32, + "[" => 33, + "i" => 34, + "p" => 35, + "l" => 37, + "j" => 38, + "'" => 39, + "k" => 40, + ";" => 41, + "\\" => 42, + "," => 43, + "/" => 44, + "n" => 45, + "m" => 46, + "." => 47, + "`" => 50, _ => anyhow::bail!("Unknown key name: {key}"), }; Ok(code) } + +#[cfg(test)] +mod v2_tests { + use super::*; + + #[test] + fn normalization_canonicalizes_one_chord_and_rejects_unknown_keys() { + let chord = normalize_chord(&KeyStroke { + key: "+".to_owned(), + modifiers: vec![ + Modifier::Meta, + Modifier::Alt, + Modifier::Control, + Modifier::Shift, + Modifier::Meta, + ], + }) + .unwrap(); + assert_eq!(chord.key_code, 24); + assert_eq!( + chord + .modifiers + .iter() + .map(|modifier| modifier.modifier) + .collect::>(), + vec![ + Modifier::Shift, + Modifier::Control, + Modifier::Alt, + Modifier::Meta, + ] + ); + + let events = chord_events(&chord); + assert_eq!( + events.iter().map(|event| event.kind).collect::>(), + vec![ + TargetedKeyEventKind::ModifierDown(Modifier::Shift), + TargetedKeyEventKind::ModifierDown(Modifier::Control), + TargetedKeyEventKind::ModifierDown(Modifier::Alt), + TargetedKeyEventKind::ModifierDown(Modifier::Meta), + TargetedKeyEventKind::KeyDown, + TargetedKeyEventKind::KeyUp, + TargetedKeyEventKind::ModifierUp(Modifier::Meta), + TargetedKeyEventKind::ModifierUp(Modifier::Alt), + TargetedKeyEventKind::ModifierUp(Modifier::Control), + TargetedKeyEventKind::ModifierUp(Modifier::Shift), + ] + ); + + assert_eq!( + normalize_chord(&KeyStroke { + key: "return".to_owned(), + modifiers: vec![], + }) + .unwrap() + .key_code, + 36 + ); + let uppercase = normalize_chord(&KeyStroke { + key: "A".to_owned(), + modifiers: vec![], + }) + .unwrap(); + assert_eq!(uppercase.key_code, 0); + assert_eq!(uppercase.modifiers[0].modifier, Modifier::Shift); + + let error = normalize_chord(&KeyStroke { + key: "definitely-not-a-key".to_owned(), + modifiers: vec![], + }) + .unwrap_err(); + assert_eq!(error.code, ErrorCode::InvalidRequest); + assert_eq!(error.phase, ErrorPhase::Preflight); + } +} diff --git a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs index e6b21d960d..c8fbef1016 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/input/skylight.rs @@ -180,6 +180,13 @@ pub fn is_available() -> bool { post_to_pid_fn().is_some() } +/// `true` only when the complete v2 targeted-pointer post and window-stamp +/// route is present. The v2 driver refuses the capability instead of silently +/// falling back to an unstamped or process-global event path. +pub fn is_targeted_pointer_route_available() -> bool { + post_to_pid_fn().is_some() && set_window_loc_fn().is_some() && set_int_field_fn().is_some() +} + /// `true` when all three focus-without-raise SPIs resolved. pub fn is_focus_without_raise_available() -> bool { get_front_process_fn().is_some() diff --git a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs index 79dd4d9a90..ed714c506b 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/lib.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/lib.rs @@ -45,10 +45,10 @@ pub mod driver; use cua_driver_core::tool::ToolRegistry; -/// Register all macOS tools. For programs that don't restructure `main` -/// (e.g. test harnesses), the overlay is skipped. +/// Register the legacy macOS MCP/v1 tools. The native v2 server never calls +/// this registry. pub fn register_tools() -> ToolRegistry { - register_tools_with_compat(false) + register_legacy_tools_with_compat(false) } /// Same as `register_tools` but lets the caller pick the Claude Code @@ -56,10 +56,16 @@ pub fn register_tools() -> ToolRegistry { /// tool for the window-scoped variant (pid + window_id required, /// JPEG @ 85%, text note pointing at pixel tools). pub fn register_tools_with_compat(compat: bool) -> ToolRegistry { + register_legacy_tools_with_compat(compat) +} + +/// Explicitly named legacy registry constructor used at the production +/// composition root to keep MCP/v1 dispatch separate from native v2. +pub fn register_legacy_tools_with_compat(compat: bool) -> ToolRegistry { #[cfg(target_os = "macos")] { let mut r = ToolRegistry::new(); - tools::register_all(&mut r, compat); + tools::register_all_legacy(&mut r, compat); r } #[cfg(not(target_os = "macos"))] @@ -86,7 +92,7 @@ pub fn register_tools_with_cursor(cfg: cursor_overlay::CursorConfig, compat: boo cursor::overlay::init(cfg); } let mut r = ToolRegistry::new(); - tools::register_all(&mut r, compat); + tools::register_all_legacy(&mut r, compat); r } #[cfg(not(target_os = "macos"))] diff --git a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs index c55ca22f4b..abf9f83198 100644 --- a/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs +++ b/libs/cua-driver/rust/crates/platform-macos/src/tools/mod.rs @@ -366,7 +366,7 @@ impl Default for ToolState { /// regular `screenshot` tool for the Claude Code computer-use compat /// variant — same name, stricter args, window-scoped JPEG @ 85% + a text /// note telling the caller to use pixel-addressed tools. -pub fn register_all(registry: &mut ToolRegistry, compat: bool) { +pub fn register_all_legacy(registry: &mut ToolRegistry, compat: bool) { let state = Arc::new(ToolState::default()); // Share the element cache with the recording-hook layer so it can // resolve element_index → window-local screenshot coords for click.png.