Clean rebase agent rewrite onto master

Cargo.toml

@@ -22,6 +22,7 @@ thiserror = "1.0.63"
 async-trait = "0.1.81"
 jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
 jwt-compact = { version = "0.9.0-beta.1", features = ["es256k"] }
+dspy-rs = { git = "https://github.com/OpenSecretCloud/DSRs.git", branch = "main" }
 diesel = { version = "=2.2.3", features = [
     "postgres",
     "postgres_backend",
@@ -33,6 +34,7 @@ diesel = { version = "=2.2.3", features = [
 ] }
 diesel-derive-enum = { version = "2.1.0", features = ["postgres"] }
 chrono = { version = "0.4.26", features = ["serde"] }
+chrono-tz = "0.10"
 dotenv = "0.15.0"
 aes-gcm = "0.10.1"
 aes-siv = "0.7"
@@ -43,13 +45,14 @@ cbc = "0.1.2"
 secp256k1 = { version = "0.29.0", features = ["rand"] }
 hyper = { version = "0.14", features = ["full"] }
 hyper-tls = "0.5.0"
-reqwest = { version = "0.11", features = ["json"] }
+reqwest = { version = "0.11", features = ["json", "native-tls-alpn"] }
 futures = "0.3.30"
 uuid = { version = "1.10.0", features = ["v4", "serde"] }
 tokio-stream = "0.1"
 async-stream = "0.3"
 bytes = "1.0"
 sha2 = { version = "0.10", default-features = false }
+p256 = { version = "0.13", features = ["ecdh", "pkcs8"] }
 hex = "0.4.3"
 base64 = "0.22.1"
 vsock = "0.5.1"
@@ -75,6 +78,7 @@ lazy_static = "1.4.0"
 subtle = "2.6.1"
 tiktoken-rs = "0.5"
 once_cell = "1.19"
+unicode-segmentation = "1.12"
 
 [dev-dependencies]
 openssl = "0.10.78"

docs/nitro-deploy.md

@@ -586,6 +586,103 @@ A restart of this should not be needed but if you need to:
 sudo systemctl restart vsock-apple-proxy.service
 ```
 
+## Vsock APNs and FCM proxies
+Create vsock proxy services so that enclave program can talk to push providers:
+
+First configure the endpoints into their allowlist:
+
+```
+sudo vim /etc/nitro_enclaves/vsock-proxy.yaml
+```
+
+Add these lines:
+```
+- {address: api.push.apple.com, port: 443}
+- {address: api.sandbox.push.apple.com, port: 443}
+- {address: fcm.googleapis.com, port: 443}
+```
+
+#### APNs Production
+Now create a service that spins this up automatically:
+
+```
+sudo vim /etc/systemd/system/vsock-apns-prod-proxy.service
+```
+
+```
+[Unit]
+Description=Vsock APNs Production Proxy Service
+After=network.target
+
+[Service]
+User=root
+ExecStart=/usr/bin/vsock-proxy 8024 api.push.apple.com 443
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+```
+
+#### APNs Sandbox
+```
+sudo vim /etc/systemd/system/vsock-apns-sandbox-proxy.service
+```
+
+```
+[Unit]
+Description=Vsock APNs Sandbox Proxy Service
+After=network.target
+
+[Service]
+User=root
+ExecStart=/usr/bin/vsock-proxy 8025 api.sandbox.push.apple.com 443
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+```
+
+#### FCM
+```
+sudo vim /etc/systemd/system/vsock-fcm-proxy.service
+```
+
+```
+[Unit]
+Description=Vsock FCM Proxy Service
+After=network.target
+
+[Service]
+User=root
+ExecStart=/usr/bin/vsock-proxy 8029 fcm.googleapis.com 443
+Restart=always
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Activate services:
+
+```
+sudo systemctl daemon-reload
+sudo systemctl enable vsock-apns-prod-proxy.service
+sudo systemctl start vsock-apns-prod-proxy.service
+sudo systemctl status vsock-apns-prod-proxy.service
+sudo systemctl enable vsock-apns-sandbox-proxy.service
+sudo systemctl start vsock-apns-sandbox-proxy.service
+sudo systemctl status vsock-apns-sandbox-proxy.service
+sudo systemctl enable vsock-fcm-proxy.service
+sudo systemctl start vsock-fcm-proxy.service
+sudo systemctl status vsock-fcm-proxy.service
+```
+
+A restart of these should not be needed but if you need to:
+```
+sudo systemctl restart vsock-apns-prod-proxy.service
+sudo systemctl restart vsock-apns-sandbox-proxy.service
+sudo systemctl restart vsock-fcm-proxy.service
+```
+
 ## Vsock Resend proxy
 Create a vsock proxy service so that enclave program can talk to resend:
 

entrypoint.sh

@@ -325,6 +325,12 @@ log "Added Google OAuth domains to /etc/hosts"
 echo "127.0.0.15 appleid.apple.com" >> /etc/hosts
 log "Added Apple OAuth domain to /etc/hosts"
 
+# Add push provider hostnames to /etc/hosts
+echo "127.0.0.21 api.push.apple.com" >> /etc/hosts
+echo "127.0.0.22 api.sandbox.push.apple.com" >> /etc/hosts
+echo "127.0.0.20 fcm.googleapis.com" >> /etc/hosts
+log "Added APNs and FCM domains to /etc/hosts"
+
 # Add AWS SQS hostname to /etc/hosts
 echo "127.0.0.13 sqs.us-east-2.amazonaws.com" >> /etc/hosts
 log "Added AWS SQS domain to /etc/hosts"
@@ -459,6 +465,16 @@ run_forever tf_os_flags python3 /app/traffic_forwarder.py 127.0.0.18 443 3 8028
 log "Starting Apple OAuth traffic forwarder"
 run_forever tf_apple_oauth python3 /app/traffic_forwarder.py 127.0.0.15 443 3 8018 &
 
+# Start the traffic forwarders for push providers in the background
+log "Starting APNs production traffic forwarder"
+run_forever tf_apns_prod python3 /app/traffic_forwarder.py 127.0.0.21 443 3 8024 &
+
+log "Starting APNs sandbox traffic forwarder"
+run_forever tf_apns_sandbox python3 /app/traffic_forwarder.py 127.0.0.22 443 3 8025 &
+
+log "Starting FCM traffic forwarder"
+run_forever tf_fcm python3 /app/traffic_forwarder.py 127.0.0.20 443 3 8029 &
+
 # Start the traffic forwarders for Tinfoil proxy in the background
 log "Starting Tinfoil GitHub proxy traffic forwarder"
 run_forever tf_tinfoil_github_proxy python3 /app/traffic_forwarder.py 127.0.0.16 443 3 8019 &
@@ -657,6 +673,28 @@ else
     log "Apple OAuth connection failed"
 fi
 
+# Test the connections to push providers
+log "Testing connection to APNs production:"
+if timeout 5 bash -c '</dev/tcp/127.0.0.21/443'; then
+    log "APNs production connection successful"
+else
+    log "APNs production connection failed"
+fi
+
+log "Testing connection to APNs sandbox:"
+if timeout 5 bash -c '</dev/tcp/127.0.0.22/443'; then
+    log "APNs sandbox connection successful"
+else
+    log "APNs sandbox connection failed"
+fi
+
+log "Testing connection to FCM:"
+if timeout 5 bash -c '</dev/tcp/127.0.0.20/443'; then
+    log "FCM connection successful"
+else
+    log "FCM connection failed"
+fi
+
 # Test the connections to Tinfoil proxy services
 log "Testing connection to Tinfoil GitHub proxy:"
 if timeout 5 bash -c '</dev/tcp/127.0.0.16/443'; then

flake.nix

@@ -27,6 +27,10 @@
         rustToolchain = builtins.fromTOML (builtins.readFile ./rust-toolchain.toml);
         rustChannel = rustToolchain.toolchain.channel;
         rustAnalyzer = pkgs.rust-bin.stable."${rustChannel}".rust-analyzer;
+        rustPlatform = pkgs.makeRustPlatform {
+          cargo = rust;
+          rustc = rust;
+        };
 
         commonInputs = [
           rust
@@ -249,7 +253,8 @@
           entrypoint = "/bin/entrypoint";
         };
 
-        opensecret = pkgs.rustPlatform.buildRustPackage {
+        # Build the main Rust package
+        opensecret = rustPlatform.buildRustPackage {
           pname = "opensecret";
           version = "0.1.0";
           src = pkgs.lib.cleanSourceWith {
@@ -272,6 +277,11 @@
           };
           cargoLock = {
             lockFile = ./Cargo.lock;
+            outputHashes = {
+              "baml-ids-0.0.1" = "sha256-wjgwOcZvZIWEAjpq0gZwtZccQ+FupudBzpxR4+udKEQ=";
+              "minijinja-2.16.0" = "sha256-55Zo8mVgMhCgYzE6662oTXfn7W908LplZv6ys/aHveY=";
+              "rig-core-0.26.0" = "sha256-/1C2/UTuJrre4IYNW7i1pYaOGy0dxzhLyogR+5NL1nk=";
+            };
           };
           nativeBuildInputs = [
             pkgs.pkg-config

migrations/2026-03-07-120000_push_notifications_v1/down.sql

@@ -3,6 +3,7 @@ DROP TRIGGER IF EXISTS update_push_devices_updated_at ON push_devices;
 
 DROP INDEX IF EXISTS idx_notification_deliveries_pending;
 DROP INDEX IF EXISTS idx_notification_events_user_due;
+DROP INDEX IF EXISTS idx_push_devices_project_active;
 DROP INDEX IF EXISTS idx_push_devices_user_active;
 
 DROP TABLE IF EXISTS notification_deliveries;

migrations/2026-03-07-120000_push_notifications_v1/up.sql

@@ -2,14 +2,15 @@ CREATE TABLE push_devices (
     id BIGSERIAL PRIMARY KEY,
     uuid UUID NOT NULL DEFAULT uuid_generate_v4() UNIQUE,
     user_id UUID NOT NULL REFERENCES users(uuid) ON DELETE CASCADE,
+    project_id INTEGER NOT NULL REFERENCES org_projects(id) ON DELETE CASCADE,
     installation_id UUID NOT NULL,
     platform TEXT NOT NULL CHECK (platform IN ('ios', 'android')),
     provider TEXT NOT NULL CHECK (provider IN ('apns', 'fcm')),
     environment TEXT NOT NULL CHECK (environment IN ('dev', 'prod')),
     app_id TEXT NOT NULL,
-    push_token_enc BYTEA NOT NULL,
     push_token_hash BYTEA NOT NULL,
-    notification_public_key BYTEA NOT NULL,
+    capability_enc BYTEA NOT NULL,
+    notification_public_key_hash BYTEA NOT NULL,
     key_algorithm TEXT NOT NULL CHECK (key_algorithm IN ('p256_ecdh_v1')),
     supports_encrypted_preview BOOLEAN NOT NULL DEFAULT false,
     supports_background_processing BOOLEAN NOT NULL DEFAULT false,
@@ -38,7 +39,10 @@ CREATE TABLE notification_events (
     not_before_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
     expires_at TIMESTAMP WITH TIME ZONE,
     created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
-    cancelled_at TIMESTAMP WITH TIME ZONE
+    cancelled_at TIMESTAMP WITH TIME ZONE,
+    source_kind TEXT NOT NULL DEFAULT 'request_continuation'
+        CHECK (source_kind IN ('request_continuation')),
+    source_request_id UUID
 );
 
 CREATE TABLE notification_deliveries (
@@ -65,6 +69,9 @@ CREATE TABLE notification_deliveries (
 CREATE INDEX idx_push_devices_user_active
     ON push_devices(user_id, revoked_at);
 
+CREATE INDEX idx_push_devices_project_active
+    ON push_devices(project_id, revoked_at);
+
 CREATE UNIQUE INDEX idx_push_devices_installation_active
     ON push_devices(installation_id, environment)
     WHERE revoked_at IS NULL;

migrations/2026-06-26-224528_agent_background_push_aead/down.sql

@@ -0,0 +1,33 @@
+DELETE FROM notification_deliveries;
+DELETE FROM notification_events;
+DELETE FROM agent_schedule_runs;
+DELETE FROM agent_background_grants;
+DELETE FROM agent_schedules;
+
+DROP INDEX IF EXISTS idx_notification_events_background_grant;
+
+ALTER TABLE notification_events
+    DROP COLUMN IF EXISTS background_grant_id;
+
+ALTER TABLE notification_events
+    DROP CONSTRAINT IF EXISTS notification_events_source_kind_check;
+
+ALTER TABLE notification_events
+    ADD CONSTRAINT notification_events_source_kind_check
+    CHECK (source_kind IN ('request_continuation'));
+
+DROP TRIGGER IF EXISTS update_agent_background_grants_updated_at ON agent_background_grants;
+DROP INDEX IF EXISTS idx_agent_background_grants_user;
+DROP INDEX IF EXISTS idx_agent_background_grants_schedule_active;
+DROP TABLE IF EXISTS agent_background_grants;
+
+ALTER TABLE agent_schedules
+    ADD COLUMN description TEXT NOT NULL,
+    DROP COLUMN IF EXISTS description_enc;
+
+ALTER TABLE user_seed_wrappings
+    DROP CONSTRAINT IF EXISTS user_seed_wrappings_credential_kind_check;
+
+ALTER TABLE user_seed_wrappings
+    ADD CONSTRAINT user_seed_wrappings_credential_kind_check
+    CHECK (credential_kind IN ('password', 'oauth'));

migrations/2026-06-26-224528_agent_background_push_aead/up.sql

@@ -0,0 +1,53 @@
+DELETE FROM agent_schedule_runs;
+DELETE FROM agent_schedules;
+
+ALTER TABLE user_seed_wrappings
+    DROP CONSTRAINT IF EXISTS user_seed_wrappings_credential_kind_check;
+
+ALTER TABLE user_seed_wrappings
+    ADD CONSTRAINT user_seed_wrappings_credential_kind_check
+    CHECK (credential_kind IN ('password', 'oauth', 'agent_background'));
+
+ALTER TABLE agent_schedules
+    ADD COLUMN description_enc BYTEA NOT NULL,
+    DROP COLUMN description;
+
+CREATE TABLE agent_background_grants (
+    id BIGSERIAL PRIMARY KEY,
+    uuid UUID NOT NULL DEFAULT gen_random_uuid() UNIQUE,
+    user_id UUID NOT NULL REFERENCES users(uuid) ON DELETE CASCADE,
+    project_id INTEGER NOT NULL REFERENCES org_projects(id) ON DELETE CASCADE,
+    agent_id BIGINT NOT NULL REFERENCES agents(id) ON DELETE CASCADE,
+    schedule_id BIGINT NOT NULL REFERENCES agent_schedules(id) ON DELETE CASCADE,
+    grant_enc BYTEA NOT NULL,
+    seed_wrap_lookup_hash BYTEA NOT NULL,
+    revoked_at TIMESTAMP WITH TIME ZONE,
+    created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    updated_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP
+);
+
+CREATE UNIQUE INDEX idx_agent_background_grants_schedule_active
+    ON agent_background_grants(schedule_id)
+    WHERE revoked_at IS NULL;
+
+CREATE INDEX idx_agent_background_grants_user
+    ON agent_background_grants(user_id, revoked_at);
+
+CREATE TRIGGER update_agent_background_grants_updated_at
+    BEFORE UPDATE ON agent_background_grants
+    FOR EACH ROW
+    EXECUTE FUNCTION update_updated_at_column();
+
+ALTER TABLE notification_events
+    DROP CONSTRAINT IF EXISTS notification_events_source_kind_check;
+
+ALTER TABLE notification_events
+    ADD CONSTRAINT notification_events_source_kind_check
+    CHECK (source_kind IN ('request_continuation', 'agent_background'));
+
+ALTER TABLE notification_events
+    ADD COLUMN background_grant_id BIGINT REFERENCES agent_background_grants(id) ON DELETE SET NULL;
+
+CREATE INDEX idx_notification_events_background_grant
+    ON notification_events(background_grant_id)
+    WHERE background_grant_id IS NOT NULL;

src/aead_db_tamper_tests.rs

@@ -1200,7 +1200,9 @@ fn insert_response_storage_stack_for_user(app_state: &AppState, user_id: Uuid) {
         response_id: Some(response.id),
         user_id,
         content_enc: vec![7, 8, 9],
+        attachment_text_enc: None,
         prompt_tokens: 3,
+        assistant_reaction: None,
     }
     .insert(conn)
     .expect("test user message should insert");
@@ -1215,6 +1217,7 @@ fn insert_response_storage_stack_for_user(app_state: &AppState, user_id: Uuid) {
         status: "completed".to_string(),
         finish_reason: Some("stop".to_string()),
         created_at: Utc::now(),
+        user_reaction: None,
     }
     .insert(conn)
     .expect("test assistant message should insert");