diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ca9a5d3..31ef585 100755 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -11,7 +11,9 @@ "dramatiq-worker", "valkey-exporter", "prometheus", - "grafana" + "grafana", + "loki", + "promtail" ], "features": { "ghcr.io/devcontainers/features/git:1": {}, diff --git a/.vscode/settings.json b/.vscode/settings.json index 8b836ae..d689082 100755 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -24,5 +24,6 @@ "shellscript": true, "python": true }, - "baml.enablePlaygroundProxy": false + "baml.enablePlaygroundProxy": false, + "python.languageServer": "None" } diff --git a/docker-compose.yml b/docker-compose.yml index 5bb894c..6430561 100755 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,8 @@ volumes: db_data_dev: valkey_data_dev: + loki_data_dev: + promtail_data_dev: services: clepsy: @@ -16,8 +18,8 @@ services: VALKEY_URL: redis://valkey:6379/0 MONITORING_ENABLED: "true" CLEPSY_MODE: dev - CLEPSY_GID : "0" - CLEPSY_UID : "0" + CLEPSY_GID: "0" + CLEPSY_UID: "0" volumes: - .:/app @@ -27,10 +29,10 @@ services: - .env ports: - "8000:8000" - depends_on: valkey: condition: service_healthy + restart: no valkey: image: valkey/valkey:9.0.0 @@ -108,3 +110,26 @@ services: prometheus: condition: service_started restart: unless-stopped + + loki: + image: grafana/loki:latest + ports: + - "3100:3100" + command: -config.file=/etc/loki/local-config.yml + volumes: + - ./ops/loki-config.yml:/etc/loki/local-config.yml:ro + - loki_data_dev:/loki + restart: unless-stopped + + promtail: + image: grafana/promtail:latest + volumes: + - /var/lib/docker/containers:/var/lib/docker/containers:ro + - /var/run/docker.sock:/var/run/docker.sock + - ./ops/promtail-config.yml:/etc/promtail/config.yml:ro + - promtail_data_dev:/data + command: -config.file=/etc/promtail/config.yml + depends_on: + loki: + condition: service_started + restart: unless-stopped diff --git a/migrations/00001_first.sql b/migrations/00001_first.sql index e2ea67b..5a96069 100755 --- a/migrations/00001_first.sql +++ b/migrations/00001_first.sql @@ -298,6 +298,25 @@ ON aggregations(start_time, end_time); CREATE UNIQUE INDEX IF NOT EXISTS idx_activity_events_unique ON activity_events(activity_id, event_time, event_type); + +CREATE TABLE scheduled_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + schedule_key TEXT NOT NULL UNIQUE, + job_type TEXT NOT NULL, + cron_expr TEXT, + next_run_at DATETIME NOT NULL, + enabled INTEGER NOT NULL DEFAULT 1 CHECK (enabled IN (0,1)), + payload TEXT, + running_count INTEGER NOT NULL DEFAULT 0 CHECK (running_count >= 0), + max_concurrent INTEGER NOT NULL DEFAULT 1 CHECK (max_concurrent >= 1), + last_started_at DATETIME, + status TEXT NOT NULL DEFAULT 'idle' CHECK (status IN ('idle','error','disabled')), + created_at DATETIME NOT NULL DEFAULT (datetime('now')) +); +CREATE INDEX idx_scheduled_jobs_next_run ON scheduled_jobs(next_run_at); +CREATE INDEX idx_scheduled_jobs_status ON scheduled_jobs(status); + + -- +goose StatementEnd -- +goose Down @@ -372,4 +391,9 @@ DROP TABLE IF EXISTS sessionization_run; DROP INDEX IF EXISTS idx_activity_events_unique; DROP INDEX IF EXISTS idx_aggregations_window_unique; + +DROP TABLE IF EXISTS scheduled_jobs; +DROP index IF EXISTS idx_scheduled_jobs_next_run; +DROP index IF EXISTS idx_scheduled_jobs_status; + -- +goose StatementEnd diff --git a/pyproject.toml b/pyproject.toml index 80c9765..5ae43ce 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,17 +29,17 @@ dependencies = [ "baml-py==0.213.0", "loguru>=0.7.3", "argon2-cffi>=25.1.0", - "sqlalchemy>=2.0.43", "torch>=2.0.0", "gliner>=0.2.22", "valkey[libvalkey]>=6.1.1", "dramatiq[redis,watch]>=1.18.0", - "apscheduler[sqlalchemy]>=4.0.0.0a6", "prometheus-client>=0.20.0", "prometheus-fastapi-instrumentator>=7.1.0", "opencv-contrib-python-headless>=4.12.0.88", "paddleocr>=3.3.1", "paddlepaddle>=3.2.1", + "croniter>=6.0.0", + "aiosqlitepool>=1.0.0", ] [dependency-groups] diff --git a/scripts/worker_entrypoint.sh b/scripts/worker_entrypoint.sh index 0d0fa98..c830705 100755 --- a/scripts/worker_entrypoint.sh +++ b/scripts/worker_entrypoint.sh @@ -11,4 +11,4 @@ PGID="${CLEPSY_GID}" /app/scripts/fix_permissions.sh log "Starting Dramatiq worker..." -exec gosu "${PUID}:${PGID}" bash -lc "mkdir -p /tmp/dramatiq-prometheus && rm -rf /tmp/dramatiq-prometheus/* && uv run dramatiq clepsy.jobs.desktop clepsy.jobs.mobile clepsy.jobs.aggregation clepsy.jobs.sessions clepsy.jobs.goals --processes 2 --threads 4" +exec gosu "${PUID}:${PGID}" bash -lc "mkdir -p /tmp/dramatiq-prometheus && rm -rf /tmp/dramatiq-prometheus/* && uv run dramatiq clepsy.jobs.desktop clepsy.jobs.mobile clepsy.jobs.aggregation clepsy.jobs.sessions clepsy.jobs.goals clepsy.jobs.scheduler_tick clepsy.jobs.scheduled_job_dispatch --processes 4 --threads 4" diff --git a/src/clepsy/aggregator_worker.py b/src/clepsy/aggregator_worker.py index 8c9c5c8..b646656 100755 --- a/src/clepsy/aggregator_worker.py +++ b/src/clepsy/aggregator_worker.py @@ -4,7 +4,8 @@ import asyncio from collections import defaultdict from datetime import datetime -from typing import NamedTuple +from typing import Any, NamedTuple +import sqlite3 import aiosqlite from baml_py import Collector @@ -29,6 +30,7 @@ select_tags, select_user_settings, update_activity, + update_scheduled_job_next_run_at, ) import clepsy.entities as E from clepsy.llm import create_client_registry @@ -37,6 +39,10 @@ ) from clepsy.modules.aggregator.stitching import stitch_timeline +logger = logger.patch( + lambda record: record.update(message=f"[aggregator] {record['message']}") +) + async def llm_productivity_level_activity( activity_name: str, activity_description: str, user_settings: E.UserSettings @@ -483,6 +489,10 @@ async def aggregator_core( text_model_config: E.LLMConfig, collector: Collector, ) -> E.AggregatorCoreOutput: + specific_aggregation_interval_seconds = int( + aggregation_time_span.duration.total_seconds() + ) + ( generated_timeline_activities, generated_timeline_events, @@ -494,7 +504,7 @@ async def aggregator_core( qc_policy=E.IsolatedTimelineQCPolicy.ALWAYS, max_desktop_screenshot_log_interval_seconds=config.max_desktop_screenshot_log_interval_seconds, max_pause_time_seconds=config.max_pause_time_seconds, - aggregation_interval_seconds=config.aggregation_interval_minutes * 60, + aggregation_interval_seconds=specific_aggregation_interval_seconds, ) ( @@ -642,6 +652,7 @@ async def aggregator( async def do_aggregation( input_logs: list[E.AggregationInputEvent], aggregation_time_span: E.TimeSpan, + schedule_update: E.ScheduleUpdate | None = None, ): collector = Collector(name="Aggregation") @@ -683,59 +694,15 @@ async def do_aggregation( ) ) - logger.info( - "[aggregator] Starting DEFERRED transaction for persisting aggregation results" + await persist_aggregation_results( + aggregation_time_span=aggregation_time_span, + aggregation_events_time_span=aggregation_events_time_span, + core_output=core_output, + activity_extras=activity_extras, + schedule_update=schedule_update, ) - async with get_db_connection( - start_transaction=True, commit_on_exit=True, transaction_type="DEFERRED" - ) as conn: - aggregation_id = await insert_aggregation( - aggregation=E.Aggregation( - start_time=aggregation_time_span.start_time, - end_time=aggregation_time_span.end_time, - first_timestamp=aggregation_events_time_span.start_time, - last_timestamp=aggregation_events_time_span.end_time, - ), - conn=conn, - ) - - events_to_insert: list[E.ActivityEventInsert] = [] - for ev in ( - core_output.stitched_activities_events - + core_output.unstitched_activities_close_events - ): - events_to_insert.append( - E.ActivityEventInsert( - activity_id=ev.activity_id, - event_time=ev.event_time, - event_type=ev.event_type, - aggregation_id=aggregation_id, - last_manual_action_time=None, - ) - ) - if core_output.new_activities: - await persist_new_activities( - new_activities=core_output.new_activities, - new_events=core_output.new_activity_events, - conn=conn, - activity_extras=activity_extras, - events_to_insert=events_to_insert, - aggregation_time_span=aggregation_time_span, - aggregation_id=aggregation_id, - ) - elif events_to_insert: - await insert_activity_events(conn, events_to_insert) - - if core_output.activities_to_update: - await asyncio.gather( - *( - update_activity(conn, activity_id, kv_pairs) - for activity_id, kv_pairs in core_output.activities_to_update - ) - ) - - logger.info("[aggregator] DEFERRED transaction committed successfully") + logger.info("Aggregation results persisted") formatted_function_logs = "\n".join( [utils.format_function_log(x) for x in collector.logs] @@ -747,12 +714,94 @@ async def do_aggregation( ) -async def do_empty_aggregation(): +async def persist_aggregation_results( + *, + aggregation_time_span: E.TimeSpan, + aggregation_events_time_span: E.TimeSpan, + core_output: Any, + activity_extras: list[Any], + schedule_update: E.ScheduleUpdate | None, +) -> None: + logger.info("Starting IMMEDIATE transaction for persisting aggregation results") + try: + async with get_db_connection( + start_transaction=True, commit_on_exit=True, transaction_type="IMMEDIATE" + ) as conn: + aggregation_id = await insert_aggregation( + aggregation=E.Aggregation( + start_time=aggregation_time_span.start_time, + end_time=aggregation_time_span.end_time, + first_timestamp=aggregation_events_time_span.start_time, + last_timestamp=aggregation_events_time_span.end_time, + ), + conn=conn, + ) + + events_to_insert: list[E.ActivityEventInsert] = [] + for ev in ( + core_output.stitched_activities_events + + core_output.unstitched_activities_close_events + ): + events_to_insert.append( + E.ActivityEventInsert( + activity_id=ev.activity_id, + event_time=ev.event_time, + event_type=ev.event_type, + aggregation_id=aggregation_id, + last_manual_action_time=None, + ) + ) + + if core_output.new_activities: + await persist_new_activities( + new_activities=core_output.new_activities, + new_events=core_output.new_activity_events, + conn=conn, + activity_extras=activity_extras, + events_to_insert=events_to_insert, + aggregation_time_span=aggregation_time_span, + aggregation_id=aggregation_id, + ) + elif events_to_insert: + await insert_activity_events(conn, events_to_insert) + + if core_output.activities_to_update: + await asyncio.gather( + *( + update_activity(conn, activity_id, kv_pairs) + for activity_id, kv_pairs in core_output.activities_to_update + ) + ) + + if schedule_update: + await update_scheduled_job_next_run_at( + conn, + schedule_id=schedule_update.schedule_id, + next_run_at=schedule_update.next_run_at, + ) + + except (sqlite3.OperationalError, aiosqlite.OperationalError) as exc: + if "database is locked" in str(exc).lower(): + logger.warning("Database locked while persisting aggregation; will retry") + raise + + +async def do_empty_aggregation(schedule_update: E.ScheduleUpdate | None = None): # Open a connection and perform both read and optional write within the same context - async with get_db_connection(start_transaction=False, commit_on_exit=True) as conn: + async with get_db_connection( + start_transaction=False, + commit_on_exit=True, + transaction_type="DEFERRED", + ) as conn: previous_aggregation = await select_latest_aggregation(conn) close_events = await get_interrupted_activity_close_events( conn, previous_aggregation=previous_aggregation ) if close_events: await insert_activity_events(conn, close_events) + if schedule_update: + await update_scheduled_job_next_run_at( + conn, + schedule_id=schedule_update.schedule_id, + next_run_at=schedule_update.next_run_at, + ) diff --git a/src/clepsy/config.py b/src/clepsy/config.py index 8fdce66..e6366eb 100755 --- a/src/clepsy/config.py +++ b/src/clepsy/config.py @@ -130,6 +130,8 @@ class Config(BaseSettings): source_enrollment_code_ttl: timedelta = timedelta(minutes=30) log_level: str | None = None + scheduled_job_timeout: timedelta = timedelta(minutes=15) + max_session_gap: timedelta = timedelta(minutes=10) min_session_length: timedelta = timedelta(minutes=15) min_activities_per_session: int = 3 @@ -141,12 +143,8 @@ class Config(BaseSettings): gliner_pii_threshold: float = 0.5 gliner_cache_dir: Path = cache_dir / "gliner" valkey_url: str - ap_scheduler_sqlite_db_path: Path = Path("/var/lib/clepsy/apscheduler.sqlite3") monitoring_enabled: bool = False - - @property - def ap_scheduler_db_connection_string(self) -> str: - return f"sqlite:////{self.ap_scheduler_sqlite_db_path.as_posix()}" + preserve_dev_streams: bool = False @property def is_dev(self) -> bool: diff --git a/src/clepsy/db/db.py b/src/clepsy/db/db.py index 0040f01..b25146e 100755 --- a/src/clepsy/db/db.py +++ b/src/clepsy/db/db.py @@ -22,7 +22,7 @@ async def get_db_connection( commit_on_exit: bool = True, parse_decltypes: bool = True, busy_timeout: int = 5000, - transaction_type: TransactionTypes = "IMMEDIATE", + transaction_type: TransactionTypes = "DEFERRED", pragma_synchronous: str = "NORMAL", cache_size: int = 2000, ) -> AsyncGenerator[aiosqlite.Connection, None]: diff --git a/src/clepsy/db/queries.py b/src/clepsy/db/queries.py index 0fa38b5..d099921 100755 --- a/src/clepsy/db/queries.py +++ b/src/clepsy/db/queries.py @@ -1,3239 +1,3704 @@ -from __future__ import annotations - -from collections import defaultdict -from datetime import datetime, timedelta -from functools import lru_cache -import json -from typing import Any, List, Optional, Sequence, cast - -import aiosqlite -from loguru import logger - -from clepsy.auth.auth import decrypt_secret -from clepsy.config import config -from clepsy.entities import ( - AADS, - Activity, - ActivityEventInsert, - Aggregation, - AnthropicConfig, - AvgProductivityOperators, - CandidateSession, - CandidateSessionToActivity, - DBActivity, - DBActivityEvent, - DBActivitySpec, - DBActivitySpecWithTags, - DBActivitySpecWithTagsAndSessions, - DBActivityWithLatestEvent, - DBAggregation, - DBAvgProductivityGoal, - DBAvgProductivityGoalDefinition, - DBCandidateSession, - DBCandidateSessionSpec, - DBDeviceSource, - DBGoal, - DBGoalDefinition, - DBGoalResult, - DBSession, - DBSessionizationRun, - DBTag, - DBTotalActivityDurationGoal, - DBTotalActivityDurationGoalDefinition, - EvalState, - GoalMetric, - GoalPeriod, - GoalProgressCurrent, - GoalWithLatestResult, - GoogleAIConfig, - ImageProcessingApproach, - IncludeMode, - MetricOperator, - ModelProvider, - OpenAIConfig, - OpenAIGenericConfig, - ProductivityGoalProgressCurrent, - ProductivityLevel, - Session, - SessionizationRun, - SourceEnrollmentCode, - SourceStatus, - SourceType, - Tag, - TagMapping, - TotalActivityDurationGoalProgressCurrent, - TotalActivityDurationOperators, - UserSettings, -) - - -async def get_filtered_activity_specs_for_goal( - conn: aiosqlite.Connection, - start: datetime, - end: datetime, - include_tag_ids: Optional[List[int]] = None, - exclude_tag_ids: Optional[List[int]] = None, - include_mode: str = "any", - productivity_levels: Optional[List[str]] = None, - day_filter: Optional[List[str]] = None, - time_filter: Optional[List[tuple[str, str]]] = None, -) -> List[DBActivitySpecWithTags]: - """ - Get activity specs with tags, filtered by time, tags, productivity, day, and time filters as much as possible in SQL. - - include_tag_ids: only include activities with these tags (all or any, per include_mode) - - exclude_tag_ids: exclude activities with any of these tags - - productivity_levels: only include activities with these productivity levels - - day_filter: only include activities with events on these weekdays (e.g. ["monday", ...]) - - time_filter: only include activities with events overlapping these time ranges (list of (start, end) in HH:MM:SS) - """ - # Build base query - query = """ - SELECT a.id AS activity_id, a.name, a.description, a.productivity_level, a.source, a.last_manual_action_time AS activity_last_manual_action_time, - e.id AS event_id, e.event_time, e.event_type, e.aggregation_id, e.last_manual_action_time AS event_last_manual_action_time - FROM activities a - JOIN activity_events e ON a.id = e.activity_id - LEFT JOIN tag_mappings tm ON a.id = tm.activity_id - LEFT JOIN tags t ON tm.tag_id = t.id AND t.deleted_at IS NULL - WHERE datetime(e.event_time) >= datetime(?) AND datetime(e.event_time) <= datetime(?) - """ - params: list[Any] = [start, end] - # Productivity filter - if productivity_levels: - query += ( - " AND a.productivity_level IN (" - + ",".join(["?"] * len(productivity_levels)) - + ")" - ) - params.extend(productivity_levels) - # Exclude tags - if exclude_tag_ids: - query += ( - " AND a.id NOT IN (SELECT activity_id FROM tag_mappings WHERE tag_id IN (" - + ",".join(["?"] * len(exclude_tag_ids)) - + "))" - ) - params.extend(exclude_tag_ids) - # Include tags (any or all) - if include_tag_ids: - if include_mode == "all": - for tag_id in include_tag_ids: - query += " AND a.id IN (SELECT activity_id FROM tag_mappings WHERE tag_id = ?)" - params.append(tag_id) - else: - query += ( - " AND a.id IN (SELECT activity_id FROM tag_mappings WHERE tag_id IN (" - + ",".join(["?"] * len(include_tag_ids)) - + "))" - ) - params.extend(include_tag_ids) - # Day filter (event weekday) - if day_filter: - # SQLite: strftime('%w', e.event_time) gives 0=Sunday ... 6=Saturday - weekday_map = { - "sunday": 0, - "monday": 1, - "tuesday": 2, - "wednesday": 3, - "thursday": 4, - "friday": 5, - "saturday": 6, - } - weekday_nums = [ - str(weekday_map[d.lower()]) for d in day_filter if d.lower() in weekday_map - ] - if weekday_nums: - query += ( - " AND CAST(strftime('%w', e.event_time) AS INTEGER) IN (" - + ",".join(["?"] * len(weekday_nums)) - + ")" - ) - params.extend(weekday_nums) - # Time filter (event time overlaps any range) - if time_filter: - # Each time range is (start, end) in HH:MM:SS - time_clauses = [] - for start_str, end_str in time_filter: - time_clauses.append("(time(e.event_time) >= ? AND time(e.event_time) <= ?)") - params.extend([start_str, end_str]) - if time_clauses: - query += " AND (" + " OR ".join(time_clauses) + ")" - # Run query and group by activity - async with conn.execute(query, params) as cursor: - rows = await cursor.fetchall() - grouped = defaultdict(list) - for row in rows: - activity_id = row["activity_id"] - grouped[activity_id].append(row) - # Now fetch tags for each activity - activity_ids = list(grouped.keys()) - tags_by_activity = {} - if activity_ids: - placeholders = ",".join("?" for _ in activity_ids) - tag_query = f""" - SELECT a.id AS activity_id, t.id AS tag_id, t.name, t.description - FROM activities a - LEFT JOIN tag_mappings tm ON a.id = tm.activity_id - LEFT JOIN tags t ON tm.tag_id = t.id AND t.deleted_at IS NULL - WHERE a.id IN ({placeholders}) - """ - async with conn.execute(tag_query, activity_ids) as cursor: - tag_rows = await cursor.fetchall() - for row in tag_rows: - activity_id = row["activity_id"] - tag_id = row["tag_id"] - if activity_id not in tags_by_activity: - tags_by_activity[activity_id] = [] - if tag_id is not None: - tag = DBTag(id=tag_id, name=row["name"], description=row["description"]) - tags_by_activity[activity_id].append(tag) - # Build DBActivitySpecWithTags objects - result = [] - for activity_id, rows in grouped.items(): - db_activity = DBActivity( - id=activity_id, - name=rows[0]["name"], - description=rows[0]["description"], - productivity_level=rows[0]["productivity_level"], - source=rows[0]["source"], - last_manual_action_time=rows[0]["activity_last_manual_action_time"], - ) - events = [] - for row in rows: - db_activity_event = DBActivityEvent( - event_time=row["event_time"], - event_type=row["event_type"], - id=row["event_id"], - activity_id=activity_id, - aggregation_id=row["aggregation_id"], - last_manual_action_time=row["event_last_manual_action_time"], - ) - events.append(db_activity_event) - tags = tags_by_activity.get(activity_id, []) - spec_with_tags = DBActivitySpecWithTags( - activity=db_activity, events=events, tags=tags - ) - result.append(spec_with_tags) - return result - - -async def is_goal_paused_at( - conn: aiosqlite.Connection, *, goal_id: int, at_utc: datetime -) -> bool: - """Return True if the goal is paused at the given UTC timestamp. - - Looks up the last pause/resume event at or before the timestamp; if it's a - 'pause' event, consider the goal paused; otherwise not paused. If no event - exists, it's considered active (not paused). - """ - async with conn.execute( - """ - SELECT event_type - FROM goal_pause_events - WHERE goal_id = ? AND datetime(at) <= datetime(?) - ORDER BY datetime(at) DESC, id DESC - LIMIT 1 - """, - (goal_id, at_utc), - ) as cursor: - row = await cursor.fetchone() - if not row: - return False - try: - return row["event_type"] == "pause" - except (KeyError, TypeError, IndexError): - # Fallback for row indexing differences - return (row[0] if row else None) == "pause" - - -async def select_tags_by_ids( - conn: aiosqlite.Connection, tag_ids: List[int], include_deleted: bool = False -) -> list[dict]: - """ - Select tags by a list of IDs. If include_deleted is True, include soft-deleted tags. - Returns a list of dicts with id, name, description, and deleted_at. - """ - if not tag_ids: - return [] - placeholders = ",".join(["?"] * len(tag_ids)) - query = ( - f"SELECT id, name, description, deleted_at FROM tags WHERE id IN ({placeholders})" - if include_deleted - else f"SELECT id, name, description, deleted_at FROM tags WHERE id IN ({placeholders}) AND deleted_at IS NULL" - ) - async with conn.execute(query, tag_ids) as cursor: - rows = await cursor.fetchall() - return [dict(row) for row in rows] - - -@lru_cache -def simple_insert_query( - table_name: str, columns: tuple[str], returning_columns: tuple[str] = tuple() -) -> str: - insert_statemnent = f"INSERT INTO {table_name} ({','.join(columns)}) VALUES ({','.join([f':{col}' for col in columns])})" - if returning_columns: - return f"{insert_statemnent} RETURNING {','.join(returning_columns)};" - else: - return f"{insert_statemnent};" - - -@lru_cache -def set_clause_query(table_name: str, columns: tuple[str]) -> str: - return f"UPDATE {table_name} SET {','.join([f'{col} = :{col}' for col in columns])}" - - -# (source_events moved to Valkey Streams; DB helpers removed) - - -async def insert_user_settings( - conn: aiosqlite.Connection, - timezone: str, - username: str, - image_processing_approach: str = ImageProcessingApproach.OCR.value, - image_model_provider: str = "", - image_model_base_url: str | None = None, - image_model: str = "", - image_model_api_key_enc: bytes | None = None, - text_model_provider: str = "", - text_model_base_url: str | None = None, - text_model: str = "", - text_model_api_key_enc: bytes | None = None, - productivity_prompt: str = "", -) -> None: - sql = simple_insert_query( - "user_settings", - ( - "timezone", - "image_processing_approach", - "image_model_provider", - "image_model_base_url", - "image_model", - "image_model_api_key_enc", - "text_model_provider", - "text_model_base_url", - "text_model", - "text_model_api_key_enc", - "username", - "productivity_prompt", # Add column name - ), - ) - - await conn.execute( - sql, - { - "timezone": timezone, - "image_processing_approach": image_processing_approach, - "image_model_provider": image_model_provider, - "image_model_base_url": image_model_base_url, - "image_model": image_model, - "image_model_api_key_enc": image_model_api_key_enc, - "text_model_provider": text_model_provider, - "text_model_base_url": text_model_base_url, - "text_model": text_model, - "text_model_api_key_enc": text_model_api_key_enc, - "username": username, - "productivity_prompt": productivity_prompt, # Add parameter value - }, - ) - - -def build_user_settings_from_row(row: Any) -> UserSettings: - """Construct a UserSettings object (with decrypted API keys) from a DB row. - - Expects columns: - username, timezone, - image_processing_approach, - image_model_provider, image_model_base_url, image_model, image_model_api_key_enc, - text_model_provider, text_model_base_url, text_model, text_model_api_key_enc, - productivity_prompt - """ - # Decrypt image model key (if present) - if row["image_model_api_key_enc"]: - try: - image_model_api_key = decrypt_secret( - row["image_model_api_key_enc"], - config.master_key.get_secret_value(), - aad=AADS.LLM_API_KEY, - ) - except Exception as exc: - raise RuntimeError( - "Failed to decrypt image model API key, did you change the master key?" - ) from exc - else: - image_model_api_key = None - - # Decrypt text model key (if present) - if row["text_model_api_key_enc"]: - try: - text_model_api_key = decrypt_secret( - row["text_model_api_key_enc"], - config.master_key.get_secret_value(), - aad=AADS.LLM_API_KEY, - ) - except Exception as exc: - raise RuntimeError( - "Failed to decrypt text model API key, did you change the master key?" - ) from exc - else: - text_model_api_key = None - - # Determine the correct provider class based on the model_provider string - image_provider = row["image_model_provider"] - if image_provider == ModelProvider.GOOGLE_AI: - image_model_config = GoogleAIConfig( - model_base_url=row["image_model_base_url"], - model=row["image_model"], - api_key=image_model_api_key, - ) - elif image_provider == ModelProvider.OPENAI: - image_model_config = OpenAIConfig( - model_base_url=row["image_model_base_url"], - model=row["image_model"], - api_key=image_model_api_key, - ) - elif image_provider == ModelProvider.OPENAI_GENERIC: - image_model_config = OpenAIGenericConfig( - model_base_url=row["image_model_base_url"], - model=row["image_model"], - api_key=image_model_api_key, - ) - elif image_provider == ModelProvider.ANTHROPIC: - image_model_config = AnthropicConfig( - model_base_url=row["image_model_base_url"], - model=row["image_model"], - api_key=image_model_api_key, - ) - else: - raise ValueError(f"Unknown image model provider: {image_provider}") - - # Determine the correct provider class based on the model_provider string - text_provider = row["text_model_provider"] - if text_provider == ModelProvider.GOOGLE_AI: - text_model_config = GoogleAIConfig( - model_base_url=row["text_model_base_url"], - model=row["text_model"], - api_key=text_model_api_key, - ) - elif text_provider == ModelProvider.OPENAI: - text_model_config = OpenAIConfig( - model_base_url=row["text_model_base_url"], - model=row["text_model"], - api_key=text_model_api_key, - ) - elif text_provider == ModelProvider.OPENAI_GENERIC: - text_model_config = OpenAIGenericConfig( - model_base_url=row["text_model_base_url"], - model=row["text_model"], - api_key=text_model_api_key, - ) - elif text_provider == ModelProvider.ANTHROPIC: - text_model_config = AnthropicConfig( - model_base_url=row["text_model_base_url"], - model=row["text_model"], - api_key=text_model_api_key, - ) - else: - raise ValueError(f"Unknown text model provider: {text_provider}") - - image_processing_approach = ImageProcessingApproach( - row["image_processing_approach"] - ) - - return UserSettings( - timezone=row["timezone"], - image_model_config=image_model_config, - text_model_config=text_model_config, - username=row["username"], - productivity_prompt=row["productivity_prompt"], - image_processing_approach=image_processing_approach, - ) - - -async def select_user_settings(conn) -> UserSettings | None: - async with conn.execute( - """SELECT username, timezone, - image_processing_approach, - image_model_provider, image_model_base_url, image_model, image_model_api_key_enc, - text_model_provider, text_model_base_url, text_model, text_model_api_key_enc, - productivity_prompt - FROM user_settings LIMIT 1""" - ) as cursor: - row = await cursor.fetchone() - if row: - return build_user_settings_from_row(row) - - -async def select_user_auth(conn: aiosqlite.Connection) -> Optional[dict]: - async with conn.execute( - "SELECT id, password_hash, created_at FROM user_auth WHERE id='default' LIMIT 1" - ) as cursor: - row = await cursor.fetchone() - return dict(row) if row else None - - -async def update_user_password(conn: aiosqlite.Connection, password_hash: str) -> None: - logger.debug("Updating password for user") - sql = "UPDATE user_auth SET password_hash = :password_hash WHERE id='default'" - params = {"password_hash": password_hash} - try: - await conn.execute(sql, params) - logger.info("Password updated successfully for user") - except Exception as e: # noqa: BLE001 - propagate as runtime error with context - raise RuntimeError("Error updating password for user") from e - - -async def create_user_auth(conn: aiosqlite.Connection, password_hash: str) -> None: - sql = "INSERT INTO user_auth (id, password_hash) VALUES ('default', :password_hash)" - try: - await conn.execute(sql, {"password_hash": password_hash}) - logger.info("Initialized user_auth with bootstrap password") - except Exception as e: - raise RuntimeError("Error initializing user_auth") from e - - -async def get_user_settings_draft( - conn: aiosqlite.Connection, *, wizard_id: str -) -> dict | None: - async with conn.execute( - """ - SELECT * - FROM user_settings_draft - WHERE wizard_id = ? - LIMIT 1 - """, - (wizard_id,), - ) as cursor: - row = await cursor.fetchone() - return dict(row) if row else None - - -async def upsert_user_settings_draft_basics( - conn: aiosqlite.Connection, - *, - wizard_id: str, - username: str, - timezone: str, - description: str | None = None, -) -> None: - await conn.execute( - """ - INSERT INTO user_settings_draft ( - wizard_id, username, timezone, description - ) VALUES (:wizard_id, :username, :timezone, :description) - ON CONFLICT(wizard_id) DO UPDATE SET - username = excluded.username, - timezone = excluded.timezone, - description = excluded.description - """, - { - "wizard_id": wizard_id, - "username": username, - "timezone": timezone, - "description": description, - }, - ) - - -async def update_user_settings_draft_productivity( - conn: aiosqlite.Connection, *, wizard_id: str, productivity_prompt: str -) -> None: - await conn.execute( - """ - INSERT INTO user_settings_draft (wizard_id, productivity_prompt) - VALUES (:wizard_id, :productivity_prompt) - ON CONFLICT(wizard_id) DO UPDATE SET - productivity_prompt = excluded.productivity_prompt - """, - {"wizard_id": wizard_id, "productivity_prompt": productivity_prompt}, - ) - - -async def update_user_settings_draft_tags( - conn: aiosqlite.Connection, *, wizard_id: str, tags_json: str | None -) -> None: - await conn.execute( - """ - INSERT INTO user_settings_draft (wizard_id, tags_json) - VALUES (:wizard_id, :tags_json) - ON CONFLICT(wizard_id) DO UPDATE SET - tags_json = excluded.tags_json - """, - {"wizard_id": wizard_id, "tags_json": tags_json}, - ) - - -async def update_user_settings_draft_llm_configs( - conn: aiosqlite.Connection, - *, - wizard_id: str, - image_model_provider: str | None, - image_model_base_url: str | None, - image_model: str | None, - image_model_api_key_enc: bytes | None, - text_model_provider: str | None, - text_model_base_url: str | None, - text_model: str | None, - text_model_api_key_enc: bytes | None, - image_processing_approach: str | None = None, -) -> None: - await conn.execute( - """ - INSERT INTO user_settings_draft ( - wizard_id, - image_model_provider, - image_model_base_url, - image_model, - image_model_api_key_enc, - text_model_provider, - text_model_base_url, - text_model, - text_model_api_key_enc, - image_processing_approach - ) VALUES ( - :wizard_id, - :image_model_provider, - :image_model_base_url, - :image_model, - :image_model_api_key_enc, - :text_model_provider, - :text_model_base_url, - :text_model, - :text_model_api_key_enc, - :image_processing_approach - ) - ON CONFLICT(wizard_id) DO UPDATE SET - image_model_provider = excluded.image_model_provider, - image_model_base_url = excluded.image_model_base_url, - image_model = excluded.image_model, - image_model_api_key_enc = excluded.image_model_api_key_enc, - text_model_provider = excluded.text_model_provider, - text_model_base_url = excluded.text_model_base_url, - text_model = excluded.text_model, - text_model_api_key_enc = excluded.text_model_api_key_enc, - image_processing_approach = excluded.image_processing_approach - """, - { - "wizard_id": wizard_id, - "image_model_provider": image_model_provider, - "image_model_base_url": image_model_base_url, - "image_model": image_model, - "image_model_api_key_enc": image_model_api_key_enc, - "text_model_provider": text_model_provider, - "text_model_base_url": text_model_base_url, - "text_model": text_model, - "text_model_api_key_enc": text_model_api_key_enc, - "image_processing_approach": ( - image_processing_approach or ImageProcessingApproach.OCR.value - ), - }, - ) - - -async def delete_user_settings_draft( - conn: aiosqlite.Connection, *, wizard_id: str -) -> None: - await conn.execute( - "DELETE FROM user_settings_draft WHERE wizard_id = ?", (wizard_id,) - ) - - -async def finalize_user_settings_from_draft( - conn: aiosqlite.Connection, *, wizard_id: str -) -> None: - """Promote draft to user_settings and create initial tags if provided. - - Assumes that no user_settings row exists yet (single-user system). - """ - draft = await get_user_settings_draft(conn, wizard_id=wizard_id) - if not draft: - raise ValueError("Draft not found") - - username = (draft.get("username") or "").strip() - if not username: - raise ValueError("Username is required to finalize account creation") - - # Insert user_settings - await insert_user_settings( - conn=conn, - timezone=(draft.get("timezone") or "UTC"), - image_processing_approach=( - draft.get("image_processing_approach") or ImageProcessingApproach.OCR.value - ), - image_model_provider=(draft.get("image_model_provider") or ""), - image_model_base_url=(draft.get("image_model_base_url") or None), - image_model=(draft.get("image_model") or ""), - image_model_api_key_enc=(draft.get("image_model_api_key_enc") or None), - text_model_provider=(draft.get("text_model_provider") or ""), - text_model_base_url=(draft.get("text_model_base_url") or None), - text_model=(draft.get("text_model") or ""), - text_model_api_key_enc=(draft.get("text_model_api_key_enc") or None), - username=username, - productivity_prompt=(draft.get("productivity_prompt") or ""), - ) - - # Create initial tags if provided - try: - tags_json = draft.get("tags_json") - if tags_json: - data = json.loads(tags_json) - if isinstance(data, list) and data: - # Prepare rows for bulk insert - rows = [] - for item in data: - name = (item or {}).get("name") or "" - description = (item or {}).get("description") or "" - if name.strip(): - rows.append({"name": name.strip(), "description": description}) - if rows: - sql = simple_insert_query("tags", ("name", "description")) - await conn.executemany(sql, rows) - except (json.JSONDecodeError, TypeError, KeyError) as exc: - logger.exception( - "Failed to parse or insert initial tags from draft: {error}", - error=exc, - ) - - # Remove the draft - await delete_user_settings_draft(conn, wizard_id=wizard_id) - - -async def update_user_settings( - conn: aiosqlite.Connection, settings: dict[str, Any] -) -> UserSettings: - """ - Update user settings from a dictionary and return the full user_settings row - using a RETURNING clause. Password changes should be handled separately. - - Returns the updated UserSettings object, or None if no row was updated. - """ - if "password" in settings: - raise ValueError("Password updates are not allowed through this function.") - assert settings, "update_user_settings called with no settings to update." - - logger.debug(f"Updating user settings with keys: {list(settings.keys())}") - - # Build the SET clause dynamically from the dictionary keys - set_clause = ", ".join(f"{key} = :{key}" for key in settings.keys()) - - # Explicitly list columns to return to build a proper UserSettings - returning_cols = ( - "username, timezone, image_processing_approach, " - "image_model_provider, image_model_base_url, image_model, image_model_api_key_enc, " - "text_model_provider, text_model_base_url, text_model, text_model_api_key_enc, " - "productivity_prompt" - ) - - sql = f"UPDATE user_settings SET {set_clause} RETURNING {returning_cols}" - - params = {**settings} - - try: - cursor = await conn.execute(sql, params) - row = await cursor.fetchone() - assert row is not None, "No user_settings row returned after update." - - updated = build_user_settings_from_row(row) - - logger.debug("User settings updated successfully with RETURNING") - return updated - except Exception as exc: - logger.exception("Error updating user settings: {error}", error=exc) - raise - - # Note: update_user_password moved above to user_auth section - - -async def insert_activity_events( - conn: aiosqlite.Connection, events: Sequence[ActivityEventInsert] -) -> None: - # Use INSERT OR IGNORE to provide idempotency when a unique index exists - # on (activity_id, event_time, event_type). This prevents duplicate events - # from causing IntegrityError on retries or replays. - sql = ( - "INSERT OR IGNORE INTO activity_events (activity_id,event_time,event_type,aggregation_id,last_manual_action_time) " - "VALUES (:activity_id,:event_time,:event_type,:aggregation_id,:last_manual_action_time);" - ) - - dicts = [ - { - "activity_id": event.activity_id, - "event_time": event.event_time, - "event_type": event.event_type, - "aggregation_id": event.aggregation_id, - "last_manual_action_time": event.last_manual_action_time, - } - for event in events - ] - await conn.executemany(sql, dicts) - - -async def select_open_activities_with_last_event( - conn: aiosqlite.Connection, -) -> list[DBActivityWithLatestEvent]: - query = """ - SELECT - a.id, a.name, a.description, a.productivity_level, a.source, -- Select specific columns - a.last_manual_action_time AS activity_last_manual_action_time, -- Alias activity time - le.event_type AS state, - le.event_time, - le.aggregation_id, - le.id AS last_event_id, - ag.start_time as aggregation_start_time, - ag.end_time as aggregation_end_time, - ag.first_timestamp as aggregation_first_timestamp, - ag.last_timestamp as aggregation_last_timestamp, - le.last_manual_action_time AS event_last_manual_action_time -- Alias event time - - FROM activities a - JOIN ( - SELECT * - FROM activity_events ae - WHERE ae.id = ( - SELECT ae2.id - FROM activity_events ae2 - WHERE ae2.activity_id = ae.activity_id - ORDER BY ae2.event_time DESC - LIMIT 1 - ) - ) le ON le.activity_id = a.id - LEFT JOIN aggregations ag ON le.aggregation_id = ag.id - WHERE le.event_type <> 'close'; - """ - - return_list = [] - async with conn.execute(query) as cursor: - rows = await cursor.fetchall() - for row in rows: - db_activity = DBActivity( - id=row["id"], - name=row["name"], - description=row["description"], - productivity_level=row["productivity_level"], - source=row["source"], # Assign source here - last_manual_action_time=row[ - "activity_last_manual_action_time" - ], # Use aliased activity time - # source=row["source"], # Removed source from here - ) - - db_activity_event = DBActivityEvent( - event_time=row["event_time"], - event_type=row["state"], - id=row["last_event_id"], - activity_id=row["id"], - aggregation_id=row["aggregation_id"], - last_manual_action_time=row[ - "event_last_manual_action_time" - ], # Use aliased event time - ) - latest_aggregation = DBAggregation( - id=row["aggregation_id"], - start_time=row["aggregation_start_time"], - end_time=row["aggregation_end_time"], - first_timestamp=row["aggregation_first_timestamp"], - last_timestamp=row["aggregation_last_timestamp"], - ) - return_list.append( - DBActivityWithLatestEvent( - activity=db_activity, - latest_event=db_activity_event, - latest_aggregation=latest_aggregation, - ) - ) - return return_list - - -async def select_open_auto_activities_with_last_event( - conn: aiosqlite.Connection, -) -> list[DBActivityWithLatestEvent]: - query = """ - SELECT - a.id, a.name, a.description, a.productivity_level, a.source, -- Select specific columns - a.last_manual_action_time AS activity_last_manual_action_time, -- Alias activity time - le.event_type AS state, - le.event_time, - le.aggregation_id, - le.id AS last_event_id, - ag.start_time as aggregation_start_time, - ag.end_time as aggregation_end_time, - ag.first_timestamp as aggregation_first_timestamp, - ag.last_timestamp as aggregation_last_timestamp, - le.last_manual_action_time AS event_last_manual_action_time -- Alias event time - - FROM activities a - JOIN ( - SELECT * - FROM activity_events ae - WHERE ae.id = ( - SELECT ae2.id - FROM activity_events ae2 - WHERE ae2.activity_id = ae.activity_id - ORDER BY ae2.event_time DESC - LIMIT 1 - ) - ) le ON le.activity_id = a.id - LEFT JOIN aggregations ag ON le.aggregation_id = ag.id - WHERE le.event_type <> 'close' AND a.source = 'auto'; - """ - - return_list = [] - async with conn.execute(query) as cursor: - rows = await cursor.fetchall() - for row in rows: - db_activity = DBActivity( - id=row["id"], - name=row["name"], - description=row["description"], - productivity_level=row["productivity_level"], - source=row["source"], # Assign source here - last_manual_action_time=row[ - "activity_last_manual_action_time" - ], # Use aliased activity time - ) - - db_activity_event = DBActivityEvent( - event_time=row["event_time"], - event_type=row["state"], - id=row["last_event_id"], - activity_id=row["id"], - aggregation_id=row["aggregation_id"], - last_manual_action_time=row[ - "event_last_manual_action_time" - ], # Use aliased event time - ) - latest_aggregation = DBAggregation( - id=row["aggregation_id"], - start_time=row["aggregation_start_time"], - end_time=row["aggregation_end_time"], - first_timestamp=row["aggregation_first_timestamp"], - last_timestamp=row["aggregation_last_timestamp"], - ) - return_list.append( - DBActivityWithLatestEvent( - activity=db_activity, - latest_event=db_activity_event, - latest_aggregation=latest_aggregation, - ) - ) - return return_list - - -async def insert_activities( - conn: aiosqlite.Connection, activities: list[Activity] -) -> list[int]: - if not activities: - return [] - - sql = simple_insert_query( - "activities", - ( - "name", - "description", - "productivity_level", - "source", - "last_manual_action_time", - ), - ) - - insert_dicts = [ - { - "name": activity.name, - "description": activity.description, - "productivity_level": activity.productivity_level, - "source": activity.source, - "last_manual_action_time": activity.last_manual_action_time, - } - for activity in activities - ] - - await conn.executemany(sql, insert_dicts) - - max_id_cursor = await conn.execute( - f"SELECT id FROM activities ORDER BY id DESC LIMIT {len(activities)}" - ) - max_id_rows = await max_id_cursor.fetchall() - assert max_id_rows is not None, "No activities were inserted" - ids = [row["id"] for row in max_id_rows] - ids.reverse() - - return ids - - -async def insert_aggregation( - conn: aiosqlite.Connection, aggregation: Aggregation -) -> int: - sql = simple_insert_query( - "aggregations", ("start_time", "end_time", "first_timestamp", "last_timestamp") - ) - - cursor = await conn.execute( - sql, - {key: val for key, val in aggregation.model_dump().items() if key != "id"}, - ) - last_row_id = cursor.lastrowid - assert last_row_id is not None, "No row was inserted" - return last_row_id - - -async def get_specs_after_cutoff( - conn: aiosqlite.Connection, cutoff: datetime -) -> list[DBActivitySpec]: - query = """ - SELECT a.id AS activity_id, - a.name, - a.description, - a.productivity_level, - a.source, -- Select source - a.last_manual_action_time AS activity_last_manual_action_time, -- Alias - e.id AS event_id, - e.event_time, - e.event_type, - e.aggregation_id, - e.last_manual_action_time AS event_last_manual_action_time -- Alias - - FROM activities a - JOIN activity_events e ON a.id = e.activity_id - WHERE datetime(e.event_time) >= datetime(?); - """ - - # WHERE datetime(e.event_time) >= datetime(?) - async with conn.execute(query, (cutoff,)) as cursor: - rows = await cursor.fetchall() - grouped = defaultdict(list) - for row in rows: - activity_id = row["activity_id"] - grouped[activity_id].append(row) - return_list = [] - - for activity_id, rows in grouped.items(): - db_activity = DBActivity( - id=activity_id, - name=rows[0]["name"], - description=rows[0]["description"], - productivity_level=rows[0]["productivity_level"], - source=rows[0]["source"], # Assign source here - last_manual_action_time=rows[0][ - "activity_last_manual_action_time" - ], # Use aliased activity time - # source=rows[0]["source"], # Removed source from here - ) - - events = [] - for row in rows: - db_activity_event = DBActivityEvent( - event_time=row["event_time"], - event_type=row["event_type"], - id=row["event_id"], - activity_id=activity_id, - aggregation_id=row["aggregation_id"], - last_manual_action_time=row[ - "event_last_manual_action_time" - ], # Use aliased event time - ) - events.append(db_activity_event) - - db_activity_spec = DBActivitySpec(activity=db_activity, events=events) - - return_list.append(db_activity_spec) - - return return_list - - -async def select_tags( - conn: aiosqlite.Connection, include_deleted: bool = False -) -> list[DBTag]: - """Select tags. - - By default, excludes soft-deleted tags (deleted_at IS NULL). - Set include_deleted=True to return all tags regardless of deleted_at. - """ - query = ( - "SELECT id, name, description FROM tags" - if include_deleted - else "SELECT id, name, description FROM tags WHERE deleted_at IS NULL" - ) - - try: - async with conn.execute(query) as cursor: - rows = await cursor.fetchall() - - tags = [ - DBTag(id=row["id"], name=row["name"], description=row["description"]) - for row in rows - ] - - logger.trace(f"Selected {len(tags)} tags from database") - for tag in tags: - logger.trace( - f" - Tag {tag.id}: name='{tag.name}', description='{tag.description}'" - ) - - return tags - except Exception as exc: - logger.exception("Error selecting tags: {error}", error=exc) - raise - - -async def insert_tags(conn: aiosqlite.Connection, tags: list[Tag]) -> list[int]: - if not tags: - return [] - - try: - sql = simple_insert_query("tags", ("name", "description")) - - insert_dicts = [ - { - "name": tag.name or "", # Ensure name is never None - "description": tag.description - or "", # Ensure description is never None - } - for tag in tags - ] - - for i, tag_dict in enumerate(insert_dicts): - logger.debug(f"Inserting tag {i}: {tag_dict}") - - await conn.executemany(sql, insert_dicts) - - # Fetch the IDs of the newly inserted tags - max_id_cursor = await conn.execute( - f"SELECT id FROM tags ORDER BY id DESC LIMIT {len(tags)}" - ) - max_id_rows = await max_id_cursor.fetchall() - assert max_id_rows is not None, "No tags were inserted" - ids = [row["id"] for row in max_id_rows] - ids.reverse() # Reverse to match the order of the input tags - - logger.info(f"Inserted {len(ids)} tags with IDs: {ids}") - return ids - - except Exception as exc: - logger.exception("Error inserting tags: {error}", error=exc) - raise - - -async def update_tags(conn: aiosqlite.Connection, tags: list[DBTag]) -> None: - if not tags: - return - - try: - # All DBTag instances have IDs, so no need to filter - valid_tags = tags - - # Prepare statement and parameters for executemany - update_params = [ - ( - tag.name, - tag.description or "", - tag.id, - ) # Ensure description is never None - for tag in valid_tags - ] - - logger.debug(f"Updating {len(valid_tags)} tags") - for tag in valid_tags: - logger.debug( - f" - Updating tag {tag.id}: name='{tag.name}', description='{tag.description}'" - ) - - # Use named placeholders for clarity - query = "UPDATE tags SET name = ?, description = ? WHERE id = ?" - await conn.executemany(query, update_params) - - # Verify the updates took effect - for tag in valid_tags: - verify_query = "SELECT name, description FROM tags WHERE id = ?" - async with conn.execute(verify_query, (tag.id,)) as cursor: - row = await cursor.fetchone() - if row: - logger.debug( - f" - Verified tag {tag.id}: name='{row['name']}', description='{row['description']}'" - ) - - except Exception as exc: - logger.exception("Error updating tags: {error}", error=exc) - raise - - -async def delete_tags(conn: aiosqlite.Connection, tag_ids: list[int]) -> None: - """Soft-delete multiple tags by their IDs by setting deleted_at. - - Note: existing UNIQUE constraint on tags.name remains; soft-deleted rows - still occupy their names. Consider adjusting uniqueness in a follow-up - migration if you need to re-create tags with the same name after delete. - """ - if not tag_ids: - return - - # Use IN clause with placeholders - placeholders = ",".join("?" for _ in tag_ids) - await conn.execute( - f"UPDATE tags SET deleted_at = datetime('now') WHERE deleted_at IS NULL AND id IN ({placeholders})", - tuple(tag_ids), - ) - - -async def bulk_upsert_tags( - conn: aiosqlite.Connection, - tags_to_update: list[DBTag], - tags_to_insert: list[Tag], - ids_to_delete: list[int], -) -> list[int]: - """Perform bulk tag operations (update, insert, delete) in a single transaction - - Returns the IDs of the newly inserted tags. - """ - try: - logger.debug( - f"Bulk upsert: updating {len(tags_to_update)} tags, " - + f"inserting {len(tags_to_insert)} tags, " - + f"deleting {len(ids_to_delete)} tags" - ) - - # Delete tags in bulk - if ids_to_delete: - await delete_tags(conn, ids_to_delete) - - # Update tags in bulk - if tags_to_update: - await update_tags(conn, tags_to_update) - - # Insert tags in bulk - new_ids = [] - if tags_to_insert: - new_ids = await insert_tags(conn, tags_to_insert) - - return new_ids - - except Exception as exc: - logger.exception("Error in bulk_upsert_tags: {error}", error=exc) - # Still do rollback on error - await conn.rollback() - raise - - -async def insert_tag_mappings( - conn: aiosqlite.Connection, mappings: list[TagMapping] -) -> list[int]: - if not mappings: - return [] - - try: - sql = simple_insert_query("tag_mappings", ("tag_id", "activity_id")) - - insert_dicts = [ - { - "tag_id": mapping.tag_id, - "activity_id": mapping.activity_id, - } - for mapping in mappings - ] - - logger.trace(f"Inserting {len(mappings)} tag mappings") - for mapping in mappings: - logger.trace( - f" - Mapping activity {mapping.activity_id} to tag {mapping.tag_id}" - ) - - await conn.executemany(sql, insert_dicts) - - # Get the IDs of newly inserted mappings - max_id_cursor = await conn.execute( - f"SELECT id FROM tag_mappings ORDER BY id DESC LIMIT {len(mappings)}" - ) - max_id_rows = await max_id_cursor.fetchall() - assert max_id_rows is not None, "No tag mappings were inserted" - ids = [row["id"] for row in max_id_rows] - ids.reverse() # Reverse to match the order of the input mappings - - logger.trace(f"Inserted {len(ids)} tag mappings with IDs: {ids}") - return ids - - except Exception as exc: - logger.exception("Error inserting tag mappings: {error}", error=exc) - raise - - -async def delete_tag_mappings( - conn: aiosqlite.Connection, activity_id: int, tag_ids: list[int] -) -> None: - if not tag_ids: - return - - # Use IN clause with placeholders - placeholders = ",".join("?" for _ in tag_ids) - await conn.execute( - f"DELETE FROM tag_mappings WHERE activity_id = ? AND tag_id IN ({placeholders})", - (activity_id, *tag_ids), - ) - - -async def delete_activity(conn: aiosqlite.Connection, activity_id: int) -> None: - # Deletes an activity by its ID. Assumes related events/tags are handled by CASCADE. - # Note: Ensure foreign keys in activity_events and tag_mappings have ON DELETE CASCADE - await conn.execute("DELETE FROM activities WHERE id = ?", (activity_id,)) - # No explicit commit needed if the caller manages the transaction - - -async def update_activity( - conn: aiosqlite.Connection, - activity_id: int, - key_value_pairs: dict[str, Any], -): - keys = tuple(key_value_pairs.keys()) - - base_query = set_clause_query("activities", keys) - - query_dict = {**key_value_pairs, "id": activity_id} - - query = f""" - {base_query} - WHERE id = :id; - """ - await conn.execute(query, query_dict) - - -async def delete_activity_events( - conn: aiosqlite.Connection, activity_ids: list[int] -) -> None: - await conn.execute( - f"DELETE FROM activity_events WHERE activity_id IN ({','.join(['?'] * len(activity_ids))})", - tuple(activity_ids), - ) - - -async def select_activity_spec_with_tags( - conn: aiosqlite.Connection, activity_id: int -) -> DBActivitySpecWithTags: - # Query to get the specific activity and its events - activity_query = """ - SELECT a.id AS activity_id, - a.name, - a.description, - a.productivity_level, - a.source, -- Select source - a.last_manual_action_time AS activity_last_manual_action_time, -- Alias - e.id AS event_id, - e.event_time, - e.event_type, - e.aggregation_id, - e.last_manual_action_time AS event_last_manual_action_time -- Alias - FROM activities a - JOIN activity_events e ON a.id = e.activity_id - WHERE a.id = ?; - """ - - async with conn.execute(activity_query, (activity_id,)) as cursor: - rows = await cursor.fetchall() - activity_rows = list(rows) - - if not activity_rows: - raise ValueError(f"No activity found with ID {activity_id}") - - # Process the activity and its events - activity_data = activity_rows[0] - - db_activity = DBActivity( - id=activity_id, - name=activity_data["name"], - description=activity_data["description"], - productivity_level=activity_data["productivity_level"], - source=activity_data["source"], # Assign source here - last_manual_action_time=activity_data[ - "activity_last_manual_action_time" - ], # Use aliased activity time - # source=activity_data["source"], # Removed source from here - ) - - events = [] - for row in activity_rows: - db_activity_event = DBActivityEvent( - event_time=row["event_time"], - event_type=row["event_type"], - id=row["event_id"], - activity_id=activity_id, - aggregation_id=row["aggregation_id"], - last_manual_action_time=row[ - "event_last_manual_action_time" - ], # Use aliased event time - ) - events.append(db_activity_event) - - # Query to get tags for this specific activity - tags_query = """ - SELECT t.id AS tag_id, t.name, t.description - FROM tags t - JOIN tag_mappings tm ON t.id = tm.tag_id - WHERE tm.activity_id = ? AND t.deleted_at IS NULL; - """ - tags = [] - async with conn.execute(tags_query, (activity_id,)) as cursor: - tag_rows = await cursor.fetchall() - for row in tag_rows: - assert row["tag_id"] is not None, "Tag ID cannot be None" - tags.append( - DBTag( - id=row["tag_id"], name=row["name"], description=row["description"] - ) - ) - - # Combine into the final spec object - spec_with_tags = DBActivitySpecWithTags( - activity=db_activity, events=events, tags=tags - ) - - return spec_with_tags - - -async def select_goals_with_latest_definition( - conn: aiosqlite.Connection, - last_successes_limit: int = 5, -) -> list[GoalWithLatestResult]: - """Return all goals with their current definition, include/exclude tags, - the latest result (if any), and last N success booleans using a single DB call. - - - Respects tag soft-deletes (only includes tags where t.deleted_at IS NULL). - - Maps DB string fields to enums in entities, with lenient fallbacks. - """ - query = """ - WITH latest_defs AS ( - SELECT gd.* - FROM goal_definitions gd - JOIN ( - SELECT goal_id, MAX(effective_from) AS max_eff - FROM goal_definitions - WHERE datetime(effective_from) <= datetime('now') - GROUP BY goal_id - ) m ON m.goal_id = gd.goal_id AND m.max_eff = gd.effective_from - ), - top_results AS ( - SELECT * - FROM ( - SELECT r.*, - ROW_NUMBER() OVER ( - PARTITION BY r.goal_definition_id - ORDER BY r.period_start DESC, r.id DESC - ) AS rn - FROM goal_results r - ) - WHERE rn <= :limit - ), - last_pe AS ( - SELECT goal_id, event_type, at, - ROW_NUMBER() OVER (PARTITION BY goal_id ORDER BY datetime(at) DESC, id DESC) rn - FROM goal_pause_events - ) - SELECT - -- Goal fields - g.id AS goal_id, - CASE WHEN lpe.event_type = 'pause' THEN lpe.at ELSE NULL END AS goal_paused_since, - g.created_at AS goal_created_at, - g.metric AS metric, - g.operator AS operator, - g.period AS period, - g.timezone AS timezone, - - -- Definition fields - ld.id AS def_id, - ld.goal_id AS goal_id_for_def, - ld.name AS def_name, - ld.description AS def_description, - ld.metric_params_json AS metric_params_json, - ld.target_value AS target_value, - ld.include_mode AS include_mode, - ld.day_filter_json AS day_filter_json, - ld.time_filter_json AS time_filter_json, - ld.productivity_filter_json AS productivity_filter_json, - ld.effective_from AS effective_from, - ld.created_at AS def_created_at, - - -- Tag fields - t.id AS tag_id, - t.name AS tag_name, - t.description AS tag_description, - gt.role AS tag_role, - - -- Top results (last N), with rn=1 being latest - tr.id AS res_id, - tr.period_start AS res_period_start, - tr.period_end AS res_period_end, - tr.metric_value AS res_metric_value, - tr.success AS res_success, - tr.eval_state AS res_eval_state, - tr.eval_state_reason AS res_eval_state_reason, - tr.created_at AS res_created_at, - tr.rn AS res_rn - FROM goals g - JOIN latest_defs ld ON ld.goal_id = g.id - LEFT JOIN last_pe lpe ON lpe.goal_id = g.id AND lpe.rn = 1 - LEFT JOIN goal_tags gt ON gt.goal_definition_id = ld.id - LEFT JOIN tags t ON t.id = gt.tag_id AND t.deleted_at IS NULL - LEFT JOIN top_results tr ON tr.goal_definition_id = ld.id - ORDER BY g.id, ld.id, t.id, res_rn - """ - - async with conn.execute(query, {"limit": last_successes_limit}) as cursor: - rows = await cursor.fetchall() - - if not rows: - return [] - - # Aggregate per goal and definition - results_by_goal: dict[int, dict[str, Any]] = {} - seen_tag_include: dict[int, set[int]] = {} - seen_tag_exclude: dict[int, set[int]] = {} - seen_success_rn: dict[int, set[int]] = {} - - for r in rows: - goal_id = r["goal_id"] - def_id = r["def_id"] - - # Initialize structures for this goal if first time - if goal_id not in results_by_goal: - # Build goal object - metric = GoalMetric(r["metric"]) # may raise if invalid - op = ( - MetricOperator(r["operator"]) if r["operator"] else MetricOperator.EQUAL - ) - period = GoalPeriod(r["period"]) if r["period"] else GoalPeriod.DAY - - def _coerce_operator(m: GoalMetric, opx: MetricOperator) -> MetricOperator: - if m in ( - GoalMetric.AVG_PRODUCTIVITY_LEVEL, - GoalMetric.TOTAL_ACTIVITY_DURATION, - ): - return ( - opx - if opx - in (MetricOperator.LESS_THAN, MetricOperator.GREATER_THAN) - else MetricOperator.GREATER_THAN - ) - return opx - - op = _coerce_operator(metric, op) - - if metric == GoalMetric.AVG_PRODUCTIVITY_LEVEL: - goal_obj: DBGoal = DBAvgProductivityGoal( - id=goal_id, - created_at=r["goal_created_at"], - paused_since=r["goal_paused_since"], - timezone=r["timezone"], - period=period, - operator=cast(AvgProductivityOperators, op), - ) - else: - goal_obj = DBTotalActivityDurationGoal( - id=goal_id, - created_at=r["goal_created_at"], - paused_since=r["goal_paused_since"], - timezone=r["timezone"], - period=period, - operator=cast(TotalActivityDurationOperators, op), - ) - - # Build definition from row - def_row = { - "id": r["def_id"], - "goal_id": r["goal_id_for_def"], - "name": r["def_name"], - "description": r["def_description"], - "metric_params_json": r["metric_params_json"], - "target_value": r["target_value"], - "include_mode": r["include_mode"], - "day_filter_json": r["day_filter_json"], - "time_filter_json": r["time_filter_json"], - "productivity_filter_json": r["productivity_filter_json"], - "effective_from": r["effective_from"], - "created_at": r["def_created_at"], - # These are used by row_to_goal_definition() - "metric": r["metric"], - "operator": r["operator"], - "period": r["period"], - "timezone": r["timezone"], - } - definition = row_to_goal_definition(def_row) - - results_by_goal[goal_id] = { - "goal": goal_obj, - "definition": definition, - "include_tags": [], - "exclude_tags": [], - "latest_result": None, - "last_successes": [], - } - seen_tag_include[goal_id] = set() - seen_tag_exclude[goal_id] = set() - seen_success_rn[goal_id] = set() - - # Accumulate tags (dedupe) - if r["tag_id"] is not None: - tag = DBTag( - id=r["tag_id"], name=r["tag_name"], description=r["tag_description"] - ) - if r["tag_role"] == "include": - if r["tag_id"] not in seen_tag_include[goal_id]: - results_by_goal[goal_id]["include_tags"].append(tag) - seen_tag_include[goal_id].add(r["tag_id"]) - else: - if r["tag_id"] not in seen_tag_exclude[goal_id]: - results_by_goal[goal_id]["exclude_tags"].append(tag) - seen_tag_exclude[goal_id].add(r["tag_id"]) - - # Accumulate latest_result and last_successes from top_results rows - rn = r["res_rn"] - if rn is not None: - # Latest result (rn == 1) - if rn == 1 and results_by_goal[goal_id]["latest_result"] is None: - results_by_goal[goal_id]["latest_result"] = DBGoalResult( - id=r["res_id"], - goal_definition_id=def_id, - period_start=r["res_period_start"], - period_end=r["res_period_end"], - metric_value=r["res_metric_value"], - success=bool(r["res_success"]) - if r["res_success"] is not None - else False, - eval_state=EvalState(r["res_eval_state"]) - if r["res_eval_state"] - else EvalState.NA, - eval_state_reason=r["res_eval_state_reason"], - created_at=r["res_created_at"], - ) - # Last successes list (dedupe by rn to avoid tag fan-out) - if ( - last_successes_limit > 0 - and rn not in seen_success_rn[goal_id] - and len(results_by_goal[goal_id]["last_successes"]) - < last_successes_limit - ): - results_by_goal[goal_id]["last_successes"].append( - bool(r["res_success"]) if r["res_success"] is not None else False - ) - seen_success_rn[goal_id].add(rn) - - # Materialize final list in goal id order for stability - ordered_goal_ids = sorted(results_by_goal.keys()) - final_results: list[GoalWithLatestResult] = [] - for gid in ordered_goal_ids: - gdict = results_by_goal[gid] - final_results.append( - GoalWithLatestResult( - goal=gdict["goal"], - definition=gdict["definition"], - include_tags=gdict["include_tags"], - exclude_tags=gdict["exclude_tags"], - latest_result=gdict["latest_result"], - last_successes=gdict["last_successes"], - ) - ) - - return final_results - - -async def select_goal_progress_current( - conn: aiosqlite.Connection, goal_definition_id: int -) -> Optional[GoalProgressCurrent]: - """Fetch current progress row for a goal definition from goal_progress_current. - - Returns the row or None. Columns: goal_definition_id, period_start, period_end, - metric_value, success, eval_state, eval_state_reason, updated_at. - """ - query = """ - SELECT goal_definition_id, period_start, period_end, metric_value, - success, eval_state, eval_state_reason, updated_at - FROM goal_progress_current - WHERE goal_definition_id = ? - """ - async with conn.execute(query, (goal_definition_id,)) as cursor: - row = await cursor.fetchone() - if row is None: - return None - # Decide variant using the goal's metric from the definition - # For this query we need the goal metric; fetch from joined goal_definitions/goals - metric_q = """ - SELECT g.metric AS metric - FROM goal_definitions gd - JOIN goals g ON g.id = gd.goal_id - WHERE gd.id = ? - """ - async with conn.execute(metric_q, (goal_definition_id,)) as cur: - mrow = await cur.fetchone() - metric = ( - GoalMetric(mrow["metric"]) - if mrow is not None - else GoalMetric.AVG_PRODUCTIVITY_LEVEL - ) - - common = dict( - goal_definition_id=row["goal_definition_id"], - period_start=row["period_start"], - period_end=row["period_end"], - success=row["success"], - eval_state=EvalState(row["eval_state"]) - if row["eval_state"] is not None - else EvalState.NA, - eval_state_reason=row["eval_state_reason"], - updated_at=row["updated_at"], - ) - - if metric == GoalMetric.TOTAL_ACTIVITY_DURATION: - return TotalActivityDurationGoalProgressCurrent( - metric_value=float(row["metric_value"]), **common - ) # type: ignore[arg-type] - else: - return ProductivityGoalProgressCurrent( - metric_value=float(row["metric_value"]), **common - ) # type: ignore[arg-type] - - -async def upsert_goal_progress_current( - conn: aiosqlite.Connection, - *, - goal_definition_id: int, - period_start_utc: datetime, - period_end_utc: datetime, - metric_value: float, - success: Optional[bool], - eval_state: EvalState, - eval_state_reason: Optional[str], - updated_at_utc: datetime, -) -> None: - sql = """ - INSERT INTO goal_progress_current ( - goal_definition_id, period_start, period_end, metric_value, - success, eval_state, eval_state_reason, updated_at - ) VALUES (:goal_definition_id, :period_start, :period_end, :metric_value, - :success, :eval_state, :eval_state_reason, :updated_at) - ON CONFLICT(goal_definition_id) DO UPDATE SET - period_start = excluded.period_start, - period_end = excluded.period_end, - metric_value = excluded.metric_value, - success = excluded.success, - eval_state = excluded.eval_state, - eval_state_reason = excluded.eval_state_reason, - updated_at = excluded.updated_at - """ - await conn.execute( - sql, - { - "goal_definition_id": goal_definition_id, - "period_start": period_start_utc, - "period_end": period_end_utc, - "metric_value": metric_value, - "success": success, - "eval_state": eval_state.value, - "eval_state_reason": eval_state_reason, - "updated_at": updated_at_utc, - }, - ) - - -async def select_last_goal_results_for_goal( - conn: aiosqlite.Connection, *, goal_id: int, limit: int -) -> list[DBGoalResult]: - """Return the last N goal_results across all definitions for a goal. - - Results are ordered newest first by period_start. - """ - query = """ - SELECT r.id, r.goal_definition_id, r.period_start, r.period_end, - r.metric_value, r.success, r.eval_state, r.eval_state_reason, r.created_at - FROM goal_results r - JOIN goal_definitions d ON d.id = r.goal_definition_id - WHERE d.goal_id = ? - ORDER BY r.period_start DESC - LIMIT ? - """ - async with conn.execute(query, (goal_id, limit)) as cursor: - rows = await cursor.fetchall() - results: list[DBGoalResult] = [] - for rr in rows: - results.append( - DBGoalResult( - id=rr["id"], - goal_definition_id=rr["goal_definition_id"], - period_start=rr["period_start"], - period_end=rr["period_end"], - metric_value=rr["metric_value"], - success=bool(rr["success"]) if rr["success"] is not None else None, - eval_state=EvalState(rr["eval_state"]) - if rr["eval_state"] - else EvalState.NA, - eval_state_reason=rr["eval_state_reason"], - created_at=rr["created_at"], - ) - ) - return results - - -async def select_goal_with_latest_definition_by_definition_id( - conn: aiosqlite.Connection, *, goal_definition_id: int -) -> Optional[GoalWithLatestResult]: - # Fetch the definition row - async with conn.execute( - """ - SELECT gd.id, gd.goal_id, gd.name, gd.description, gd.metric_params_json, - gd.target_value, gd.include_mode, - gd.day_filter_json, gd.time_filter_json, gd.productivity_filter_json, - gd.effective_from, gd.created_at, - g.metric AS metric, g.operator AS operator, g.period AS period, g.timezone AS timezone, - g.id AS goal_id_real, - ( - SELECT at FROM goal_pause_events e - WHERE e.goal_id = g.id - ORDER BY datetime(at) DESC, id DESC - LIMIT 1 - ) AS goal_last_pause_event_at, - ( - SELECT event_type FROM goal_pause_events e - WHERE e.goal_id = g.id - ORDER BY datetime(at) DESC, id DESC - LIMIT 1 - ) AS goal_last_pause_event_type, - g.created_at AS goal_created_at - FROM goal_definitions gd - JOIN goals g ON g.id = gd.goal_id - WHERE gd.id = ? - """, - (goal_definition_id,), - ) as cursor: - dr = await cursor.fetchone() - if not dr: - return None - - # Build DBGoal from the joined row - metric = GoalMetric(dr["metric"]) # raises on invalid - operator = ( - MetricOperator(dr["operator"]) if dr["operator"] else MetricOperator.EQUAL - ) - period = GoalPeriod(dr["period"]) if dr["period"] else GoalPeriod.DAY - - # Coerce operator into allowed set - def _coerce_operator(m: GoalMetric, op: MetricOperator) -> MetricOperator: - if m == GoalMetric.AVG_PRODUCTIVITY_LEVEL: - return ( - op - if op in (MetricOperator.LESS_THAN, MetricOperator.GREATER_THAN) - else MetricOperator.GREATER_THAN - ) - if m == GoalMetric.TOTAL_ACTIVITY_DURATION: - return ( - op - if op in (MetricOperator.LESS_THAN, MetricOperator.GREATER_THAN) - else MetricOperator.GREATER_THAN - ) - return op - - operator = _coerce_operator(metric, operator) - paused_since_val = ( - dr["goal_last_pause_event_at"] - if dr["goal_last_pause_event_type"] == "pause" - else None - ) - if metric == GoalMetric.AVG_PRODUCTIVITY_LEVEL: - goal: DBGoal = DBAvgProductivityGoal( - id=dr["goal_id_real"], - created_at=dr["goal_created_at"], - paused_since=paused_since_val, - timezone=dr["timezone"], - period=period, - operator=cast(AvgProductivityOperators, operator), - ) - else: - goal = DBTotalActivityDurationGoal( - id=dr["goal_id_real"], - created_at=dr["goal_created_at"], - paused_since=paused_since_val, - timezone=dr["timezone"], - period=period, - operator=cast(TotalActivityDurationOperators, operator), - ) - # Fetch tags for this def - async with conn.execute( - """ - SELECT gt.tag_id, gt.role, t.name, t.description - FROM goal_tags gt - JOIN tags t ON t.id = gt.tag_id - WHERE gt.goal_definition_id = ? - """, - (goal_definition_id,), - ) as cursor: - tag_rows = await cursor.fetchall() - include_tags: list[DBTag] = [] - exclude_tags: list[DBTag] = [] - for tr in tag_rows: - tag = DBTag(id=tr["tag_id"], name=tr["name"], description=tr["description"]) - if tr["role"] == "INCLUDE": - include_tags.append(tag) - else: - exclude_tags.append(tag) - - definition = row_to_goal_definition(dr) - return GoalWithLatestResult( - goal=goal, - definition=definition, - include_tags=include_tags, - exclude_tags=exclude_tags, - latest_result=None, - last_successes=[], - ) - - -async def select_goal_with_definition_active_at( - conn: aiosqlite.Connection, *, goal_id: int, active_at_utc: datetime -) -> Optional[GoalWithLatestResult]: - """Return the goal and the definition in effect at the given UTC timestamp. - - Picks the goal_definition with the greatest effective_from <= active_at_utc. - Includes include/exclude tags for the chosen definition. - """ - # Choose the definition active at the provided timestamp - async with conn.execute( - """ - WITH chosen AS ( - SELECT gd.* - FROM goal_definitions gd - WHERE gd.goal_id = ? AND datetime(gd.effective_from) <= datetime(?) - ORDER BY gd.effective_from DESC - LIMIT 1 - ) - SELECT c.id, c.goal_id, c.name, c.description, c.metric_params_json, - c.target_value, c.include_mode, c.day_filter_json, c.time_filter_json, - c.productivity_filter_json, c.effective_from, c.created_at, - g.metric AS metric, g.operator AS operator, g.period AS period, - g.timezone AS timezone, - g.id AS goal_id_real, - ( - SELECT at FROM goal_pause_events e - WHERE e.goal_id = g.id - ORDER BY datetime(at) DESC, id DESC - LIMIT 1 - ) AS goal_last_pause_event_at, - ( - SELECT event_type FROM goal_pause_events e - WHERE e.goal_id = g.id - ORDER BY datetime(at) DESC, id DESC - LIMIT 1 - ) AS goal_last_pause_event_type, - g.created_at AS goal_created_at - FROM chosen c - JOIN goals g ON g.id = c.goal_id - """, - (goal_id, active_at_utc), - ) as cursor: - dr = await cursor.fetchone() - if not dr: - return None - - # Build DBGoal - metric = GoalMetric(dr["metric"]) # raises on invalid - operator = ( - MetricOperator(dr["operator"]) if dr["operator"] else MetricOperator.EQUAL - ) - period = GoalPeriod(dr["period"]) if dr["period"] else GoalPeriod.DAY - - def _coerce_operator(m: GoalMetric, op: MetricOperator) -> MetricOperator: - if m == GoalMetric.AVG_PRODUCTIVITY_LEVEL: - return ( - op - if op in (MetricOperator.LESS_THAN, MetricOperator.GREATER_THAN) - else MetricOperator.GREATER_THAN - ) - if m == GoalMetric.TOTAL_ACTIVITY_DURATION: - return ( - op - if op in (MetricOperator.LESS_THAN, MetricOperator.GREATER_THAN) - else MetricOperator.GREATER_THAN - ) - return op - - operator = _coerce_operator(metric, operator) - paused_since_val = ( - dr["goal_last_pause_event_at"] - if dr["goal_last_pause_event_type"] == "pause" - else None - ) - if metric == GoalMetric.AVG_PRODUCTIVITY_LEVEL: - goal: DBGoal = DBAvgProductivityGoal( - id=dr["goal_id_real"], - created_at=dr["goal_created_at"], - paused_since=paused_since_val, - timezone=dr["timezone"], - period=period, - operator=cast(AvgProductivityOperators, operator), - ) - else: - goal = DBTotalActivityDurationGoal( - id=dr["goal_id_real"], - created_at=dr["goal_created_at"], - paused_since=paused_since_val, - timezone=dr["timezone"], - period=period, - operator=cast(TotalActivityDurationOperators, operator), - ) - - # Tags for the chosen definition - async with conn.execute( - """ - SELECT gt.tag_id, gt.role, t.name, t.description - FROM goal_tags gt - JOIN tags t ON t.id = gt.tag_id - WHERE gt.goal_definition_id = ? - """, - (dr["id"],), - ) as cursor: - tag_rows = await cursor.fetchall() - include_tags: list[DBTag] = [] - exclude_tags: list[DBTag] = [] - for tr in tag_rows: - tag = DBTag(id=tr["tag_id"], name=tr["name"], description=tr["description"]) - if tr["role"] == "INCLUDE": - include_tags.append(tag) - else: - exclude_tags.append(tag) - - definition = row_to_goal_definition(dr) - return GoalWithLatestResult( - goal=goal, - definition=definition, - include_tags=include_tags, - exclude_tags=exclude_tags, - latest_result=None, - last_successes=[], - ) - - -async def insert_goal_pause_event( - conn: aiosqlite.Connection, *, goal_id: int, event_type: str, at: datetime -) -> int: - """Insert a pause/resume event for a goal. - - event_type must be 'pause' or 'resume'. Returns inserted row id. - """ - assert event_type in ("pause", "resume") - cursor = await conn.execute( - """ - INSERT INTO goal_pause_events (goal_id, event_type, at) - VALUES (:goal_id, :event_type, :at) - """, - {"goal_id": goal_id, "event_type": event_type, "at": at}, - ) - rowid = cursor.lastrowid - assert rowid is not None, "Failed to insert goal pause event" - return int(rowid) - - -async def select_latest_goal_definition_id_for_goal( - conn: aiosqlite.Connection, *, goal_id: int -) -> Optional[int]: - async with conn.execute( - """ - SELECT gd.id - FROM goal_definitions gd - WHERE gd.goal_id = ? - ORDER BY gd.created_at DESC - LIMIT 1 - """, - (goal_id,), - ) as cursor: - row = await cursor.fetchone() - return row["id"] if row else None - - -async def select_latest_progress_updated_at_for_goal( - conn: aiosqlite.Connection, *, goal_id: int -) -> Optional[datetime]: - async with conn.execute( - """ - SELECT gpc.updated_at AS updated_at - FROM goal_definitions gd - LEFT JOIN goal_progress_current gpc ON gpc.goal_definition_id = gd.id - WHERE gd.goal_id = ? - ORDER BY gd.created_at DESC - LIMIT 1 - """, - (goal_id,), - ) as cursor: - row = await cursor.fetchone() - return row["updated_at"] if row else None - - -async def insert_goal_result( - conn: aiosqlite.Connection, - *, - goal_definition_id: int, - period_start_utc: datetime, - period_end_utc: datetime, - metric_value: float, - success: Optional[bool], - eval_state: EvalState, - eval_state_reason: Optional[str], -) -> int: - """Insert or update a goal_results row for a definition and period. - - Uses ON CONFLICT(goal_definition_id, period_start) DO UPDATE to be idempotent. - Returns the lastrowid (may be 0 for updates in SQLite). - """ - async with conn.execute( - """ - INSERT INTO goal_results ( - goal_definition_id, period_start, period_end, metric_value, - success, eval_state, eval_state_reason - ) VALUES (:goal_definition_id, :period_start, :period_end, :metric_value, - :success, :eval_state, :eval_state_reason) - ON CONFLICT(goal_definition_id, period_start) DO UPDATE SET - period_end = excluded.period_end, - metric_value = excluded.metric_value, - success = excluded.success, - eval_state = excluded.eval_state, - eval_state_reason = excluded.eval_state_reason - """, - { - "goal_definition_id": goal_definition_id, - "period_start": period_start_utc, - "period_end": period_end_utc, - "metric_value": float(metric_value), - "success": (1 if success is True else 0 if success is False else None), - "eval_state": eval_state.value - if hasattr(eval_state, "value") - else str(eval_state), - "eval_state_reason": eval_state_reason, - }, - ) as cursor: - return cursor.lastrowid or 0 - - -def row_to_goal_definition(r: Any) -> DBGoalDefinition: - # Determine metric from joined row and map include mode and filters - metric = GoalMetric(r["metric"]) # type: ignore[arg-type] - include_mode = IncludeMode(r["include_mode"]) # type: ignore[arg-type] - day_filter = json.loads(r["day_filter_json"]) if r["day_filter_json"] else None - productivity_filter = None - if r["productivity_filter_json"]: - raw = json.loads(r["productivity_filter_json"]) # Expect list of strings - productivity_filter = [ProductivityLevel(v) for v in raw] - time_filter = json.loads(r["time_filter_json"]) if r["time_filter_json"] else None - - base_kwargs = { - "id": r["id"], - "goal_id": r["goal_id"], - "name": r["name"], - "description": r["description"], - "include_mode": include_mode, - "day_filter": day_filter, - "productivity_filter": productivity_filter, - "effective_from": r["effective_from"], - "time_filter": time_filter, - } - - match metric: - case GoalMetric.AVG_PRODUCTIVITY_LEVEL: - return DBAvgProductivityGoalDefinition( - **base_kwargs, - target_value=r["target_value"], - metric_params=None, - ) - case GoalMetric.TOTAL_ACTIVITY_DURATION: - # Interpret target_value as seconds for duration - seconds = float(r["target_value"]) if r["target_value"] is not None else 0.0 - return DBTotalActivityDurationGoalDefinition( - **base_kwargs, - target_value=timedelta(seconds=seconds), - metric_params=None, - ) - case _: - raise ValueError(f"Unknown goal metric: {metric}") - - -# ---- Goal inserts ---- -async def insert_goal( - conn: aiosqlite.Connection, - *, - metric: GoalMetric, - operator: MetricOperator, - period: str, - timezone: str, -) -> int: - sql = simple_insert_query("goals", ("metric", "operator", "period", "timezone")) - cursor = await conn.execute( - sql, - { - "metric": metric.value, - "operator": operator.value, - "period": period, - "timezone": timezone, - }, - ) - last_row_id = cursor.lastrowid - assert last_row_id is not None, "No goal row was inserted" - return int(last_row_id) - - -async def insert_goal_definition( - conn: aiosqlite.Connection, - *, - goal_id: int, - name: str, - description: str | None, - target_value: float, - include_mode: IncludeMode, - day_filter_json: str | None, - time_filter_json: str | None, - productivity_filter_json: str | None, - effective_from: datetime, - metric_params_json: str | None = None, -) -> int: - sql = simple_insert_query( - "goal_definitions", - ( - "goal_id", - "name", - "description", - "metric_params_json", - "target_value", - "include_mode", - "day_filter_json", - "time_filter_json", - "productivity_filter_json", - "effective_from", - ), - ) - cursor = await conn.execute( - sql, - { - "goal_id": goal_id, - "name": name, - "description": description, - "metric_params_json": metric_params_json, - "target_value": target_value, - "include_mode": include_mode.value, - "day_filter_json": day_filter_json, - "time_filter_json": time_filter_json, - "productivity_filter_json": productivity_filter_json, - "effective_from": effective_from, - }, - ) - last_row_id = cursor.lastrowid - assert last_row_id is not None, "No goal_definition row was inserted" - return int(last_row_id) - - -async def insert_goal_tags( - conn: aiosqlite.Connection, - *, - goal_definition_id: int, - include_tag_ids: list[int], - exclude_tag_ids: list[int], -) -> None: - if not include_tag_ids and not exclude_tag_ids: - return - rows: list[dict[str, Any]] = [] - for tid in include_tag_ids: - rows.append( - { - "goal_definition_id": goal_definition_id, - "tag_id": tid, - "role": "include", - } - ) - for tid in exclude_tag_ids: - rows.append( - { - "goal_definition_id": goal_definition_id, - "tag_id": tid, - "role": "exclude", - } - ) - sql = simple_insert_query("goal_tags", ("goal_definition_id", "tag_id", "role")) - await conn.executemany(sql, rows) - - -async def delete_goal(conn: aiosqlite.Connection, goal_id: int) -> None: - """Delete a goal by ID. Cascades will remove related definitions, tags, and results. - - Assumes PRAGMA foreign_keys=ON in the connection (set by the DB context manager). - """ - await conn.execute("DELETE FROM goals WHERE id = ?", (goal_id,)) - - -async def update_activity_event( - conn: aiosqlite.Connection, db_activity_event: DBActivityEvent -) -> None: - assert db_activity_event.id is not None, "Event ID cannot be None" - query = """ - UPDATE activity_events - SET event_time = :event_time, - event_type = :event_type, - aggregation_id = :aggregation_id, - last_manual_action_time = :last_manual_action_time - WHERE id = :id; - """ - await conn.execute( - query, - { - "event_time": db_activity_event.event_time, - "event_type": db_activity_event.event_type, - "aggregation_id": db_activity_event.aggregation_id, - "id": db_activity_event.id, - "last_manual_action_time": db_activity_event.last_manual_action_time, - }, - ) - - -async def select_specs_in_time_range( - conn: aiosqlite.Connection, start: datetime, end: datetime -) -> list[DBActivitySpec]: - activity_ids_query = """ - SELECT DISTINCT a.id - FROM activities a - WHERE ( - EXISTS ( - SELECT 1 - FROM activity_events ae - WHERE ae.activity_id = a.id - AND ae.event_type = 'open' - AND datetime(ae.event_time) >= datetime(:start) - AND datetime(ae.event_time) <= datetime(:end) - ) - OR ( - SELECT ae.event_type - FROM activity_events ae - WHERE ae.activity_id = a.id - AND datetime(ae.event_time) < datetime(:start) - ORDER BY ae.event_time DESC, ae.id DESC - LIMIT 1 - ) = 'open' - ) - """ - - async with conn.execute(activity_ids_query, {"start": start, "end": end}) as cursor: - rows = await cursor.fetchall() - activity_ids = [row["id"] for row in rows] - - if not activity_ids: - return [] - - placeholders = ",".join("?" for _ in activity_ids) - specs_query = f""" - SELECT - a.id AS activity_id, - a.name, - a.description, - a.productivity_level, - a.source, - a.last_manual_action_time AS activity_last_manual_action_time, - e.id AS event_id, - e.event_time, - e.event_type, - e.aggregation_id, - e.last_manual_action_time AS event_last_manual_action_time - FROM activities a - JOIN activity_events e ON a.id = e.activity_id - WHERE a.id IN ({placeholders}) - ORDER BY a.id, datetime(e.event_time), e.id - """ - - async with conn.execute(specs_query, activity_ids) as cursor: - rows = await cursor.fetchall() - - grouped: dict[int, list[Any]] = defaultdict(list) - for row in rows: - grouped[row["activity_id"]].append(row) - - specs: list[DBActivitySpec] = [] - for activity_id, activity_rows in grouped.items(): - first_row = activity_rows[0] - db_activity = DBActivity( - id=activity_id, - name=first_row["name"], - description=first_row["description"], - productivity_level=first_row["productivity_level"], - source=first_row["source"], - last_manual_action_time=first_row["activity_last_manual_action_time"], - ) - - events: list[DBActivityEvent] = [] - for row in activity_rows: - events.append( - DBActivityEvent( - event_time=row["event_time"], - event_type=row["event_type"], - id=row["event_id"], - activity_id=activity_id, - aggregation_id=row["aggregation_id"], - last_manual_action_time=row["event_last_manual_action_time"], - ) - ) - - specs.append(DBActivitySpec(activity=db_activity, events=events)) - - return specs - - -async def select_specs_with_tags_in_time_range( - conn: aiosqlite.Connection, start: datetime, end: datetime -) -> list[DBActivitySpecWithTags]: - # First get regular activity specs - activity_specs = await select_specs_in_time_range(conn, start, end) - - # Now fetch tags for each activity - tags_by_activity = {} - - # Get all activity IDs to look up tags - activity_ids = [ - spec.activity.id for spec in activity_specs if spec.activity.id is not None - ] - - if not activity_ids: - # Return early if there are no activities - return [ - DBActivitySpecWithTags(**spec.model_dump(), tags=[]) - for spec in activity_specs - ] - - # Create placeholders for SQL query - placeholders = ",".join("?" for _ in activity_ids) - - # Query to get all tags for these activities - query = f""" - SELECT a.id AS activity_id, t.id AS tag_id, t.name, t.description - FROM activities a - LEFT JOIN tag_mappings tm ON a.id = tm.activity_id - LEFT JOIN tags t ON tm.tag_id = t.id AND t.deleted_at IS NULL - WHERE a.id IN ({placeholders}) - """ - - async with conn.execute(query, activity_ids) as cursor: - rows = await cursor.fetchall() - - # Group tags by activity - for row in rows: - activity_id = row["activity_id"] - tag_id = row["tag_id"] - - if activity_id not in tags_by_activity: - tags_by_activity[activity_id] = [] - if tag_id is not None: - tag = DBTag(id=tag_id, name=row["name"], description=row["description"]) - tags_by_activity[activity_id].append(tag) - - # Combine activities with their tags - result = [] - for spec in activity_specs: - activity_id = spec.activity.id - tags = tags_by_activity.get(activity_id, []) - - # Create enhanced spec with tags - spec_with_tags = DBActivitySpecWithTags(**spec.model_dump(), tags=tags) - result.append(spec_with_tags) - - return result - - -async def select_last_aggregation(conn: aiosqlite.Connection) -> DBAggregation | None: - query = """ - SELECT id, start_time, end_time, first_timestamp, last_timestamp - FROM aggregations - ORDER BY end_time DESC - LIMIT 1; - """ - async with conn.execute(query) as cursor: - row = await cursor.fetchone() - if row: - return DBAggregation( - id=row["id"], - start_time=row["start_time"], - end_time=row["end_time"], - first_timestamp=row["first_timestamp"], - last_timestamp=row["last_timestamp"], - ) - - -async def select_latest_aggregation( - conn: aiosqlite.Connection, -) -> DBAggregation | None: - query = """ - SELECT id, start_time, end_time, first_timestamp, last_timestamp - FROM aggregations - ORDER BY end_time DESC - LIMIT 1; - """ - async with conn.execute(query) as cursor: - row = await cursor.fetchone() - if row: - return DBAggregation( - id=row["id"], - start_time=row["start_time"], - end_time=row["end_time"], - first_timestamp=row["first_timestamp"], - last_timestamp=row["last_timestamp"], - ) - return None - - -async def select_sources(conn: aiosqlite.Connection) -> list[DBDeviceSource]: - query = """ - SELECT id, name, source_type, status, token_hash, last_seen, created_at - FROM sources - ORDER BY (last_seen IS NULL), datetime(last_seen) DESC, id DESC - """ - async with conn.execute(query) as cursor: - rows = await cursor.fetchall() - result: list[DBDeviceSource] = [] - for row in rows: - result.append( - DBDeviceSource( - id=row["id"], - name=row["name"], - source_type=SourceType(row["source_type"]), - status=SourceStatus(row["status"]), - token_hash=row["token_hash"], - last_seen=row["last_seen"], - created_at=row["created_at"], - ) - ) - return result - - -async def toggle_source_status( - conn: aiosqlite.Connection, source_id: int -) -> SourceStatus: - status = await conn.execute( - """ - UPDATE sources - SET status = CASE - WHEN status = :active THEN :revoked - ELSE :active - END - WHERE id = :source_id - RETURNING status - """, - { - "source_id": source_id, - "active": SourceStatus.ACTIVE.value, - "revoked": SourceStatus.REVOKED.value, - }, - ) - - row = await status.fetchone() - assert row is not None, "No source found with the given ID" - - return SourceStatus(row["status"]) - - -async def delete_source( - conn: aiosqlite.Connection, source_id: int -) -> DBDeviceSource | None: - async with conn.execute( - """ - DELETE FROM sources - WHERE id = :source_id - RETURNING id, name, source_type, status, token_hash, last_seen, created_at - """, - {"source_id": source_id}, - ) as cursor: - row = await cursor.fetchone() - if not row: - return None - return DBDeviceSource( - id=row["id"], - name=row["name"], - source_type=SourceType(row["source_type"]), - status=SourceStatus(row["status"]), - token_hash=row["token_hash"], - last_seen=row["last_seen"], - created_at=row["created_at"], - ) - - -async def insert_source( - conn: aiosqlite.Connection, - *, - name: str, - source_type: SourceType, - token_hash: str, - status: SourceStatus = SourceStatus.ACTIVE, -) -> DBDeviceSource: - await conn.execute( - """ - INSERT INTO sources (name, source_type, token_hash, status) - VALUES (:name, :source_type, :token_hash, :status) - """, - { - "name": name, - "source_type": source_type.value, - "token_hash": token_hash, - "status": status.value, - }, - ) - # Fetch the row we just inserted - async with conn.execute( - """ - SELECT id, name, source_type, status, token_hash, last_seen, created_at - FROM sources WHERE name = :name - """, - {"name": name}, - ) as cursor: - row = await cursor.fetchone() - assert row is not None, "Failed to fetch inserted source" - return DBDeviceSource( - id=row["id"], - name=row["name"], - source_type=SourceType(row["source_type"]), - status=SourceStatus(row["status"]), - token_hash=row["token_hash"], - last_seen=row["last_seen"], - created_at=row["created_at"], - ) - - -async def insert_source_enrollment_code( - conn: aiosqlite.Connection, - *, - code_hash: str, - expires_at: datetime | None, -) -> int: - cursor = await conn.execute( - """ - INSERT INTO source_enrollment_codes (code_hash, expires_at, used) - VALUES (:code_hash, :expires_at, 0) - """, - {"code_hash": code_hash, "expires_at": expires_at}, - ) - last_id = cursor.lastrowid - return int(last_id) if last_id is not None else 0 - - -async def deactivate_active_enrollment_codes(conn: aiosqlite.Connection) -> None: - # Remove any existing enrollment codes so only one can exist at a time. - await conn.execute("DELETE FROM source_enrollment_codes") - - -async def select_current_enrollment_code( - conn: aiosqlite.Connection, -) -> Optional[Any]: - async with conn.execute( - "SELECT id, code_hash, expires_at FROM source_enrollment_codes LIMIT 1" - ) as cursor: - row = await cursor.fetchone() - - if row is None: - return None - - return SourceEnrollmentCode( - id=row["id"], code_hash=row["code_hash"], expires_at=row["expires_at"] - ) - - -async def delete_current_enrollment_code(conn: aiosqlite.Connection) -> None: - await conn.execute("DELETE FROM source_enrollment_codes") - - -async def select_source_by_token_hash( - conn: aiosqlite.Connection, token_hash: str -) -> Optional[DBDeviceSource]: - async with conn.execute( - """ - SELECT id, name, source_type, status, token_hash, last_seen, created_at - FROM sources WHERE token_hash = :token_hash - """, - {"token_hash": token_hash}, - ) as cursor: - row = await cursor.fetchone() - if not row: - return None - return DBDeviceSource( - id=row["id"], - name=row["name"], - source_type=SourceType(row["source_type"]), - status=SourceStatus(row["status"]), - token_hash=row["token_hash"], - last_seen=row["last_seen"], - created_at=row["created_at"], - ) - - -async def update_source_last_seen( - conn: aiosqlite.Connection, source_id: int, *, when: datetime -) -> None: - await conn.execute( - "UPDATE sources SET last_seen = :when WHERE id = :id", - {"id": source_id, "when": when}, - ) - - -async def select_auto_activities_closed_within_time_range_with_last_event( - conn: aiosqlite.Connection, start: datetime, end: datetime -) -> list[DBActivityWithLatestEvent]: - query = """ - SELECT - a.id, a.name, a.description, a.productivity_level, a.source, - a.last_manual_action_time AS activity_last_manual_action_time, - le.event_type AS state, - le.event_time, - le.aggregation_id, - le.id AS last_event_id, - ag.start_time as aggregation_start_time, - ag.end_time as aggregation_end_time, - ag.first_timestamp as aggregation_first_timestamp, - ag.last_timestamp as aggregation_last_timestamp, - le.last_manual_action_time AS event_last_manual_action_time - - FROM activities a - - JOIN ( - SELECT * - FROM activity_events ae - WHERE ae.id = ( - SELECT ae2.id - FROM activity_events ae2 - WHERE ae2.activity_id = ae.activity_id - ORDER BY ae2.event_time DESC - LIMIT 1 - ) - ) le ON le.activity_id = a.id - LEFT JOIN aggregations ag ON le.aggregation_id = ag.id - WHERE a.source = 'auto' - AND le.event_type = 'close' - AND datetime(le.event_time) >= datetime(?) - AND datetime(le.event_time) <= datetime(?); - """ - - return_list = [] - async with conn.execute(query, (start, end)) as cursor: - rows = await cursor.fetchall() - for row in rows: - db_activity = DBActivity( - id=row["id"], - name=row["name"], - description=row["description"], - productivity_level=row["productivity_level"], - source=row["source"], - last_manual_action_time=row["activity_last_manual_action_time"], - ) - - db_activity_event = DBActivityEvent( - event_time=row["event_time"], - event_type=row["state"], - id=row["last_event_id"], - activity_id=row["id"], - aggregation_id=row["aggregation_id"], - last_manual_action_time=row["event_last_manual_action_time"], - ) - latest_aggregation = DBAggregation( - id=row["aggregation_id"], - start_time=row["aggregation_start_time"], - end_time=row["aggregation_end_time"], - first_timestamp=row["aggregation_first_timestamp"], - last_timestamp=row["aggregation_last_timestamp"], - ) - return_list.append( - DBActivityWithLatestEvent( - activity=db_activity, - latest_event=db_activity_event, - latest_aggregation=latest_aggregation, - ) - ) - return return_list - - -async def select_manual_activity_specs_active_within_time_range( - conn: aiosqlite.Connection, start: datetime, end: datetime -) -> list[DBActivitySpec]: - """ - Fetches all manual activities specs that are active within a given time range. - An activity is considered active if it's a manual activity and: - 1. It has an 'open' event within the specified time range (inclusive). - 2. Its last event before the start of the time range was 'open'. - """ - activity_ids_query = """ - SELECT DISTINCT a.id - FROM activities a - WHERE a.source = 'manual' - AND ( - -- Condition 1: Has an open event within the time range - EXISTS ( - SELECT 1 - FROM activity_events ae - WHERE ae.activity_id = a.id - AND ae.event_time >= :start - AND ae.event_time <= :end - AND ae.event_type = 'open' - ) - OR - -- Condition 2: Last event before start is 'open' - ( - SELECT ae.event_type - FROM activity_events ae - WHERE ae.activity_id = a.id AND ae.event_time < :start - ORDER BY ae.event_time DESC - LIMIT 1 - ) = 'open' - ) - """ - async with conn.execute(activity_ids_query, {"start": start, "end": end}) as cursor: - rows = await cursor.fetchall() - activity_ids = [row["id"] for row in rows] - - if not activity_ids: - return [] - - # Now fetch the full specs for these activities. - placeholders = ",".join("?" for _ in activity_ids) - specs_query = f""" - SELECT a.id AS activity_id, - a.name, - a.description, - a.productivity_level, - a.source, - a.last_manual_action_time AS activity_last_manual_action_time, - e.id AS event_id, - e.event_time, - e.event_type, - e.aggregation_id, - e.last_manual_action_time AS event_last_manual_action_time - FROM activities a - JOIN activity_events e ON a.id = e.activity_id - WHERE a.id IN ({placeholders}); - """ - - async with conn.execute(specs_query, activity_ids) as cursor: - rows = await cursor.fetchall() - - grouped = defaultdict(list) - for row in rows: - activity_id = row["activity_id"] - grouped[activity_id].append(row) - - activity_specs = [] - for activity_id, activity_rows in grouped.items(): - first_row = activity_rows[0] - - db_activity = DBActivity( - id=activity_id, - name=first_row["name"], - description=first_row["description"], - productivity_level=first_row["productivity_level"], - source=first_row["source"], - last_manual_action_time=first_row["activity_last_manual_action_time"], - ) - - events = [] - for row in activity_rows: - db_activity_event = DBActivityEvent( - event_time=row["event_time"], - event_type=row["event_type"], - id=row["event_id"], - activity_id=activity_id, - aggregation_id=row["aggregation_id"], - last_manual_action_time=row["event_last_manual_action_time"], - ) - events.append(db_activity_event) - - db_activity_spec = DBActivitySpec(activity=db_activity, events=events) - activity_specs.append(db_activity_spec) - - return activity_specs - - -async def insert_candidate_sessions( - conn: aiosqlite.Connection, - *, - sessionization_run_id: int, - sessions: Sequence[CandidateSession], -) -> list[int]: - if not sessions: - return [] - - placeholders = ", ".join("(?, ?, ?)" for _ in sessions) - values: list[Any] = [] - for session in sessions: - values.extend([session.name, session.llm_id, sessionization_run_id]) - - sql = f""" - INSERT INTO candidate_sessions ( - name, - llm_id, - sessionization_run_id - ) - VALUES {placeholders} - RETURNING - id - """ - - async with conn.execute(sql, values) as cursor: - rows = await cursor.fetchall() - - return [int(row["id"]) for row in rows] - - -async def insert_candidate_session_to_activity( - conn: aiosqlite.Connection, - *, - mappings: Sequence[CandidateSessionToActivity], -) -> None: - if not mappings: - return - - await conn.executemany( - """ - INSERT INTO candidate_session_to_activity ( - session_id, - activity_id - ) - VALUES (?, ?) - """, - [(mapping.candidate_session_id, mapping.activity_id) for mapping in mappings], - ) - - -async def delete_candidate_session_to_activity_by_activity_ids( - conn: aiosqlite.Connection, - *, - activity_ids: Sequence[int], -) -> None: - # Delete candidate session mappings for the given activity ids in a single query. - if not activity_ids: - return - - placeholders = ",".join("?" for _ in activity_ids) - await conn.execute( - f""" - DELETE FROM candidate_session_to_activity - WHERE activity_id IN ({placeholders}) - """, - list(activity_ids), - ) - - -async def delete_candidate_sessions_by_ids( - conn: aiosqlite.Connection, - *, - candidate_session_ids: Sequence[int], -) -> None: - if not candidate_session_ids: - return - - placeholders = ",".join("?" for _ in candidate_session_ids) - await conn.execute( - f""" - DELETE FROM candidate_sessions - WHERE id IN ({placeholders}) - """, - list(candidate_session_ids), - ) - - -async def delete_candidate_sessions_without_activities( - conn: aiosqlite.Connection, -) -> int: - # Delete all candidate sessions that have no activity mappings. Returns the number of rows deleted. - cursor = await conn.execute( - """ - DELETE FROM candidate_sessions - WHERE id NOT IN ( - SELECT DISTINCT session_id - FROM candidate_session_to_activity - ) - """ - ) - return cursor.rowcount or 0 - - -async def select_candidate_session_specs( - conn: aiosqlite.Connection, - *, - sessionization_run_id: int | None = None, -) -> list[DBCandidateSessionSpec]: - where_clause = "" - params: list[Any] = [] - if sessionization_run_id is not None: - where_clause = "WHERE cs.sessionization_run_id = ?" - params.append(sessionization_run_id) - - async with conn.execute( - f""" - SELECT - cs.id, - cs.name, - cs.llm_id, - cs.sessionization_run_id, - cs.created_at, - ( - SELECT GROUP_CONCAT(sub.activity_id, ',') - FROM ( - SELECT activity_id - FROM candidate_session_to_activity csta - WHERE csta.session_id = cs.id - ORDER BY datetime(csta.created_at) ASC, csta.activity_id ASC - ) AS sub - ) AS activity_ids_csv - FROM candidate_sessions cs - {where_clause} - ORDER BY datetime(cs.created_at) ASC, cs.id ASC - """, - params, - ) as cursor: - rows = await cursor.fetchall() - - specs: list[DBCandidateSessionSpec] = [] - for row in rows: - activity_ids_csv = row["activity_ids_csv"] - activity_ids = ( - [int(value) for value in activity_ids_csv.split(",") if value] - if activity_ids_csv - else [] - ) - - specs.append( - DBCandidateSessionSpec( - session=DBCandidateSession( - id=row["id"], - name=row["name"], - llm_id=row["llm_id"], - sessionization_run_id=row["sessionization_run_id"], - created_at=row["created_at"], - ), - activity_ids=activity_ids, - ) - ) - - return specs - - -async def insert_sessionization_run( - conn: aiosqlite.Connection, - *, - sessionization_run: SessionizationRun, -) -> int: - cursor = await conn.execute( - """ - INSERT INTO sessionization_run ( - candidate_creation_start, - candidate_creation_end, - overlap_start, - right_tail_end, - finalized_horizon - ) - VALUES (?, ?, ?, ?, ?) - RETURNING - id - """, - ( - sessionization_run.candidate_creation_start, - sessionization_run.candidate_creation_end, - sessionization_run.overlap_start, - sessionization_run.right_tail_end, - sessionization_run.finalized_horizon, - ), - ) - - row = await cursor.fetchone() - assert row is not None, "sessionization_run insert did not return a row" - - return row["id"] - - -async def select_latest_sessionization_run( - conn: aiosqlite.Connection, -) -> DBSessionizationRun | None: - async with conn.execute( - """ - SELECT - id, - created_at, - candidate_creation_start, - candidate_creation_end, - overlap_start, - right_tail_end, - finalized_horizon - FROM sessionization_run - ORDER BY created_at DESC, id DESC - LIMIT 1 - """ - ) as cursor: - row = await cursor.fetchone() - - if row is None: - return None - - return DBSessionizationRun( - id=row["id"], - created_at=row["created_at"], - candidate_creation_start=row["candidate_creation_start"], - candidate_creation_end=row["candidate_creation_end"], - overlap_start=row["overlap_start"], - right_tail_end=row["right_tail_end"], - finalized_horizon=row["finalized_horizon"], - ) - - -async def insert_sessions( - conn: aiosqlite.Connection, - *, - sessionization_run_id: int, - sessions: Sequence[Session], -) -> list[int]: - if not sessions: - return [] - - insert_sql = """ - INSERT INTO sessions ( - name, - llm_id, - sessionization_run_id - ) - VALUES (?, ?, ?) - """ - - inserted_ids: list[int] = [] - for session in sessions: - cursor = await conn.execute( - insert_sql, - ( - session.name, - session.llm_id, - sessionization_run_id, - ), - ) - last_row_id = cursor.lastrowid - assert last_row_id is not None, "sessions insert did not return an id" - inserted_ids.append(last_row_id) - - return inserted_ids - - -async def insert_session_to_activity( - conn: aiosqlite.Connection, - mappings: Sequence[tuple[int, int]], -) -> None: - if not mappings: - return - - await conn.executemany( - """ - INSERT INTO session_to_activity (session_id, activity_id) - VALUES (?, ?) - """, - mappings, - ) - - -async def select_specs_with_tags_and_sessions_in_time_range( - conn: aiosqlite.Connection, start: datetime, end: datetime -) -> list[DBActivitySpecWithTagsAndSessions]: - """Get activity specs with tags, finalized sessions, and candidate sessions in the given time range. - - Uses the same activity filtering criteria as select_specs_with_tags_in_time_range. - """ - # First get regular activity specs - activity_specs = await select_specs_in_time_range(conn, start, end) - - if not activity_specs: - return [] - - # Get all activity IDs - activity_ids = [ - spec.activity.id for spec in activity_specs if spec.activity.id is not None - ] - - if not activity_ids: - return [] - - placeholders = ",".join("?" for _ in activity_ids) - - # Query to get all tags, sessions, and candidate sessions for these activities - query = f""" - SELECT - a.id AS activity_id, - -- Tags - t.id AS tag_id, - t.name AS tag_name, - t.description AS tag_description, - -- Finalized sessions (can only be one per activity due to UNIQUE constraint) - s.id AS session_id, - s.name AS session_name, - s.llm_id AS session_llm_id, - s.created_at AS session_created_at, - s.sessionization_run_id AS session_run_id, - -- Candidate sessions (can be multiple) - cs.id AS candidate_session_id, - cs.name AS candidate_session_name, - cs.llm_id AS candidate_session_llm_id, - cs.created_at AS candidate_session_created_at, - cs.sessionization_run_id AS candidate_session_run_id - FROM activities a - LEFT JOIN tag_mappings tm ON a.id = tm.activity_id - LEFT JOIN tags t ON tm.tag_id = t.id AND t.deleted_at IS NULL - LEFT JOIN session_to_activity sta ON a.id = sta.activity_id - LEFT JOIN sessions s ON sta.session_id = s.id - LEFT JOIN candidate_session_to_activity csta ON a.id = csta.activity_id - LEFT JOIN candidate_sessions cs ON csta.session_id = cs.id - WHERE a.id IN ({placeholders}) - """ - - async with conn.execute(query, activity_ids) as cursor: - rows = await cursor.fetchall() - - # Group data by activity - tags_by_activity: dict[int, list[DBTag]] = defaultdict(list) - session_by_activity: dict[int, DBSession] = {} - candidate_sessions_by_activity: dict[int, list[DBCandidateSession]] = defaultdict( - list - ) - - for row in rows: - activity_id = row["activity_id"] - - # Collect tags - tag_id = row["tag_id"] - if tag_id is not None: - tag = DBTag( - id=tag_id, name=row["tag_name"], description=row["tag_description"] - ) - # Avoid duplicates - if tag not in tags_by_activity[activity_id]: - tags_by_activity[activity_id].append(tag) - - # Collect finalized session (only one per activity) - session_id = row["session_id"] - if session_id is not None and activity_id not in session_by_activity: - session_by_activity[activity_id] = DBSession( - id=session_id, - name=row["session_name"], - llm_id=row["session_llm_id"], - created_at=row["session_created_at"], - sessionization_run_id=row["session_run_id"], - ) - - # Collect candidate sessions (can be multiple) - candidate_session_id = row["candidate_session_id"] - if candidate_session_id is not None: - candidate_session = DBCandidateSession( - id=candidate_session_id, - name=row["candidate_session_name"], - llm_id=row["candidate_session_llm_id"], - created_at=row["candidate_session_created_at"], - sessionization_run_id=row["candidate_session_run_id"], - ) - # Avoid duplicates - if candidate_session not in candidate_sessions_by_activity[activity_id]: - candidate_sessions_by_activity[activity_id].append(candidate_session) - - # Combine activities with their relationships - result = [] - for spec in activity_specs: - activity_id = spec.activity.id - tags = tags_by_activity.get(activity_id, []) - session = session_by_activity.get(activity_id, None) - candidate_sessions = candidate_sessions_by_activity.get(activity_id, []) - - # Create enhanced spec with tags and sessions - spec_with_relationships = DBActivitySpecWithTagsAndSessions( - **spec.model_dump(), - tags=tags, - session=session, - candidate_sessions=candidate_sessions, - ) - result.append(spec_with_relationships) - - return result +from __future__ import annotations + +from collections import defaultdict +from datetime import datetime, timedelta +from functools import lru_cache +import json +from typing import Any, List, Optional, Sequence, cast + +import aiosqlite +from loguru import logger + +from clepsy.auth.auth import decrypt_secret +from clepsy.config import config +from clepsy.entities import ( + AADS, + Activity, + ActivityEventInsert, + Aggregation, + AnthropicConfig, + AvgProductivityOperators, + CandidateSession, + CandidateSessionToActivity, + DBActivity, + DBActivityEvent, + DBActivitySpec, + DBActivitySpecWithTags, + DBActivitySpecWithTagsAndSessions, + DBActivityWithLatestEvent, + DBAggregation, + DBAvgProductivityGoal, + DBAvgProductivityGoalDefinition, + DBCandidateSession, + DBCandidateSessionSpec, + DBDeviceSource, + DBGoal, + DBGoalDefinition, + DBGoalResult, + DBScheduledJob, + DBSession, + DBSessionizationRun, + DBTag, + DBTotalActivityDurationGoal, + DBTotalActivityDurationGoalDefinition, + EvalState, + GoalMetric, + GoalPeriod, + GoalProgressCurrent, + GoalWithLatestResult, + GoogleAIConfig, + ImageProcessingApproach, + IncludeMode, + JobType, + MetricOperator, + ModelProvider, + OpenAIConfig, + OpenAIGenericConfig, + ProductivityGoalProgressCurrent, + ProductivityLevel, + ScheduledJob, + ScheduleStatus, + Session, + SessionizationRun, + SourceEnrollmentCode, + SourceStatus, + SourceType, + Tag, + TagMapping, + TotalActivityDurationGoalProgressCurrent, + TotalActivityDurationOperators, + UserSettings, +) + + +async def get_filtered_activity_specs_for_goal( + conn: aiosqlite.Connection, + start: datetime, + end: datetime, + include_tag_ids: Optional[List[int]] = None, + exclude_tag_ids: Optional[List[int]] = None, + include_mode: str = "any", + productivity_levels: Optional[List[str]] = None, + day_filter: Optional[List[str]] = None, + time_filter: Optional[List[tuple[str, str]]] = None, +) -> List[DBActivitySpecWithTags]: + """ + Get activity specs with tags, filtered by time, tags, productivity, day, and time filters as much as possible in SQL. + - include_tag_ids: only include activities with these tags (all or any, per include_mode) + - exclude_tag_ids: exclude activities with any of these tags + - productivity_levels: only include activities with these productivity levels + - day_filter: only include activities with events on these weekdays (e.g. ["monday", ...]) + - time_filter: only include activities with events overlapping these time ranges (list of (start, end) in HH:MM:SS) + """ + # Build base query + query = """ + SELECT a.id AS activity_id, a.name, a.description, a.productivity_level, a.source, a.last_manual_action_time AS activity_last_manual_action_time, + e.id AS event_id, e.event_time, e.event_type, e.aggregation_id, e.last_manual_action_time AS event_last_manual_action_time + FROM activities a + JOIN activity_events e ON a.id = e.activity_id + LEFT JOIN tag_mappings tm ON a.id = tm.activity_id + LEFT JOIN tags t ON tm.tag_id = t.id AND t.deleted_at IS NULL + WHERE datetime(e.event_time) >= datetime(?) AND datetime(e.event_time) <= datetime(?) + """ + params: list[Any] = [start, end] + # Productivity filter + if productivity_levels: + query += ( + " AND a.productivity_level IN (" + + ",".join(["?"] * len(productivity_levels)) + + ")" + ) + params.extend(productivity_levels) + # Exclude tags + if exclude_tag_ids: + query += ( + " AND a.id NOT IN (SELECT activity_id FROM tag_mappings WHERE tag_id IN (" + + ",".join(["?"] * len(exclude_tag_ids)) + + "))" + ) + params.extend(exclude_tag_ids) + # Include tags (any or all) + if include_tag_ids: + if include_mode == "all": + for tag_id in include_tag_ids: + query += " AND a.id IN (SELECT activity_id FROM tag_mappings WHERE tag_id = ?)" + params.append(tag_id) + else: + query += ( + " AND a.id IN (SELECT activity_id FROM tag_mappings WHERE tag_id IN (" + + ",".join(["?"] * len(include_tag_ids)) + + "))" + ) + params.extend(include_tag_ids) + # Day filter (event weekday) + if day_filter: + # SQLite: strftime('%w', e.event_time) gives 0=Sunday ... 6=Saturday + weekday_map = { + "sunday": 0, + "monday": 1, + "tuesday": 2, + "wednesday": 3, + "thursday": 4, + "friday": 5, + "saturday": 6, + } + weekday_nums = [ + str(weekday_map[d.lower()]) for d in day_filter if d.lower() in weekday_map + ] + if weekday_nums: + query += ( + " AND CAST(strftime('%w', e.event_time) AS INTEGER) IN (" + + ",".join(["?"] * len(weekday_nums)) + + ")" + ) + params.extend(weekday_nums) + # Time filter (event time overlaps any range) + if time_filter: + # Each time range is (start, end) in HH:MM:SS + time_clauses = [] + for start_str, end_str in time_filter: + time_clauses.append("(time(e.event_time) >= ? AND time(e.event_time) <= ?)") + params.extend([start_str, end_str]) + if time_clauses: + query += " AND (" + " OR ".join(time_clauses) + ")" + # Run query and group by activity + async with conn.execute(query, params) as cursor: + rows = await cursor.fetchall() + grouped = defaultdict(list) + for row in rows: + activity_id = row["activity_id"] + grouped[activity_id].append(row) + # Now fetch tags for each activity + activity_ids = list(grouped.keys()) + tags_by_activity = {} + if activity_ids: + placeholders = ",".join("?" for _ in activity_ids) + tag_query = f""" + SELECT a.id AS activity_id, t.id AS tag_id, t.name, t.description + FROM activities a + LEFT JOIN tag_mappings tm ON a.id = tm.activity_id + LEFT JOIN tags t ON tm.tag_id = t.id AND t.deleted_at IS NULL + WHERE a.id IN ({placeholders}) + """ + async with conn.execute(tag_query, activity_ids) as cursor: + tag_rows = await cursor.fetchall() + for row in tag_rows: + activity_id = row["activity_id"] + tag_id = row["tag_id"] + if activity_id not in tags_by_activity: + tags_by_activity[activity_id] = [] + if tag_id is not None: + tag = DBTag(id=tag_id, name=row["name"], description=row["description"]) + tags_by_activity[activity_id].append(tag) + # Build DBActivitySpecWithTags objects + result = [] + for activity_id, rows in grouped.items(): + db_activity = DBActivity( + id=activity_id, + name=rows[0]["name"], + description=rows[0]["description"], + productivity_level=rows[0]["productivity_level"], + source=rows[0]["source"], + last_manual_action_time=rows[0]["activity_last_manual_action_time"], + ) + events = [] + for row in rows: + db_activity_event = DBActivityEvent( + event_time=row["event_time"], + event_type=row["event_type"], + id=row["event_id"], + activity_id=activity_id, + aggregation_id=row["aggregation_id"], + last_manual_action_time=row["event_last_manual_action_time"], + ) + events.append(db_activity_event) + tags = tags_by_activity.get(activity_id, []) + spec_with_tags = DBActivitySpecWithTags( + activity=db_activity, events=events, tags=tags + ) + result.append(spec_with_tags) + return result + + +def row_to_scheduled_job(row: aiosqlite.Row) -> DBScheduledJob: + payload_raw = row["payload"] + payload: dict[str, Any] | None = None + if payload_raw is not None: + try: + payload = json.loads(payload_raw) + except json.JSONDecodeError: + logger.warning( + "Scheduled job {} has invalid payload JSON; ignoring payload", + row["id"], + ) + + enabled_value = row["enabled"] + enabled = ( + bool(enabled_value) if not isinstance(enabled_value, bool) else enabled_value + ) + + status_value = row["status"] + try: + status = ScheduleStatus(status_value) + except ValueError: + logger.warning( + "Scheduled job {} has unknown status '{}'; defaulting to idle", + row["id"], + status_value, + ) + status = ScheduleStatus.IDLE + + job_type_value = row["job_type"] + try: + job_type = JobType(job_type_value) + except ValueError as exc: + raise ValueError( + f"Scheduled job {row['id']} has unknown job_type '{job_type_value}'" + ) from exc + + return DBScheduledJob( + id=row["id"], + schedule_key=row["schedule_key"], + job_type=job_type, + cron_expr=row["cron_expr"], + next_run_at=row["next_run_at"], + enabled=enabled, + payload=payload, + running_count=row["running_count"], + max_concurrent=row["max_concurrent"], + last_started_at=row["last_started_at"], + status=status, + ) + + +async def upsert_scheduled_job( + conn: aiosqlite.Connection, + *, + job: ScheduledJob, +) -> DBScheduledJob: + payload_json = json.dumps(job.payload) if job.payload is not None else None + cursor = await conn.execute( + """ + INSERT INTO scheduled_jobs ( + schedule_key, + job_type, + cron_expr, + next_run_at, + enabled, + payload, + running_count, + max_concurrent, + last_started_at, + status + ) + VALUES (:schedule_key, :job_type, :cron_expr, :next_run_at, :enabled, :payload, :running_count, :max_concurrent, :last_started_at, :status) + ON CONFLICT(schedule_key) DO UPDATE SET + job_type = excluded.job_type, + cron_expr = excluded.cron_expr, + next_run_at = CASE + WHEN scheduled_jobs.next_run_at > excluded.next_run_at THEN scheduled_jobs.next_run_at + ELSE excluded.next_run_at + END, + enabled = excluded.enabled, + payload = excluded.payload, + max_concurrent = excluded.max_concurrent + RETURNING + id, + schedule_key, + job_type, + cron_expr, + next_run_at, + enabled, + payload, + running_count, + max_concurrent, + last_started_at, + status + """, + { + "schedule_key": job.schedule_key, + "job_type": job.job_type.value, + "cron_expr": job.cron_expr, + "next_run_at": job.next_run_at, + "enabled": 1 if job.enabled else 0, + "payload": payload_json, + "running_count": job.running_count, + "max_concurrent": job.max_concurrent, + "last_started_at": job.last_started_at, + "status": job.status.value, + }, + ) + + row = await cursor.fetchone() + assert row is not None, "scheduled_job upsert did not return a row" + return row_to_scheduled_job(row) + + +async def select_scheduled_job_by_id( + conn: aiosqlite.Connection, *, schedule_id: int +) -> DBScheduledJob | None: + async with conn.execute( + """ + SELECT + id, + schedule_key, + job_type, + cron_expr, + next_run_at, + enabled, + payload, + running_count, + max_concurrent, + last_started_at, + status + FROM scheduled_jobs + WHERE id = :schedule_id + """, + {"schedule_id": schedule_id}, + ) as cursor: + row = await cursor.fetchone() + + if row is None: + return None + + return row_to_scheduled_job(row) + + +async def select_scheduled_job_by_key( + conn: aiosqlite.Connection, *, schedule_key: str +) -> DBScheduledJob | None: + async with conn.execute( + """ + SELECT + id, + schedule_key, + job_type, + cron_expr, + next_run_at, + enabled, + payload, + running_count, + max_concurrent, + last_started_at, + status + FROM scheduled_jobs + WHERE schedule_key = :schedule_key + """, + {"schedule_key": schedule_key}, + ) as cursor: + row = await cursor.fetchone() + + if row is None: + return None + + return row_to_scheduled_job(row) + + +async def select_all_scheduled_jobs( + conn: aiosqlite.Connection, +) -> list[DBScheduledJob]: + """Select all scheduled jobs from the database. + + Used when we want to do all logic in Python rather than multiple queries. + """ + async with conn.execute( + """ + SELECT + id, + schedule_key, + job_type, + cron_expr, + next_run_at, + enabled, + payload, + running_count, + max_concurrent, + last_started_at, + status + FROM scheduled_jobs + """, + ) as cursor: + rows = await cursor.fetchall() + + return [row_to_scheduled_job(row) for row in rows] + + +async def select_due_scheduled_jobs( + conn: aiosqlite.Connection, + *, + now: datetime, + include_disabled: bool = False, + limit: int | None = None, +) -> list[DBScheduledJob]: + query = """ + SELECT + id, + schedule_key, + job_type, + cron_expr, + next_run_at, + enabled, + payload, + running_count, + max_concurrent, + last_started_at, + status + FROM scheduled_jobs + WHERE enabled = 1 + AND next_run_at <= :now + AND running_count < max_concurrent + """ + + if not include_disabled: + query += "\n AND status != 'disabled'" + + query += "\n ORDER BY next_run_at ASC, id ASC" + + params: dict[str, Any] = {"now": now} + + if limit is not None: + query += "\n LIMIT :limit" + params["limit"] = limit + + async with conn.execute(query, params) as cursor: + rows = await cursor.fetchall() + + return [row_to_scheduled_job(row) for row in rows] + + +async def select_next_scheduled_run_after( + conn: aiosqlite.Connection, + *, + now: datetime, +) -> datetime | None: + async with conn.execute( + """ + SELECT + next_run_at + FROM scheduled_jobs + WHERE enabled = 1 + AND status != 'disabled' + AND next_run_at > :now + ORDER BY next_run_at ASC, id ASC + LIMIT 1 + """, + {"now": now}, + ) as cursor: + row = await cursor.fetchone() + + if row is None: + return None + + return row["next_run_at"] + + +async def select_potentially_timed_out_scheduled_jobs( + conn: aiosqlite.Connection, + *, + timeout_threshold: datetime, +) -> list[DBScheduledJob]: + """Read-only query to find jobs that might be timed out. + + Returns jobs that are running and have a last_started_at before the threshold. + This allows checking in Python before doing any writes. + """ + async with conn.execute( + """ + SELECT + id, + schedule_key, + job_type, + cron_expr, + next_run_at, + enabled, + payload, + running_count, + max_concurrent, + last_started_at, + status + FROM scheduled_jobs + WHERE running_count > 0 + AND last_started_at IS NOT NULL + AND last_started_at <= :timeout_threshold + AND status != 'disabled' + """, + {"timeout_threshold": timeout_threshold}, + ) as cursor: + rows = await cursor.fetchall() + + return [row_to_scheduled_job(row) for row in rows] + + +async def release_timed_out_scheduled_jobs( + conn: aiosqlite.Connection, + *, + timeout_threshold: datetime, + now: datetime, +) -> list[DBScheduledJob]: + async with conn.execute( + """ + UPDATE scheduled_jobs + SET + running_count = 0, + status = 'error', + next_run_at = CASE + WHEN next_run_at <= :now THEN next_run_at + ELSE :now + END + WHERE running_count > 0 + AND last_started_at IS NOT NULL + AND last_started_at <= :timeout_threshold + AND status != 'disabled' + RETURNING + id, + schedule_key, + job_type, + cron_expr, + next_run_at, + enabled, + payload, + running_count, + max_concurrent, + last_started_at, + status + """, + {"timeout_threshold": timeout_threshold, "now": now}, + ) as cursor: + rows = await cursor.fetchall() + + return [row_to_scheduled_job(row) for row in rows] + + +async def mark_scheduled_job_started( + conn: aiosqlite.Connection, + *, + schedule_id: int, + expected_next_run_at: datetime, + started_at: datetime, + new_next_run_at: datetime | None, +) -> bool: + """Mark a scheduled job as started. + + If new_next_run_at is None, the next_run_at field will not be updated, + allowing the job to manage its own scheduling. + """ + if new_next_run_at is not None: + # Update next_run_at from cron expression + cursor = await conn.execute( + """ + UPDATE scheduled_jobs + SET + running_count = running_count + 1, + last_started_at = :started_at, + next_run_at = :new_next_run_at, + status = CASE WHEN status = 'disabled' THEN status ELSE 'idle' END + WHERE id = :schedule_id + AND enabled = 1 + AND status != 'disabled' + AND running_count < max_concurrent + AND next_run_at = :expected_next_run_at + """, + { + "schedule_id": schedule_id, + "started_at": started_at, + "new_next_run_at": new_next_run_at, + "expected_next_run_at": expected_next_run_at, + }, + ) + else: + # Don't update next_run_at, let the job manage it + cursor = await conn.execute( + """ + UPDATE scheduled_jobs + SET + running_count = running_count + 1, + last_started_at = :started_at, + status = CASE WHEN status = 'disabled' THEN status ELSE 'idle' END + WHERE id = :schedule_id + AND enabled = 1 + AND status != 'disabled' + AND running_count < max_concurrent + AND next_run_at = :expected_next_run_at + """, + { + "schedule_id": schedule_id, + "started_at": started_at, + "expected_next_run_at": expected_next_run_at, + }, + ) + + return cursor.rowcount == 1 + + +async def decrement_scheduled_job_running_count( + conn: aiosqlite.Connection, + *, + schedule_id: int, + new_status: ScheduleStatus | None = None, +) -> None: + set_clauses = [ + "running_count = CASE WHEN running_count > 0 THEN running_count - 1 ELSE 0 END" + ] + params: dict[str, Any] = {"schedule_id": schedule_id} + + if new_status is not None: + set_clauses.append("status = :status") + params["status"] = new_status.value + + sql = """ + UPDATE scheduled_jobs + SET {set_clause} + WHERE id = :schedule_id + """.format(set_clause=", ".join(set_clauses)) + + await conn.execute(sql, params) + + +async def update_scheduled_job_next_run_at( + conn: aiosqlite.Connection, + *, + schedule_id: int, + next_run_at: datetime, +) -> bool: + cursor = await conn.execute( + """ + UPDATE scheduled_jobs + SET next_run_at = :next_run_at + WHERE id = :schedule_id + """, + {"schedule_id": schedule_id, "next_run_at": next_run_at}, + ) + + return cursor.rowcount == 1 + + +async def delete_scheduled_job_by_key( + conn: aiosqlite.Connection, *, schedule_key: str +) -> None: + await conn.execute( + "DELETE FROM scheduled_jobs WHERE schedule_key = :schedule_key", + {"schedule_key": schedule_key}, + ) + + +async def is_goal_paused_at( + conn: aiosqlite.Connection, *, goal_id: int, at_utc: datetime +) -> bool: + """Return True if the goal is paused at the given UTC timestamp. + + Looks up the last pause/resume event at or before the timestamp; if it's a + 'pause' event, consider the goal paused; otherwise not paused. If no event + exists, it's considered active (not paused). + """ + async with conn.execute( + """ + SELECT event_type + FROM goal_pause_events + WHERE goal_id = ? AND datetime(at) <= datetime(?) + ORDER BY datetime(at) DESC, id DESC + LIMIT 1 + """, + (goal_id, at_utc), + ) as cursor: + row = await cursor.fetchone() + if not row: + return False + try: + return row["event_type"] == "pause" + except (KeyError, TypeError, IndexError): + # Fallback for row indexing differences + return (row[0] if row else None) == "pause" + + +async def select_tags_by_ids( + conn: aiosqlite.Connection, tag_ids: List[int], include_deleted: bool = False +) -> list[dict]: + """ + Select tags by a list of IDs. If include_deleted is True, include soft-deleted tags. + Returns a list of dicts with id, name, description, and deleted_at. + """ + if not tag_ids: + return [] + placeholders = ",".join(["?"] * len(tag_ids)) + query = ( + f"SELECT id, name, description, deleted_at FROM tags WHERE id IN ({placeholders})" + if include_deleted + else f"SELECT id, name, description, deleted_at FROM tags WHERE id IN ({placeholders}) AND deleted_at IS NULL" + ) + async with conn.execute(query, tag_ids) as cursor: + rows = await cursor.fetchall() + return [dict(row) for row in rows] + + +@lru_cache +def simple_insert_query( + table_name: str, columns: tuple[str], returning_columns: tuple[str] = tuple() +) -> str: + insert_statemnent = f"INSERT INTO {table_name} ({','.join(columns)}) VALUES ({','.join([f':{col}' for col in columns])})" + if returning_columns: + return f"{insert_statemnent} RETURNING {','.join(returning_columns)};" + else: + return f"{insert_statemnent};" + + +@lru_cache +def set_clause_query(table_name: str, columns: tuple[str]) -> str: + return f"UPDATE {table_name} SET {','.join([f'{col} = :{col}' for col in columns])}" + + +# (source_events moved to Valkey Streams; DB helpers removed) + + +async def insert_user_settings( + conn: aiosqlite.Connection, + timezone: str, + username: str, + image_processing_approach: str = ImageProcessingApproach.OCR.value, + image_model_provider: str = "", + image_model_base_url: str | None = None, + image_model: str = "", + image_model_api_key_enc: bytes | None = None, + text_model_provider: str = "", + text_model_base_url: str | None = None, + text_model: str = "", + text_model_api_key_enc: bytes | None = None, + productivity_prompt: str = "", +) -> None: + sql = simple_insert_query( + "user_settings", + ( + "timezone", + "image_processing_approach", + "image_model_provider", + "image_model_base_url", + "image_model", + "image_model_api_key_enc", + "text_model_provider", + "text_model_base_url", + "text_model", + "text_model_api_key_enc", + "username", + "productivity_prompt", # Add column name + ), + ) + + await conn.execute( + sql, + { + "timezone": timezone, + "image_processing_approach": image_processing_approach, + "image_model_provider": image_model_provider, + "image_model_base_url": image_model_base_url, + "image_model": image_model, + "image_model_api_key_enc": image_model_api_key_enc, + "text_model_provider": text_model_provider, + "text_model_base_url": text_model_base_url, + "text_model": text_model, + "text_model_api_key_enc": text_model_api_key_enc, + "username": username, + "productivity_prompt": productivity_prompt, # Add parameter value + }, + ) + + +def build_user_settings_from_row(row: Any) -> UserSettings: + """Construct a UserSettings object (with decrypted API keys) from a DB row. + + Expects columns: + username, timezone, + image_processing_approach, + image_model_provider, image_model_base_url, image_model, image_model_api_key_enc, + text_model_provider, text_model_base_url, text_model, text_model_api_key_enc, + productivity_prompt + """ + # Decrypt image model key (if present) + if row["image_model_api_key_enc"]: + try: + image_model_api_key = decrypt_secret( + row["image_model_api_key_enc"], + config.master_key.get_secret_value(), + aad=AADS.LLM_API_KEY, + ) + except Exception as exc: + raise RuntimeError( + "Failed to decrypt image model API key, did you change the master key?" + ) from exc + else: + image_model_api_key = None + + # Decrypt text model key (if present) + if row["text_model_api_key_enc"]: + try: + text_model_api_key = decrypt_secret( + row["text_model_api_key_enc"], + config.master_key.get_secret_value(), + aad=AADS.LLM_API_KEY, + ) + except Exception as exc: + raise RuntimeError( + "Failed to decrypt text model API key, did you change the master key?" + ) from exc + else: + text_model_api_key = None + + # Determine the correct provider class based on the model_provider string + image_provider = row["image_model_provider"] + if image_provider == ModelProvider.GOOGLE_AI: + image_model_config = GoogleAIConfig( + model_base_url=row["image_model_base_url"], + model=row["image_model"], + api_key=image_model_api_key, + ) + elif image_provider == ModelProvider.OPENAI: + image_model_config = OpenAIConfig( + model_base_url=row["image_model_base_url"], + model=row["image_model"], + api_key=image_model_api_key, + ) + elif image_provider == ModelProvider.OPENAI_GENERIC: + image_model_config = OpenAIGenericConfig( + model_base_url=row["image_model_base_url"], + model=row["image_model"], + api_key=image_model_api_key, + ) + elif image_provider == ModelProvider.ANTHROPIC: + image_model_config = AnthropicConfig( + model_base_url=row["image_model_base_url"], + model=row["image_model"], + api_key=image_model_api_key, + ) + else: + raise ValueError(f"Unknown image model provider: {image_provider}") + + # Determine the correct provider class based on the model_provider string + text_provider = row["text_model_provider"] + if text_provider == ModelProvider.GOOGLE_AI: + text_model_config = GoogleAIConfig( + model_base_url=row["text_model_base_url"], + model=row["text_model"], + api_key=text_model_api_key, + ) + elif text_provider == ModelProvider.OPENAI: + text_model_config = OpenAIConfig( + model_base_url=row["text_model_base_url"], + model=row["text_model"], + api_key=text_model_api_key, + ) + elif text_provider == ModelProvider.OPENAI_GENERIC: + text_model_config = OpenAIGenericConfig( + model_base_url=row["text_model_base_url"], + model=row["text_model"], + api_key=text_model_api_key, + ) + elif text_provider == ModelProvider.ANTHROPIC: + text_model_config = AnthropicConfig( + model_base_url=row["text_model_base_url"], + model=row["text_model"], + api_key=text_model_api_key, + ) + else: + raise ValueError(f"Unknown text model provider: {text_provider}") + + image_processing_approach = ImageProcessingApproach( + row["image_processing_approach"] + ) + + return UserSettings( + timezone=row["timezone"], + image_model_config=image_model_config, + text_model_config=text_model_config, + username=row["username"], + productivity_prompt=row["productivity_prompt"], + image_processing_approach=image_processing_approach, + ) + + +async def select_user_settings(conn) -> UserSettings | None: + async with conn.execute( + """SELECT username, timezone, + image_processing_approach, + image_model_provider, image_model_base_url, image_model, image_model_api_key_enc, + text_model_provider, text_model_base_url, text_model, text_model_api_key_enc, + productivity_prompt + FROM user_settings LIMIT 1""" + ) as cursor: + row = await cursor.fetchone() + if row: + return build_user_settings_from_row(row) + + +async def select_user_auth(conn: aiosqlite.Connection) -> Optional[dict]: + async with conn.execute( + "SELECT id, password_hash, created_at FROM user_auth WHERE id='default' LIMIT 1" + ) as cursor: + row = await cursor.fetchone() + return dict(row) if row else None + + +async def update_user_password(conn: aiosqlite.Connection, password_hash: str) -> None: + logger.debug("Updating password for user") + sql = "UPDATE user_auth SET password_hash = :password_hash WHERE id='default'" + params = {"password_hash": password_hash} + try: + await conn.execute(sql, params) + logger.info("Password updated successfully for user") + except Exception as e: # noqa: BLE001 - propagate as runtime error with context + raise RuntimeError("Error updating password for user") from e + + +async def create_user_auth(conn: aiosqlite.Connection, password_hash: str) -> None: + sql = "INSERT INTO user_auth (id, password_hash) VALUES ('default', :password_hash)" + try: + await conn.execute(sql, {"password_hash": password_hash}) + logger.info("Initialized user_auth with bootstrap password") + except Exception as e: + raise RuntimeError("Error initializing user_auth") from e + + +async def get_user_settings_draft( + conn: aiosqlite.Connection, *, wizard_id: str +) -> dict | None: + async with conn.execute( + """ + SELECT * + FROM user_settings_draft + WHERE wizard_id = ? + LIMIT 1 + """, + (wizard_id,), + ) as cursor: + row = await cursor.fetchone() + return dict(row) if row else None + + +async def upsert_user_settings_draft_basics( + conn: aiosqlite.Connection, + *, + wizard_id: str, + username: str, + timezone: str, + description: str | None = None, +) -> None: + await conn.execute( + """ + INSERT INTO user_settings_draft ( + wizard_id, username, timezone, description + ) VALUES (:wizard_id, :username, :timezone, :description) + ON CONFLICT(wizard_id) DO UPDATE SET + username = excluded.username, + timezone = excluded.timezone, + description = excluded.description + """, + { + "wizard_id": wizard_id, + "username": username, + "timezone": timezone, + "description": description, + }, + ) + + +async def update_user_settings_draft_productivity( + conn: aiosqlite.Connection, *, wizard_id: str, productivity_prompt: str +) -> None: + await conn.execute( + """ + INSERT INTO user_settings_draft (wizard_id, productivity_prompt) + VALUES (:wizard_id, :productivity_prompt) + ON CONFLICT(wizard_id) DO UPDATE SET + productivity_prompt = excluded.productivity_prompt + """, + {"wizard_id": wizard_id, "productivity_prompt": productivity_prompt}, + ) + + +async def update_user_settings_draft_tags( + conn: aiosqlite.Connection, *, wizard_id: str, tags_json: str | None +) -> None: + await conn.execute( + """ + INSERT INTO user_settings_draft (wizard_id, tags_json) + VALUES (:wizard_id, :tags_json) + ON CONFLICT(wizard_id) DO UPDATE SET + tags_json = excluded.tags_json + """, + {"wizard_id": wizard_id, "tags_json": tags_json}, + ) + + +async def update_user_settings_draft_llm_configs( + conn: aiosqlite.Connection, + *, + wizard_id: str, + image_model_provider: str | None, + image_model_base_url: str | None, + image_model: str | None, + image_model_api_key_enc: bytes | None, + text_model_provider: str | None, + text_model_base_url: str | None, + text_model: str | None, + text_model_api_key_enc: bytes | None, + image_processing_approach: str | None = None, +) -> None: + await conn.execute( + """ + INSERT INTO user_settings_draft ( + wizard_id, + image_model_provider, + image_model_base_url, + image_model, + image_model_api_key_enc, + text_model_provider, + text_model_base_url, + text_model, + text_model_api_key_enc, + image_processing_approach + ) VALUES ( + :wizard_id, + :image_model_provider, + :image_model_base_url, + :image_model, + :image_model_api_key_enc, + :text_model_provider, + :text_model_base_url, + :text_model, + :text_model_api_key_enc, + :image_processing_approach + ) + ON CONFLICT(wizard_id) DO UPDATE SET + image_model_provider = excluded.image_model_provider, + image_model_base_url = excluded.image_model_base_url, + image_model = excluded.image_model, + image_model_api_key_enc = excluded.image_model_api_key_enc, + text_model_provider = excluded.text_model_provider, + text_model_base_url = excluded.text_model_base_url, + text_model = excluded.text_model, + text_model_api_key_enc = excluded.text_model_api_key_enc, + image_processing_approach = excluded.image_processing_approach + """, + { + "wizard_id": wizard_id, + "image_model_provider": image_model_provider, + "image_model_base_url": image_model_base_url, + "image_model": image_model, + "image_model_api_key_enc": image_model_api_key_enc, + "text_model_provider": text_model_provider, + "text_model_base_url": text_model_base_url, + "text_model": text_model, + "text_model_api_key_enc": text_model_api_key_enc, + "image_processing_approach": ( + image_processing_approach or ImageProcessingApproach.OCR.value + ), + }, + ) + + +async def delete_user_settings_draft( + conn: aiosqlite.Connection, *, wizard_id: str +) -> None: + await conn.execute( + "DELETE FROM user_settings_draft WHERE wizard_id = ?", (wizard_id,) + ) + + +async def finalize_user_settings_from_draft( + conn: aiosqlite.Connection, *, wizard_id: str +) -> None: + """Promote draft to user_settings and create initial tags if provided. + + Assumes that no user_settings row exists yet (single-user system). + """ + draft = await get_user_settings_draft(conn, wizard_id=wizard_id) + if not draft: + raise ValueError("Draft not found") + + username = (draft.get("username") or "").strip() + if not username: + raise ValueError("Username is required to finalize account creation") + + # Insert user_settings + await insert_user_settings( + conn=conn, + timezone=(draft.get("timezone") or "UTC"), + image_processing_approach=( + draft.get("image_processing_approach") or ImageProcessingApproach.OCR.value + ), + image_model_provider=(draft.get("image_model_provider") or ""), + image_model_base_url=(draft.get("image_model_base_url") or None), + image_model=(draft.get("image_model") or ""), + image_model_api_key_enc=(draft.get("image_model_api_key_enc") or None), + text_model_provider=(draft.get("text_model_provider") or ""), + text_model_base_url=(draft.get("text_model_base_url") or None), + text_model=(draft.get("text_model") or ""), + text_model_api_key_enc=(draft.get("text_model_api_key_enc") or None), + username=username, + productivity_prompt=(draft.get("productivity_prompt") or ""), + ) + + # Create initial tags if provided + try: + tags_json = draft.get("tags_json") + if tags_json: + data = json.loads(tags_json) + if isinstance(data, list) and data: + # Prepare rows for bulk insert + rows = [] + for item in data: + name = (item or {}).get("name") or "" + description = (item or {}).get("description") or "" + if name.strip(): + rows.append({"name": name.strip(), "description": description}) + if rows: + sql = simple_insert_query("tags", ("name", "description")) + await conn.executemany(sql, rows) + except (json.JSONDecodeError, TypeError, KeyError) as exc: + logger.exception( + "Failed to parse or insert initial tags from draft: {error}", + error=exc, + ) + + # Remove the draft + await delete_user_settings_draft(conn, wizard_id=wizard_id) + + +async def update_user_settings( + conn: aiosqlite.Connection, settings: dict[str, Any] +) -> UserSettings: + """ + Update user settings from a dictionary and return the full user_settings row + using a RETURNING clause. Password changes should be handled separately. + + Returns the updated UserSettings object, or None if no row was updated. + """ + if "password" in settings: + raise ValueError("Password updates are not allowed through this function.") + assert settings, "update_user_settings called with no settings to update." + + logger.debug(f"Updating user settings with keys: {list(settings.keys())}") + + # Build the SET clause dynamically from the dictionary keys + set_clause = ", ".join(f"{key} = :{key}" for key in settings.keys()) + + # Explicitly list columns to return to build a proper UserSettings + returning_cols = ( + "username, timezone, image_processing_approach, " + "image_model_provider, image_model_base_url, image_model, image_model_api_key_enc, " + "text_model_provider, text_model_base_url, text_model, text_model_api_key_enc, " + "productivity_prompt" + ) + + sql = f"UPDATE user_settings SET {set_clause} RETURNING {returning_cols}" + + params = {**settings} + + try: + cursor = await conn.execute(sql, params) + row = await cursor.fetchone() + assert row is not None, "No user_settings row returned after update." + + updated = build_user_settings_from_row(row) + + logger.debug("User settings updated successfully with RETURNING") + return updated + except Exception as exc: + logger.exception("Error updating user settings: {error}", error=exc) + raise + + # Note: update_user_password moved above to user_auth section + + +async def insert_activity_events( + conn: aiosqlite.Connection, events: Sequence[ActivityEventInsert] +) -> None: + # Use INSERT OR IGNORE to provide idempotency when a unique index exists + # on (activity_id, event_time, event_type). This prevents duplicate events + # from causing IntegrityError on retries or replays. + sql = ( + "INSERT OR IGNORE INTO activity_events (activity_id,event_time,event_type,aggregation_id,last_manual_action_time) " + "VALUES (:activity_id,:event_time,:event_type,:aggregation_id,:last_manual_action_time);" + ) + + dicts = [ + { + "activity_id": event.activity_id, + "event_time": event.event_time, + "event_type": event.event_type, + "aggregation_id": event.aggregation_id, + "last_manual_action_time": event.last_manual_action_time, + } + for event in events + ] + await conn.executemany(sql, dicts) + + +async def select_open_activities_with_last_event( + conn: aiosqlite.Connection, +) -> list[DBActivityWithLatestEvent]: + query = """ + SELECT + a.id, a.name, a.description, a.productivity_level, a.source, -- Select specific columns + a.last_manual_action_time AS activity_last_manual_action_time, -- Alias activity time + le.event_type AS state, + le.event_time, + le.aggregation_id, + le.id AS last_event_id, + ag.start_time as aggregation_start_time, + ag.end_time as aggregation_end_time, + ag.first_timestamp as aggregation_first_timestamp, + ag.last_timestamp as aggregation_last_timestamp, + le.last_manual_action_time AS event_last_manual_action_time -- Alias event time + + FROM activities a + JOIN ( + SELECT * + FROM activity_events ae + WHERE ae.id = ( + SELECT ae2.id + FROM activity_events ae2 + WHERE ae2.activity_id = ae.activity_id + ORDER BY ae2.event_time DESC + LIMIT 1 + ) + ) le ON le.activity_id = a.id + LEFT JOIN aggregations ag ON le.aggregation_id = ag.id + WHERE le.event_type <> 'close'; + """ + + return_list = [] + async with conn.execute(query) as cursor: + rows = await cursor.fetchall() + for row in rows: + db_activity = DBActivity( + id=row["id"], + name=row["name"], + description=row["description"], + productivity_level=row["productivity_level"], + source=row["source"], # Assign source here + last_manual_action_time=row[ + "activity_last_manual_action_time" + ], # Use aliased activity time + # source=row["source"], # Removed source from here + ) + + db_activity_event = DBActivityEvent( + event_time=row["event_time"], + event_type=row["state"], + id=row["last_event_id"], + activity_id=row["id"], + aggregation_id=row["aggregation_id"], + last_manual_action_time=row[ + "event_last_manual_action_time" + ], # Use aliased event time + ) + latest_aggregation = DBAggregation( + id=row["aggregation_id"], + start_time=row["aggregation_start_time"], + end_time=row["aggregation_end_time"], + first_timestamp=row["aggregation_first_timestamp"], + last_timestamp=row["aggregation_last_timestamp"], + ) + return_list.append( + DBActivityWithLatestEvent( + activity=db_activity, + latest_event=db_activity_event, + latest_aggregation=latest_aggregation, + ) + ) + return return_list + + +async def select_open_auto_activities_with_last_event( + conn: aiosqlite.Connection, +) -> list[DBActivityWithLatestEvent]: + query = """ + SELECT + a.id, a.name, a.description, a.productivity_level, a.source, -- Select specific columns + a.last_manual_action_time AS activity_last_manual_action_time, -- Alias activity time + le.event_type AS state, + le.event_time, + le.aggregation_id, + le.id AS last_event_id, + ag.start_time as aggregation_start_time, + ag.end_time as aggregation_end_time, + ag.first_timestamp as aggregation_first_timestamp, + ag.last_timestamp as aggregation_last_timestamp, + le.last_manual_action_time AS event_last_manual_action_time -- Alias event time + + FROM activities a + JOIN ( + SELECT * + FROM activity_events ae + WHERE ae.id = ( + SELECT ae2.id + FROM activity_events ae2 + WHERE ae2.activity_id = ae.activity_id + ORDER BY ae2.event_time DESC + LIMIT 1 + ) + ) le ON le.activity_id = a.id + LEFT JOIN aggregations ag ON le.aggregation_id = ag.id + WHERE le.event_type <> 'close' AND a.source = 'auto'; + """ + + return_list = [] + async with conn.execute(query) as cursor: + rows = await cursor.fetchall() + for row in rows: + db_activity = DBActivity( + id=row["id"], + name=row["name"], + description=row["description"], + productivity_level=row["productivity_level"], + source=row["source"], # Assign source here + last_manual_action_time=row[ + "activity_last_manual_action_time" + ], # Use aliased activity time + ) + + db_activity_event = DBActivityEvent( + event_time=row["event_time"], + event_type=row["state"], + id=row["last_event_id"], + activity_id=row["id"], + aggregation_id=row["aggregation_id"], + last_manual_action_time=row[ + "event_last_manual_action_time" + ], # Use aliased event time + ) + latest_aggregation = DBAggregation( + id=row["aggregation_id"], + start_time=row["aggregation_start_time"], + end_time=row["aggregation_end_time"], + first_timestamp=row["aggregation_first_timestamp"], + last_timestamp=row["aggregation_last_timestamp"], + ) + return_list.append( + DBActivityWithLatestEvent( + activity=db_activity, + latest_event=db_activity_event, + latest_aggregation=latest_aggregation, + ) + ) + return return_list + + +async def insert_activities( + conn: aiosqlite.Connection, activities: list[Activity] +) -> list[int]: + if not activities: + return [] + + sql = simple_insert_query( + "activities", + ( + "name", + "description", + "productivity_level", + "source", + "last_manual_action_time", + ), + ) + + insert_dicts = [ + { + "name": activity.name, + "description": activity.description, + "productivity_level": activity.productivity_level, + "source": activity.source, + "last_manual_action_time": activity.last_manual_action_time, + } + for activity in activities + ] + + await conn.executemany(sql, insert_dicts) + + max_id_cursor = await conn.execute( + f"SELECT id FROM activities ORDER BY id DESC LIMIT {len(activities)}" + ) + max_id_rows = await max_id_cursor.fetchall() + assert max_id_rows is not None, "No activities were inserted" + ids = [row["id"] for row in max_id_rows] + ids.reverse() + + return ids + + +async def insert_aggregation( + conn: aiosqlite.Connection, aggregation: Aggregation +) -> int: + sql = simple_insert_query( + "aggregations", ("start_time", "end_time", "first_timestamp", "last_timestamp") + ) + + cursor = await conn.execute( + sql, + {key: val for key, val in aggregation.model_dump().items() if key != "id"}, + ) + last_row_id = cursor.lastrowid + assert last_row_id is not None, "No row was inserted" + return last_row_id + + +async def get_specs_after_cutoff( + conn: aiosqlite.Connection, cutoff: datetime +) -> list[DBActivitySpec]: + query = """ + SELECT a.id AS activity_id, + a.name, + a.description, + a.productivity_level, + a.source, -- Select source + a.last_manual_action_time AS activity_last_manual_action_time, -- Alias + e.id AS event_id, + e.event_time, + e.event_type, + e.aggregation_id, + e.last_manual_action_time AS event_last_manual_action_time -- Alias + + FROM activities a + JOIN activity_events e ON a.id = e.activity_id + WHERE datetime(e.event_time) >= datetime(?); + """ + + # WHERE datetime(e.event_time) >= datetime(?) + async with conn.execute(query, (cutoff,)) as cursor: + rows = await cursor.fetchall() + grouped = defaultdict(list) + for row in rows: + activity_id = row["activity_id"] + grouped[activity_id].append(row) + return_list = [] + + for activity_id, rows in grouped.items(): + db_activity = DBActivity( + id=activity_id, + name=rows[0]["name"], + description=rows[0]["description"], + productivity_level=rows[0]["productivity_level"], + source=rows[0]["source"], # Assign source here + last_manual_action_time=rows[0][ + "activity_last_manual_action_time" + ], # Use aliased activity time + # source=rows[0]["source"], # Removed source from here + ) + + events = [] + for row in rows: + db_activity_event = DBActivityEvent( + event_time=row["event_time"], + event_type=row["event_type"], + id=row["event_id"], + activity_id=activity_id, + aggregation_id=row["aggregation_id"], + last_manual_action_time=row[ + "event_last_manual_action_time" + ], # Use aliased event time + ) + events.append(db_activity_event) + + db_activity_spec = DBActivitySpec(activity=db_activity, events=events) + + return_list.append(db_activity_spec) + + return return_list + + +async def select_tags( + conn: aiosqlite.Connection, include_deleted: bool = False +) -> list[DBTag]: + """Select tags. + + By default, excludes soft-deleted tags (deleted_at IS NULL). + Set include_deleted=True to return all tags regardless of deleted_at. + """ + query = ( + "SELECT id, name, description FROM tags" + if include_deleted + else "SELECT id, name, description FROM tags WHERE deleted_at IS NULL" + ) + + try: + async with conn.execute(query) as cursor: + rows = await cursor.fetchall() + + tags = [ + DBTag(id=row["id"], name=row["name"], description=row["description"]) + for row in rows + ] + + logger.trace(f"Selected {len(tags)} tags from database") + for tag in tags: + logger.trace( + f" - Tag {tag.id}: name='{tag.name}', description='{tag.description}'" + ) + + return tags + except Exception as exc: + logger.exception("Error selecting tags: {error}", error=exc) + raise + + +async def insert_tags(conn: aiosqlite.Connection, tags: list[Tag]) -> list[int]: + if not tags: + return [] + + try: + sql = simple_insert_query("tags", ("name", "description")) + + insert_dicts = [ + { + "name": tag.name or "", # Ensure name is never None + "description": tag.description + or "", # Ensure description is never None + } + for tag in tags + ] + + for i, tag_dict in enumerate(insert_dicts): + logger.debug(f"Inserting tag {i}: {tag_dict}") + + await conn.executemany(sql, insert_dicts) + + # Fetch the IDs of the newly inserted tags + max_id_cursor = await conn.execute( + f"SELECT id FROM tags ORDER BY id DESC LIMIT {len(tags)}" + ) + max_id_rows = await max_id_cursor.fetchall() + assert max_id_rows is not None, "No tags were inserted" + ids = [row["id"] for row in max_id_rows] + ids.reverse() # Reverse to match the order of the input tags + + logger.info(f"Inserted {len(ids)} tags with IDs: {ids}") + return ids + + except Exception as exc: + logger.exception("Error inserting tags: {error}", error=exc) + raise + + +async def update_tags(conn: aiosqlite.Connection, tags: list[DBTag]) -> None: + if not tags: + return + + try: + # All DBTag instances have IDs, so no need to filter + valid_tags = tags + + # Prepare statement and parameters for executemany + update_params = [ + ( + tag.name, + tag.description or "", + tag.id, + ) # Ensure description is never None + for tag in valid_tags + ] + + logger.debug(f"Updating {len(valid_tags)} tags") + for tag in valid_tags: + logger.debug( + f" - Updating tag {tag.id}: name='{tag.name}', description='{tag.description}'" + ) + + # Use named placeholders for clarity + query = "UPDATE tags SET name = ?, description = ? WHERE id = ?" + await conn.executemany(query, update_params) + + # Verify the updates took effect + for tag in valid_tags: + verify_query = "SELECT name, description FROM tags WHERE id = ?" + async with conn.execute(verify_query, (tag.id,)) as cursor: + row = await cursor.fetchone() + if row: + logger.debug( + f" - Verified tag {tag.id}: name='{row['name']}', description='{row['description']}'" + ) + + except Exception as exc: + logger.exception("Error updating tags: {error}", error=exc) + raise + + +async def delete_tags(conn: aiosqlite.Connection, tag_ids: list[int]) -> None: + """Soft-delete multiple tags by their IDs by setting deleted_at. + + Note: existing UNIQUE constraint on tags.name remains; soft-deleted rows + still occupy their names. Consider adjusting uniqueness in a follow-up + migration if you need to re-create tags with the same name after delete. + """ + if not tag_ids: + return + + # Use IN clause with placeholders + placeholders = ",".join("?" for _ in tag_ids) + await conn.execute( + f"UPDATE tags SET deleted_at = datetime('now') WHERE deleted_at IS NULL AND id IN ({placeholders})", + tuple(tag_ids), + ) + + +async def bulk_upsert_tags( + conn: aiosqlite.Connection, + tags_to_update: list[DBTag], + tags_to_insert: list[Tag], + ids_to_delete: list[int], +) -> list[int]: + """Perform bulk tag operations (update, insert, delete) in a single transaction + + Returns the IDs of the newly inserted tags. + """ + try: + logger.debug( + f"Bulk upsert: updating {len(tags_to_update)} tags, " + + f"inserting {len(tags_to_insert)} tags, " + + f"deleting {len(ids_to_delete)} tags" + ) + + # Delete tags in bulk + if ids_to_delete: + await delete_tags(conn, ids_to_delete) + + # Update tags in bulk + if tags_to_update: + await update_tags(conn, tags_to_update) + + # Insert tags in bulk + new_ids = [] + if tags_to_insert: + new_ids = await insert_tags(conn, tags_to_insert) + + return new_ids + + except Exception as exc: + logger.exception("Error in bulk_upsert_tags: {error}", error=exc) + # Still do rollback on error + await conn.rollback() + raise + + +async def insert_tag_mappings( + conn: aiosqlite.Connection, mappings: list[TagMapping] +) -> list[int]: + if not mappings: + return [] + + try: + sql = simple_insert_query("tag_mappings", ("tag_id", "activity_id")) + + insert_dicts = [ + { + "tag_id": mapping.tag_id, + "activity_id": mapping.activity_id, + } + for mapping in mappings + ] + + logger.trace(f"Inserting {len(mappings)} tag mappings") + for mapping in mappings: + logger.trace( + f" - Mapping activity {mapping.activity_id} to tag {mapping.tag_id}" + ) + + await conn.executemany(sql, insert_dicts) + + # Get the IDs of newly inserted mappings + max_id_cursor = await conn.execute( + f"SELECT id FROM tag_mappings ORDER BY id DESC LIMIT {len(mappings)}" + ) + max_id_rows = await max_id_cursor.fetchall() + assert max_id_rows is not None, "No tag mappings were inserted" + ids = [row["id"] for row in max_id_rows] + ids.reverse() # Reverse to match the order of the input mappings + + logger.trace(f"Inserted {len(ids)} tag mappings with IDs: {ids}") + return ids + + except Exception as exc: + logger.exception("Error inserting tag mappings: {error}", error=exc) + raise + + +async def delete_tag_mappings( + conn: aiosqlite.Connection, activity_id: int, tag_ids: list[int] +) -> None: + if not tag_ids: + return + + # Use IN clause with placeholders + placeholders = ",".join("?" for _ in tag_ids) + await conn.execute( + f"DELETE FROM tag_mappings WHERE activity_id = ? AND tag_id IN ({placeholders})", + (activity_id, *tag_ids), + ) + + +async def delete_activity(conn: aiosqlite.Connection, activity_id: int) -> None: + # Deletes an activity by its ID. Assumes related events/tags are handled by CASCADE. + # Note: Ensure foreign keys in activity_events and tag_mappings have ON DELETE CASCADE + await conn.execute("DELETE FROM activities WHERE id = ?", (activity_id,)) + # No explicit commit needed if the caller manages the transaction + + +async def update_activity( + conn: aiosqlite.Connection, + activity_id: int, + key_value_pairs: dict[str, Any], +): + keys = tuple(key_value_pairs.keys()) + + base_query = set_clause_query("activities", keys) + + query_dict = {**key_value_pairs, "id": activity_id} + + query = f""" + {base_query} + WHERE id = :id; + """ + await conn.execute(query, query_dict) + + +async def delete_activity_events( + conn: aiosqlite.Connection, activity_ids: list[int] +) -> None: + await conn.execute( + f"DELETE FROM activity_events WHERE activity_id IN ({','.join(['?'] * len(activity_ids))})", + tuple(activity_ids), + ) + + +async def select_activity_spec_with_tags( + conn: aiosqlite.Connection, activity_id: int +) -> DBActivitySpecWithTags: + # Query to get the specific activity and its events + activity_query = """ + SELECT a.id AS activity_id, + a.name, + a.description, + a.productivity_level, + a.source, -- Select source + a.last_manual_action_time AS activity_last_manual_action_time, -- Alias + e.id AS event_id, + e.event_time, + e.event_type, + e.aggregation_id, + e.last_manual_action_time AS event_last_manual_action_time -- Alias + FROM activities a + JOIN activity_events e ON a.id = e.activity_id + WHERE a.id = ?; + """ + + async with conn.execute(activity_query, (activity_id,)) as cursor: + rows = await cursor.fetchall() + activity_rows = list(rows) + + if not activity_rows: + raise ValueError(f"No activity found with ID {activity_id}") + + # Process the activity and its events + activity_data = activity_rows[0] + + db_activity = DBActivity( + id=activity_id, + name=activity_data["name"], + description=activity_data["description"], + productivity_level=activity_data["productivity_level"], + source=activity_data["source"], # Assign source here + last_manual_action_time=activity_data[ + "activity_last_manual_action_time" + ], # Use aliased activity time + # source=activity_data["source"], # Removed source from here + ) + + events = [] + for row in activity_rows: + db_activity_event = DBActivityEvent( + event_time=row["event_time"], + event_type=row["event_type"], + id=row["event_id"], + activity_id=activity_id, + aggregation_id=row["aggregation_id"], + last_manual_action_time=row[ + "event_last_manual_action_time" + ], # Use aliased event time + ) + events.append(db_activity_event) + + # Query to get tags for this specific activity + tags_query = """ + SELECT t.id AS tag_id, t.name, t.description + FROM tags t + JOIN tag_mappings tm ON t.id = tm.tag_id + WHERE tm.activity_id = ? AND t.deleted_at IS NULL; + """ + tags = [] + async with conn.execute(tags_query, (activity_id,)) as cursor: + tag_rows = await cursor.fetchall() + for row in tag_rows: + assert row["tag_id"] is not None, "Tag ID cannot be None" + tags.append( + DBTag( + id=row["tag_id"], name=row["name"], description=row["description"] + ) + ) + + # Combine into the final spec object + spec_with_tags = DBActivitySpecWithTags( + activity=db_activity, events=events, tags=tags + ) + + return spec_with_tags + + +async def select_goals_with_latest_definition( + conn: aiosqlite.Connection, + last_successes_limit: int = 5, +) -> list[GoalWithLatestResult]: + """Return all goals with their current definition, include/exclude tags, + the latest result (if any), and last N success booleans using a single DB call. + + - Respects tag soft-deletes (only includes tags where t.deleted_at IS NULL). + - Maps DB string fields to enums in entities, with lenient fallbacks. + """ + query = """ + WITH latest_defs AS ( + SELECT gd.* + FROM goal_definitions gd + JOIN ( + SELECT goal_id, MAX(effective_from) AS max_eff + FROM goal_definitions + WHERE datetime(effective_from) <= datetime('now') + GROUP BY goal_id + ) m ON m.goal_id = gd.goal_id AND m.max_eff = gd.effective_from + ), + top_results AS ( + SELECT * + FROM ( + SELECT r.*, + ROW_NUMBER() OVER ( + PARTITION BY r.goal_definition_id + ORDER BY r.period_start DESC, r.id DESC + ) AS rn + FROM goal_results r + ) + WHERE rn <= :limit + ), + last_pe AS ( + SELECT goal_id, event_type, at, + ROW_NUMBER() OVER (PARTITION BY goal_id ORDER BY datetime(at) DESC, id DESC) rn + FROM goal_pause_events + ) + SELECT + -- Goal fields + g.id AS goal_id, + CASE WHEN lpe.event_type = 'pause' THEN lpe.at ELSE NULL END AS goal_paused_since, + g.created_at AS goal_created_at, + g.metric AS metric, + g.operator AS operator, + g.period AS period, + g.timezone AS timezone, + + -- Definition fields + ld.id AS def_id, + ld.goal_id AS goal_id_for_def, + ld.name AS def_name, + ld.description AS def_description, + ld.metric_params_json AS metric_params_json, + ld.target_value AS target_value, + ld.include_mode AS include_mode, + ld.day_filter_json AS day_filter_json, + ld.time_filter_json AS time_filter_json, + ld.productivity_filter_json AS productivity_filter_json, + ld.effective_from AS effective_from, + ld.created_at AS def_created_at, + + -- Tag fields + t.id AS tag_id, + t.name AS tag_name, + t.description AS tag_description, + gt.role AS tag_role, + + -- Top results (last N), with rn=1 being latest + tr.id AS res_id, + tr.period_start AS res_period_start, + tr.period_end AS res_period_end, + tr.metric_value AS res_metric_value, + tr.success AS res_success, + tr.eval_state AS res_eval_state, + tr.eval_state_reason AS res_eval_state_reason, + tr.created_at AS res_created_at, + tr.rn AS res_rn + FROM goals g + JOIN latest_defs ld ON ld.goal_id = g.id + LEFT JOIN last_pe lpe ON lpe.goal_id = g.id AND lpe.rn = 1 + LEFT JOIN goal_tags gt ON gt.goal_definition_id = ld.id + LEFT JOIN tags t ON t.id = gt.tag_id AND t.deleted_at IS NULL + LEFT JOIN top_results tr ON tr.goal_definition_id = ld.id + ORDER BY g.id, ld.id, t.id, res_rn + """ + + async with conn.execute(query, {"limit": last_successes_limit}) as cursor: + rows = await cursor.fetchall() + + if not rows: + return [] + + # Aggregate per goal and definition + results_by_goal: dict[int, dict[str, Any]] = {} + seen_tag_include: dict[int, set[int]] = {} + seen_tag_exclude: dict[int, set[int]] = {} + seen_success_rn: dict[int, set[int]] = {} + + for r in rows: + goal_id = r["goal_id"] + def_id = r["def_id"] + + # Initialize structures for this goal if first time + if goal_id not in results_by_goal: + # Build goal object + metric = GoalMetric(r["metric"]) # may raise if invalid + op = ( + MetricOperator(r["operator"]) if r["operator"] else MetricOperator.EQUAL + ) + period = GoalPeriod(r["period"]) if r["period"] else GoalPeriod.DAY + + def _coerce_operator(m: GoalMetric, opx: MetricOperator) -> MetricOperator: + if m in ( + GoalMetric.AVG_PRODUCTIVITY_LEVEL, + GoalMetric.TOTAL_ACTIVITY_DURATION, + ): + return ( + opx + if opx + in (MetricOperator.LESS_THAN, MetricOperator.GREATER_THAN) + else MetricOperator.GREATER_THAN + ) + return opx + + op = _coerce_operator(metric, op) + + if metric == GoalMetric.AVG_PRODUCTIVITY_LEVEL: + goal_obj: DBGoal = DBAvgProductivityGoal( + id=goal_id, + created_at=r["goal_created_at"], + paused_since=r["goal_paused_since"], + timezone=r["timezone"], + period=period, + operator=cast(AvgProductivityOperators, op), + ) + else: + goal_obj = DBTotalActivityDurationGoal( + id=goal_id, + created_at=r["goal_created_at"], + paused_since=r["goal_paused_since"], + timezone=r["timezone"], + period=period, + operator=cast(TotalActivityDurationOperators, op), + ) + + # Build definition from row + def_row = { + "id": r["def_id"], + "goal_id": r["goal_id_for_def"], + "name": r["def_name"], + "description": r["def_description"], + "metric_params_json": r["metric_params_json"], + "target_value": r["target_value"], + "include_mode": r["include_mode"], + "day_filter_json": r["day_filter_json"], + "time_filter_json": r["time_filter_json"], + "productivity_filter_json": r["productivity_filter_json"], + "effective_from": r["effective_from"], + "created_at": r["def_created_at"], + # These are used by row_to_goal_definition() + "metric": r["metric"], + "operator": r["operator"], + "period": r["period"], + "timezone": r["timezone"], + } + definition = row_to_goal_definition(def_row) + + results_by_goal[goal_id] = { + "goal": goal_obj, + "definition": definition, + "include_tags": [], + "exclude_tags": [], + "latest_result": None, + "last_successes": [], + } + seen_tag_include[goal_id] = set() + seen_tag_exclude[goal_id] = set() + seen_success_rn[goal_id] = set() + + # Accumulate tags (dedupe) + if r["tag_id"] is not None: + tag = DBTag( + id=r["tag_id"], name=r["tag_name"], description=r["tag_description"] + ) + if r["tag_role"] == "include": + if r["tag_id"] not in seen_tag_include[goal_id]: + results_by_goal[goal_id]["include_tags"].append(tag) + seen_tag_include[goal_id].add(r["tag_id"]) + else: + if r["tag_id"] not in seen_tag_exclude[goal_id]: + results_by_goal[goal_id]["exclude_tags"].append(tag) + seen_tag_exclude[goal_id].add(r["tag_id"]) + + # Accumulate latest_result and last_successes from top_results rows + rn = r["res_rn"] + if rn is not None: + # Latest result (rn == 1) + if rn == 1 and results_by_goal[goal_id]["latest_result"] is None: + results_by_goal[goal_id]["latest_result"] = DBGoalResult( + id=r["res_id"], + goal_definition_id=def_id, + period_start=r["res_period_start"], + period_end=r["res_period_end"], + metric_value=r["res_metric_value"], + success=bool(r["res_success"]) + if r["res_success"] is not None + else False, + eval_state=EvalState(r["res_eval_state"]) + if r["res_eval_state"] + else EvalState.NA, + eval_state_reason=r["res_eval_state_reason"], + created_at=r["res_created_at"], + ) + # Last successes list (dedupe by rn to avoid tag fan-out) + if ( + last_successes_limit > 0 + and rn not in seen_success_rn[goal_id] + and len(results_by_goal[goal_id]["last_successes"]) + < last_successes_limit + ): + results_by_goal[goal_id]["last_successes"].append( + bool(r["res_success"]) if r["res_success"] is not None else False + ) + seen_success_rn[goal_id].add(rn) + + # Materialize final list in goal id order for stability + ordered_goal_ids = sorted(results_by_goal.keys()) + final_results: list[GoalWithLatestResult] = [] + for gid in ordered_goal_ids: + gdict = results_by_goal[gid] + final_results.append( + GoalWithLatestResult( + goal=gdict["goal"], + definition=gdict["definition"], + include_tags=gdict["include_tags"], + exclude_tags=gdict["exclude_tags"], + latest_result=gdict["latest_result"], + last_successes=gdict["last_successes"], + ) + ) + + return final_results + + +async def select_goal_progress_current( + conn: aiosqlite.Connection, goal_definition_id: int +) -> Optional[GoalProgressCurrent]: + """Fetch current progress row for a goal definition from goal_progress_current. + + Returns the row or None. Columns: goal_definition_id, period_start, period_end, + metric_value, success, eval_state, eval_state_reason, updated_at. + """ + query = """ + SELECT goal_definition_id, period_start, period_end, metric_value, + success, eval_state, eval_state_reason, updated_at + FROM goal_progress_current + WHERE goal_definition_id = ? + """ + async with conn.execute(query, (goal_definition_id,)) as cursor: + row = await cursor.fetchone() + if row is None: + return None + # Decide variant using the goal's metric from the definition + # For this query we need the goal metric; fetch from joined goal_definitions/goals + metric_q = """ + SELECT g.metric AS metric + FROM goal_definitions gd + JOIN goals g ON g.id = gd.goal_id + WHERE gd.id = ? + """ + async with conn.execute(metric_q, (goal_definition_id,)) as cur: + mrow = await cur.fetchone() + metric = ( + GoalMetric(mrow["metric"]) + if mrow is not None + else GoalMetric.AVG_PRODUCTIVITY_LEVEL + ) + + common = dict( + goal_definition_id=row["goal_definition_id"], + period_start=row["period_start"], + period_end=row["period_end"], + success=row["success"], + eval_state=EvalState(row["eval_state"]) + if row["eval_state"] is not None + else EvalState.NA, + eval_state_reason=row["eval_state_reason"], + updated_at=row["updated_at"], + ) + + if metric == GoalMetric.TOTAL_ACTIVITY_DURATION: + return TotalActivityDurationGoalProgressCurrent( + metric_value=float(row["metric_value"]), **common + ) # type: ignore[arg-type] + else: + return ProductivityGoalProgressCurrent( + metric_value=float(row["metric_value"]), **common + ) # type: ignore[arg-type] + + +async def upsert_goal_progress_current( + conn: aiosqlite.Connection, + *, + goal_definition_id: int, + period_start_utc: datetime, + period_end_utc: datetime, + metric_value: float, + success: Optional[bool], + eval_state: EvalState, + eval_state_reason: Optional[str], + updated_at_utc: datetime, +) -> None: + sql = """ + INSERT INTO goal_progress_current ( + goal_definition_id, period_start, period_end, metric_value, + success, eval_state, eval_state_reason, updated_at + ) VALUES (:goal_definition_id, :period_start, :period_end, :metric_value, + :success, :eval_state, :eval_state_reason, :updated_at) + ON CONFLICT(goal_definition_id) DO UPDATE SET + period_start = excluded.period_start, + period_end = excluded.period_end, + metric_value = excluded.metric_value, + success = excluded.success, + eval_state = excluded.eval_state, + eval_state_reason = excluded.eval_state_reason, + updated_at = excluded.updated_at + """ + await conn.execute( + sql, + { + "goal_definition_id": goal_definition_id, + "period_start": period_start_utc, + "period_end": period_end_utc, + "metric_value": metric_value, + "success": success, + "eval_state": eval_state.value, + "eval_state_reason": eval_state_reason, + "updated_at": updated_at_utc, + }, + ) + + +async def select_last_goal_results_for_goal( + conn: aiosqlite.Connection, *, goal_id: int, limit: int +) -> list[DBGoalResult]: + """Return the last N goal_results across all definitions for a goal. + + Results are ordered newest first by period_start. + """ + query = """ + SELECT r.id, r.goal_definition_id, r.period_start, r.period_end, + r.metric_value, r.success, r.eval_state, r.eval_state_reason, r.created_at + FROM goal_results r + JOIN goal_definitions d ON d.id = r.goal_definition_id + WHERE d.goal_id = ? + ORDER BY r.period_start DESC + LIMIT ? + """ + async with conn.execute(query, (goal_id, limit)) as cursor: + rows = await cursor.fetchall() + results: list[DBGoalResult] = [] + for rr in rows: + results.append( + DBGoalResult( + id=rr["id"], + goal_definition_id=rr["goal_definition_id"], + period_start=rr["period_start"], + period_end=rr["period_end"], + metric_value=rr["metric_value"], + success=bool(rr["success"]) if rr["success"] is not None else None, + eval_state=EvalState(rr["eval_state"]) + if rr["eval_state"] + else EvalState.NA, + eval_state_reason=rr["eval_state_reason"], + created_at=rr["created_at"], + ) + ) + return results + + +async def select_goal_with_latest_definition_by_definition_id( + conn: aiosqlite.Connection, *, goal_definition_id: int +) -> Optional[GoalWithLatestResult]: + # Fetch the definition row + async with conn.execute( + """ + SELECT gd.id, gd.goal_id, gd.name, gd.description, gd.metric_params_json, + gd.target_value, gd.include_mode, + gd.day_filter_json, gd.time_filter_json, gd.productivity_filter_json, + gd.effective_from, gd.created_at, + g.metric AS metric, g.operator AS operator, g.period AS period, g.timezone AS timezone, + g.id AS goal_id_real, + ( + SELECT at FROM goal_pause_events e + WHERE e.goal_id = g.id + ORDER BY datetime(at) DESC, id DESC + LIMIT 1 + ) AS goal_last_pause_event_at, + ( + SELECT event_type FROM goal_pause_events e + WHERE e.goal_id = g.id + ORDER BY datetime(at) DESC, id DESC + LIMIT 1 + ) AS goal_last_pause_event_type, + g.created_at AS goal_created_at + FROM goal_definitions gd + JOIN goals g ON g.id = gd.goal_id + WHERE gd.id = ? + """, + (goal_definition_id,), + ) as cursor: + dr = await cursor.fetchone() + if not dr: + return None + + # Build DBGoal from the joined row + metric = GoalMetric(dr["metric"]) # raises on invalid + operator = ( + MetricOperator(dr["operator"]) if dr["operator"] else MetricOperator.EQUAL + ) + period = GoalPeriod(dr["period"]) if dr["period"] else GoalPeriod.DAY + + # Coerce operator into allowed set + def _coerce_operator(m: GoalMetric, op: MetricOperator) -> MetricOperator: + if m == GoalMetric.AVG_PRODUCTIVITY_LEVEL: + return ( + op + if op in (MetricOperator.LESS_THAN, MetricOperator.GREATER_THAN) + else MetricOperator.GREATER_THAN + ) + if m == GoalMetric.TOTAL_ACTIVITY_DURATION: + return ( + op + if op in (MetricOperator.LESS_THAN, MetricOperator.GREATER_THAN) + else MetricOperator.GREATER_THAN + ) + return op + + operator = _coerce_operator(metric, operator) + paused_since_val = ( + dr["goal_last_pause_event_at"] + if dr["goal_last_pause_event_type"] == "pause" + else None + ) + if metric == GoalMetric.AVG_PRODUCTIVITY_LEVEL: + goal: DBGoal = DBAvgProductivityGoal( + id=dr["goal_id_real"], + created_at=dr["goal_created_at"], + paused_since=paused_since_val, + timezone=dr["timezone"], + period=period, + operator=cast(AvgProductivityOperators, operator), + ) + else: + goal = DBTotalActivityDurationGoal( + id=dr["goal_id_real"], + created_at=dr["goal_created_at"], + paused_since=paused_since_val, + timezone=dr["timezone"], + period=period, + operator=cast(TotalActivityDurationOperators, operator), + ) + # Fetch tags for this def + async with conn.execute( + """ + SELECT gt.tag_id, gt.role, t.name, t.description + FROM goal_tags gt + JOIN tags t ON t.id = gt.tag_id + WHERE gt.goal_definition_id = ? + """, + (goal_definition_id,), + ) as cursor: + tag_rows = await cursor.fetchall() + include_tags: list[DBTag] = [] + exclude_tags: list[DBTag] = [] + for tr in tag_rows: + tag = DBTag(id=tr["tag_id"], name=tr["name"], description=tr["description"]) + if tr["role"] == "INCLUDE": + include_tags.append(tag) + else: + exclude_tags.append(tag) + + definition = row_to_goal_definition(dr) + return GoalWithLatestResult( + goal=goal, + definition=definition, + include_tags=include_tags, + exclude_tags=exclude_tags, + latest_result=None, + last_successes=[], + ) + + +async def select_goal_with_definition_active_at( + conn: aiosqlite.Connection, *, goal_id: int, active_at_utc: datetime +) -> Optional[GoalWithLatestResult]: + """Return the goal and the definition in effect at the given UTC timestamp. + + Picks the goal_definition with the greatest effective_from <= active_at_utc. + Includes include/exclude tags for the chosen definition. + """ + # Choose the definition active at the provided timestamp + async with conn.execute( + """ + WITH chosen AS ( + SELECT gd.* + FROM goal_definitions gd + WHERE gd.goal_id = ? AND datetime(gd.effective_from) <= datetime(?) + ORDER BY gd.effective_from DESC + LIMIT 1 + ) + SELECT c.id, c.goal_id, c.name, c.description, c.metric_params_json, + c.target_value, c.include_mode, c.day_filter_json, c.time_filter_json, + c.productivity_filter_json, c.effective_from, c.created_at, + g.metric AS metric, g.operator AS operator, g.period AS period, + g.timezone AS timezone, + g.id AS goal_id_real, + ( + SELECT at FROM goal_pause_events e + WHERE e.goal_id = g.id + ORDER BY datetime(at) DESC, id DESC + LIMIT 1 + ) AS goal_last_pause_event_at, + ( + SELECT event_type FROM goal_pause_events e + WHERE e.goal_id = g.id + ORDER BY datetime(at) DESC, id DESC + LIMIT 1 + ) AS goal_last_pause_event_type, + g.created_at AS goal_created_at + FROM chosen c + JOIN goals g ON g.id = c.goal_id + """, + (goal_id, active_at_utc), + ) as cursor: + dr = await cursor.fetchone() + if not dr: + return None + + # Build DBGoal + metric = GoalMetric(dr["metric"]) # raises on invalid + operator = ( + MetricOperator(dr["operator"]) if dr["operator"] else MetricOperator.EQUAL + ) + period = GoalPeriod(dr["period"]) if dr["period"] else GoalPeriod.DAY + + def _coerce_operator(m: GoalMetric, op: MetricOperator) -> MetricOperator: + if m == GoalMetric.AVG_PRODUCTIVITY_LEVEL: + return ( + op + if op in (MetricOperator.LESS_THAN, MetricOperator.GREATER_THAN) + else MetricOperator.GREATER_THAN + ) + if m == GoalMetric.TOTAL_ACTIVITY_DURATION: + return ( + op + if op in (MetricOperator.LESS_THAN, MetricOperator.GREATER_THAN) + else MetricOperator.GREATER_THAN + ) + return op + + operator = _coerce_operator(metric, operator) + paused_since_val = ( + dr["goal_last_pause_event_at"] + if dr["goal_last_pause_event_type"] == "pause" + else None + ) + if metric == GoalMetric.AVG_PRODUCTIVITY_LEVEL: + goal: DBGoal = DBAvgProductivityGoal( + id=dr["goal_id_real"], + created_at=dr["goal_created_at"], + paused_since=paused_since_val, + timezone=dr["timezone"], + period=period, + operator=cast(AvgProductivityOperators, operator), + ) + else: + goal = DBTotalActivityDurationGoal( + id=dr["goal_id_real"], + created_at=dr["goal_created_at"], + paused_since=paused_since_val, + timezone=dr["timezone"], + period=period, + operator=cast(TotalActivityDurationOperators, operator), + ) + + # Tags for the chosen definition + async with conn.execute( + """ + SELECT gt.tag_id, gt.role, t.name, t.description + FROM goal_tags gt + JOIN tags t ON t.id = gt.tag_id + WHERE gt.goal_definition_id = ? + """, + (dr["id"],), + ) as cursor: + tag_rows = await cursor.fetchall() + include_tags: list[DBTag] = [] + exclude_tags: list[DBTag] = [] + for tr in tag_rows: + tag = DBTag(id=tr["tag_id"], name=tr["name"], description=tr["description"]) + if tr["role"] == "INCLUDE": + include_tags.append(tag) + else: + exclude_tags.append(tag) + + definition = row_to_goal_definition(dr) + return GoalWithLatestResult( + goal=goal, + definition=definition, + include_tags=include_tags, + exclude_tags=exclude_tags, + latest_result=None, + last_successes=[], + ) + + +async def insert_goal_pause_event( + conn: aiosqlite.Connection, *, goal_id: int, event_type: str, at: datetime +) -> int: + """Insert a pause/resume event for a goal. + + event_type must be 'pause' or 'resume'. Returns inserted row id. + """ + assert event_type in ("pause", "resume") + cursor = await conn.execute( + """ + INSERT INTO goal_pause_events (goal_id, event_type, at) + VALUES (:goal_id, :event_type, :at) + """, + {"goal_id": goal_id, "event_type": event_type, "at": at}, + ) + rowid = cursor.lastrowid + assert rowid is not None, "Failed to insert goal pause event" + return int(rowid) + + +async def select_latest_goal_definition_id_for_goal( + conn: aiosqlite.Connection, *, goal_id: int +) -> Optional[int]: + async with conn.execute( + """ + SELECT gd.id + FROM goal_definitions gd + WHERE gd.goal_id = ? + ORDER BY gd.created_at DESC + LIMIT 1 + """, + (goal_id,), + ) as cursor: + row = await cursor.fetchone() + return row["id"] if row else None + + +async def select_latest_progress_updated_at_for_goal( + conn: aiosqlite.Connection, *, goal_id: int +) -> Optional[datetime]: + async with conn.execute( + """ + SELECT gpc.updated_at AS updated_at + FROM goal_definitions gd + LEFT JOIN goal_progress_current gpc ON gpc.goal_definition_id = gd.id + WHERE gd.goal_id = ? + ORDER BY gd.created_at DESC + LIMIT 1 + """, + (goal_id,), + ) as cursor: + row = await cursor.fetchone() + return row["updated_at"] if row else None + + +async def insert_goal_result( + conn: aiosqlite.Connection, + *, + goal_definition_id: int, + period_start_utc: datetime, + period_end_utc: datetime, + metric_value: float, + success: Optional[bool], + eval_state: EvalState, + eval_state_reason: Optional[str], +) -> int: + """Insert or update a goal_results row for a definition and period. + + Uses ON CONFLICT(goal_definition_id, period_start) DO UPDATE to be idempotent. + Returns the lastrowid (may be 0 for updates in SQLite). + """ + async with conn.execute( + """ + INSERT INTO goal_results ( + goal_definition_id, period_start, period_end, metric_value, + success, eval_state, eval_state_reason + ) VALUES (:goal_definition_id, :period_start, :period_end, :metric_value, + :success, :eval_state, :eval_state_reason) + ON CONFLICT(goal_definition_id, period_start) DO UPDATE SET + period_end = excluded.period_end, + metric_value = excluded.metric_value, + success = excluded.success, + eval_state = excluded.eval_state, + eval_state_reason = excluded.eval_state_reason + """, + { + "goal_definition_id": goal_definition_id, + "period_start": period_start_utc, + "period_end": period_end_utc, + "metric_value": float(metric_value), + "success": (1 if success is True else 0 if success is False else None), + "eval_state": eval_state.value + if hasattr(eval_state, "value") + else str(eval_state), + "eval_state_reason": eval_state_reason, + }, + ) as cursor: + return cursor.lastrowid or 0 + + +def row_to_goal_definition(r: Any) -> DBGoalDefinition: + # Determine metric from joined row and map include mode and filters + metric = GoalMetric(r["metric"]) # type: ignore[arg-type] + include_mode = IncludeMode(r["include_mode"]) # type: ignore[arg-type] + day_filter = json.loads(r["day_filter_json"]) if r["day_filter_json"] else None + productivity_filter = None + if r["productivity_filter_json"]: + raw = json.loads(r["productivity_filter_json"]) # Expect list of strings + productivity_filter = [ProductivityLevel(v) for v in raw] + time_filter = json.loads(r["time_filter_json"]) if r["time_filter_json"] else None + + base_kwargs = { + "id": r["id"], + "goal_id": r["goal_id"], + "name": r["name"], + "description": r["description"], + "include_mode": include_mode, + "day_filter": day_filter, + "productivity_filter": productivity_filter, + "effective_from": r["effective_from"], + "time_filter": time_filter, + } + + match metric: + case GoalMetric.AVG_PRODUCTIVITY_LEVEL: + return DBAvgProductivityGoalDefinition( + **base_kwargs, + target_value=r["target_value"], + metric_params=None, + ) + case GoalMetric.TOTAL_ACTIVITY_DURATION: + # Interpret target_value as seconds for duration + seconds = float(r["target_value"]) if r["target_value"] is not None else 0.0 + return DBTotalActivityDurationGoalDefinition( + **base_kwargs, + target_value=timedelta(seconds=seconds), + metric_params=None, + ) + case _: + raise ValueError(f"Unknown goal metric: {metric}") + + +# ---- Goal inserts ---- +async def insert_goal( + conn: aiosqlite.Connection, + *, + metric: GoalMetric, + operator: MetricOperator, + period: str, + timezone: str, +) -> int: + sql = simple_insert_query("goals", ("metric", "operator", "period", "timezone")) + cursor = await conn.execute( + sql, + { + "metric": metric.value, + "operator": operator.value, + "period": period, + "timezone": timezone, + }, + ) + last_row_id = cursor.lastrowid + assert last_row_id is not None, "No goal row was inserted" + return int(last_row_id) + + +async def insert_goal_definition( + conn: aiosqlite.Connection, + *, + goal_id: int, + name: str, + description: str | None, + target_value: float, + include_mode: IncludeMode, + day_filter_json: str | None, + time_filter_json: str | None, + productivity_filter_json: str | None, + effective_from: datetime, + metric_params_json: str | None = None, +) -> int: + sql = simple_insert_query( + "goal_definitions", + ( + "goal_id", + "name", + "description", + "metric_params_json", + "target_value", + "include_mode", + "day_filter_json", + "time_filter_json", + "productivity_filter_json", + "effective_from", + ), + ) + cursor = await conn.execute( + sql, + { + "goal_id": goal_id, + "name": name, + "description": description, + "metric_params_json": metric_params_json, + "target_value": target_value, + "include_mode": include_mode.value, + "day_filter_json": day_filter_json, + "time_filter_json": time_filter_json, + "productivity_filter_json": productivity_filter_json, + "effective_from": effective_from, + }, + ) + last_row_id = cursor.lastrowid + assert last_row_id is not None, "No goal_definition row was inserted" + return int(last_row_id) + + +async def insert_goal_tags( + conn: aiosqlite.Connection, + *, + goal_definition_id: int, + include_tag_ids: list[int], + exclude_tag_ids: list[int], +) -> None: + if not include_tag_ids and not exclude_tag_ids: + return + rows: list[dict[str, Any]] = [] + for tid in include_tag_ids: + rows.append( + { + "goal_definition_id": goal_definition_id, + "tag_id": tid, + "role": "include", + } + ) + for tid in exclude_tag_ids: + rows.append( + { + "goal_definition_id": goal_definition_id, + "tag_id": tid, + "role": "exclude", + } + ) + sql = simple_insert_query("goal_tags", ("goal_definition_id", "tag_id", "role")) + await conn.executemany(sql, rows) + + +async def delete_goal(conn: aiosqlite.Connection, goal_id: int) -> None: + """Delete a goal by ID. Cascades will remove related definitions, tags, and results. + + Assumes PRAGMA foreign_keys=ON in the connection (set by the DB context manager). + """ + await conn.execute("DELETE FROM goals WHERE id = ?", (goal_id,)) + + +async def update_activity_event( + conn: aiosqlite.Connection, db_activity_event: DBActivityEvent +) -> None: + assert db_activity_event.id is not None, "Event ID cannot be None" + query = """ + UPDATE activity_events + SET event_time = :event_time, + event_type = :event_type, + aggregation_id = :aggregation_id, + last_manual_action_time = :last_manual_action_time + WHERE id = :id; + """ + await conn.execute( + query, + { + "event_time": db_activity_event.event_time, + "event_type": db_activity_event.event_type, + "aggregation_id": db_activity_event.aggregation_id, + "id": db_activity_event.id, + "last_manual_action_time": db_activity_event.last_manual_action_time, + }, + ) + + +async def select_specs_in_time_range( + conn: aiosqlite.Connection, start: datetime, end: datetime +) -> list[DBActivitySpec]: + activity_ids_query = """ + SELECT DISTINCT a.id + FROM activities a + WHERE ( + EXISTS ( + SELECT 1 + FROM activity_events ae + WHERE ae.activity_id = a.id + AND ae.event_type = 'open' + AND datetime(ae.event_time) >= datetime(:start) + AND datetime(ae.event_time) <= datetime(:end) + ) + OR ( + SELECT ae.event_type + FROM activity_events ae + WHERE ae.activity_id = a.id + AND datetime(ae.event_time) < datetime(:start) + ORDER BY ae.event_time DESC, ae.id DESC + LIMIT 1 + ) = 'open' + ) + """ + + async with conn.execute(activity_ids_query, {"start": start, "end": end}) as cursor: + rows = await cursor.fetchall() + activity_ids = [row["id"] for row in rows] + + if not activity_ids: + return [] + + placeholders = ",".join("?" for _ in activity_ids) + specs_query = f""" + SELECT + a.id AS activity_id, + a.name, + a.description, + a.productivity_level, + a.source, + a.last_manual_action_time AS activity_last_manual_action_time, + e.id AS event_id, + e.event_time, + e.event_type, + e.aggregation_id, + e.last_manual_action_time AS event_last_manual_action_time + FROM activities a + JOIN activity_events e ON a.id = e.activity_id + WHERE a.id IN ({placeholders}) + ORDER BY a.id, datetime(e.event_time), e.id + """ + + async with conn.execute(specs_query, activity_ids) as cursor: + rows = await cursor.fetchall() + + grouped: dict[int, list[Any]] = defaultdict(list) + for row in rows: + grouped[row["activity_id"]].append(row) + + specs: list[DBActivitySpec] = [] + for activity_id, activity_rows in grouped.items(): + first_row = activity_rows[0] + db_activity = DBActivity( + id=activity_id, + name=first_row["name"], + description=first_row["description"], + productivity_level=first_row["productivity_level"], + source=first_row["source"], + last_manual_action_time=first_row["activity_last_manual_action_time"], + ) + + events: list[DBActivityEvent] = [] + for row in activity_rows: + events.append( + DBActivityEvent( + event_time=row["event_time"], + event_type=row["event_type"], + id=row["event_id"], + activity_id=activity_id, + aggregation_id=row["aggregation_id"], + last_manual_action_time=row["event_last_manual_action_time"], + ) + ) + + specs.append(DBActivitySpec(activity=db_activity, events=events)) + + return specs + + +async def select_specs_with_tags_in_time_range( + conn: aiosqlite.Connection, start: datetime, end: datetime +) -> list[DBActivitySpecWithTags]: + # First get regular activity specs + activity_specs = await select_specs_in_time_range(conn, start, end) + + # Now fetch tags for each activity + tags_by_activity = {} + + # Get all activity IDs to look up tags + activity_ids = [ + spec.activity.id for spec in activity_specs if spec.activity.id is not None + ] + + if not activity_ids: + # Return early if there are no activities + return [ + DBActivitySpecWithTags(**spec.model_dump(), tags=[]) + for spec in activity_specs + ] + + # Create placeholders for SQL query + placeholders = ",".join("?" for _ in activity_ids) + + # Query to get all tags for these activities + query = f""" + SELECT a.id AS activity_id, t.id AS tag_id, t.name, t.description + FROM activities a + LEFT JOIN tag_mappings tm ON a.id = tm.activity_id + LEFT JOIN tags t ON tm.tag_id = t.id AND t.deleted_at IS NULL + WHERE a.id IN ({placeholders}) + """ + + async with conn.execute(query, activity_ids) as cursor: + rows = await cursor.fetchall() + + # Group tags by activity + for row in rows: + activity_id = row["activity_id"] + tag_id = row["tag_id"] + + if activity_id not in tags_by_activity: + tags_by_activity[activity_id] = [] + if tag_id is not None: + tag = DBTag(id=tag_id, name=row["name"], description=row["description"]) + tags_by_activity[activity_id].append(tag) + + # Combine activities with their tags + result = [] + for spec in activity_specs: + activity_id = spec.activity.id + tags = tags_by_activity.get(activity_id, []) + + # Create enhanced spec with tags + spec_with_tags = DBActivitySpecWithTags(**spec.model_dump(), tags=tags) + result.append(spec_with_tags) + + return result + + +async def select_last_aggregation(conn: aiosqlite.Connection) -> DBAggregation | None: + query = """ + SELECT id, start_time, end_time, first_timestamp, last_timestamp + FROM aggregations + ORDER BY end_time DESC + LIMIT 1; + """ + async with conn.execute(query) as cursor: + row = await cursor.fetchone() + if row: + return DBAggregation( + id=row["id"], + start_time=row["start_time"], + end_time=row["end_time"], + first_timestamp=row["first_timestamp"], + last_timestamp=row["last_timestamp"], + ) + + +async def select_latest_aggregation( + conn: aiosqlite.Connection, +) -> DBAggregation | None: + query = """ + SELECT id, start_time, end_time, first_timestamp, last_timestamp + FROM aggregations + ORDER BY end_time DESC + LIMIT 1; + """ + async with conn.execute(query) as cursor: + row = await cursor.fetchone() + if row: + return DBAggregation( + id=row["id"], + start_time=row["start_time"], + end_time=row["end_time"], + first_timestamp=row["first_timestamp"], + last_timestamp=row["last_timestamp"], + ) + return None + + +async def select_sources(conn: aiosqlite.Connection) -> list[DBDeviceSource]: + query = """ + SELECT id, name, source_type, status, token_hash, last_seen, created_at + FROM sources + ORDER BY (last_seen IS NULL), datetime(last_seen) DESC, id DESC + """ + async with conn.execute(query) as cursor: + rows = await cursor.fetchall() + result: list[DBDeviceSource] = [] + for row in rows: + result.append( + DBDeviceSource( + id=row["id"], + name=row["name"], + source_type=SourceType(row["source_type"]), + status=SourceStatus(row["status"]), + token_hash=row["token_hash"], + last_seen=row["last_seen"], + created_at=row["created_at"], + ) + ) + return result + + +async def toggle_source_status( + conn: aiosqlite.Connection, source_id: int +) -> SourceStatus: + status = await conn.execute( + """ + UPDATE sources + SET status = CASE + WHEN status = :active THEN :revoked + ELSE :active + END + WHERE id = :source_id + RETURNING status + """, + { + "source_id": source_id, + "active": SourceStatus.ACTIVE.value, + "revoked": SourceStatus.REVOKED.value, + }, + ) + + row = await status.fetchone() + assert row is not None, "No source found with the given ID" + + return SourceStatus(row["status"]) + + +async def delete_source( + conn: aiosqlite.Connection, source_id: int +) -> DBDeviceSource | None: + async with conn.execute( + """ + DELETE FROM sources + WHERE id = :source_id + RETURNING id, name, source_type, status, token_hash, last_seen, created_at + """, + {"source_id": source_id}, + ) as cursor: + row = await cursor.fetchone() + if not row: + return None + return DBDeviceSource( + id=row["id"], + name=row["name"], + source_type=SourceType(row["source_type"]), + status=SourceStatus(row["status"]), + token_hash=row["token_hash"], + last_seen=row["last_seen"], + created_at=row["created_at"], + ) + + +async def insert_source( + conn: aiosqlite.Connection, + *, + name: str, + source_type: SourceType, + token_hash: str, + status: SourceStatus = SourceStatus.ACTIVE, +) -> DBDeviceSource: + await conn.execute( + """ + INSERT INTO sources (name, source_type, token_hash, status) + VALUES (:name, :source_type, :token_hash, :status) + """, + { + "name": name, + "source_type": source_type.value, + "token_hash": token_hash, + "status": status.value, + }, + ) + # Fetch the row we just inserted + async with conn.execute( + """ + SELECT id, name, source_type, status, token_hash, last_seen, created_at + FROM sources WHERE name = :name + """, + {"name": name}, + ) as cursor: + row = await cursor.fetchone() + assert row is not None, "Failed to fetch inserted source" + return DBDeviceSource( + id=row["id"], + name=row["name"], + source_type=SourceType(row["source_type"]), + status=SourceStatus(row["status"]), + token_hash=row["token_hash"], + last_seen=row["last_seen"], + created_at=row["created_at"], + ) + + +async def insert_source_enrollment_code( + conn: aiosqlite.Connection, + *, + code_hash: str, + expires_at: datetime | None, +) -> int: + cursor = await conn.execute( + """ + INSERT INTO source_enrollment_codes (code_hash, expires_at, used) + VALUES (:code_hash, :expires_at, 0) + """, + {"code_hash": code_hash, "expires_at": expires_at}, + ) + last_id = cursor.lastrowid + return int(last_id) if last_id is not None else 0 + + +async def deactivate_active_enrollment_codes(conn: aiosqlite.Connection) -> None: + # Remove any existing enrollment codes so only one can exist at a time. + await conn.execute("DELETE FROM source_enrollment_codes") + + +async def select_current_enrollment_code( + conn: aiosqlite.Connection, +) -> Optional[Any]: + async with conn.execute( + "SELECT id, code_hash, expires_at FROM source_enrollment_codes LIMIT 1" + ) as cursor: + row = await cursor.fetchone() + + if row is None: + return None + + return SourceEnrollmentCode( + id=row["id"], code_hash=row["code_hash"], expires_at=row["expires_at"] + ) + + +async def delete_current_enrollment_code(conn: aiosqlite.Connection) -> None: + await conn.execute("DELETE FROM source_enrollment_codes") + + +async def select_source_by_token_hash( + conn: aiosqlite.Connection, token_hash: str +) -> Optional[DBDeviceSource]: + async with conn.execute( + """ + SELECT id, name, source_type, status, token_hash, last_seen, created_at + FROM sources WHERE token_hash = :token_hash + """, + {"token_hash": token_hash}, + ) as cursor: + row = await cursor.fetchone() + if not row: + return None + return DBDeviceSource( + id=row["id"], + name=row["name"], + source_type=SourceType(row["source_type"]), + status=SourceStatus(row["status"]), + token_hash=row["token_hash"], + last_seen=row["last_seen"], + created_at=row["created_at"], + ) + + +async def update_source_last_seen( + conn: aiosqlite.Connection, source_id: int, *, when: datetime +) -> None: + await conn.execute( + "UPDATE sources SET last_seen = :when WHERE id = :id", + {"id": source_id, "when": when}, + ) + + +async def select_auto_activities_closed_within_time_range_with_last_event( + conn: aiosqlite.Connection, start: datetime, end: datetime +) -> list[DBActivityWithLatestEvent]: + query = """ + SELECT + a.id, a.name, a.description, a.productivity_level, a.source, + a.last_manual_action_time AS activity_last_manual_action_time, + le.event_type AS state, + le.event_time, + le.aggregation_id, + le.id AS last_event_id, + ag.start_time as aggregation_start_time, + ag.end_time as aggregation_end_time, + ag.first_timestamp as aggregation_first_timestamp, + ag.last_timestamp as aggregation_last_timestamp, + le.last_manual_action_time AS event_last_manual_action_time + + FROM activities a + + JOIN ( + SELECT * + FROM activity_events ae + WHERE ae.id = ( + SELECT ae2.id + FROM activity_events ae2 + WHERE ae2.activity_id = ae.activity_id + ORDER BY ae2.event_time DESC + LIMIT 1 + ) + ) le ON le.activity_id = a.id + LEFT JOIN aggregations ag ON le.aggregation_id = ag.id + WHERE a.source = 'auto' + AND le.event_type = 'close' + AND datetime(le.event_time) >= datetime(?) + AND datetime(le.event_time) <= datetime(?); + """ + + return_list = [] + async with conn.execute(query, (start, end)) as cursor: + rows = await cursor.fetchall() + for row in rows: + db_activity = DBActivity( + id=row["id"], + name=row["name"], + description=row["description"], + productivity_level=row["productivity_level"], + source=row["source"], + last_manual_action_time=row["activity_last_manual_action_time"], + ) + + db_activity_event = DBActivityEvent( + event_time=row["event_time"], + event_type=row["state"], + id=row["last_event_id"], + activity_id=row["id"], + aggregation_id=row["aggregation_id"], + last_manual_action_time=row["event_last_manual_action_time"], + ) + latest_aggregation = DBAggregation( + id=row["aggregation_id"], + start_time=row["aggregation_start_time"], + end_time=row["aggregation_end_time"], + first_timestamp=row["aggregation_first_timestamp"], + last_timestamp=row["aggregation_last_timestamp"], + ) + return_list.append( + DBActivityWithLatestEvent( + activity=db_activity, + latest_event=db_activity_event, + latest_aggregation=latest_aggregation, + ) + ) + return return_list + + +async def select_manual_activity_specs_active_within_time_range( + conn: aiosqlite.Connection, start: datetime, end: datetime +) -> list[DBActivitySpec]: + """ + Fetches all manual activities specs that are active within a given time range. + An activity is considered active if it's a manual activity and: + 1. It has an 'open' event within the specified time range (inclusive). + 2. Its last event before the start of the time range was 'open'. + """ + activity_ids_query = """ + SELECT DISTINCT a.id + FROM activities a + WHERE a.source = 'manual' + AND ( + -- Condition 1: Has an open event within the time range + EXISTS ( + SELECT 1 + FROM activity_events ae + WHERE ae.activity_id = a.id + AND ae.event_time >= :start + AND ae.event_time <= :end + AND ae.event_type = 'open' + ) + OR + -- Condition 2: Last event before start is 'open' + ( + SELECT ae.event_type + FROM activity_events ae + WHERE ae.activity_id = a.id AND ae.event_time < :start + ORDER BY ae.event_time DESC + LIMIT 1 + ) = 'open' + ) + """ + async with conn.execute(activity_ids_query, {"start": start, "end": end}) as cursor: + rows = await cursor.fetchall() + activity_ids = [row["id"] for row in rows] + + if not activity_ids: + return [] + + # Now fetch the full specs for these activities. + placeholders = ",".join("?" for _ in activity_ids) + specs_query = f""" + SELECT a.id AS activity_id, + a.name, + a.description, + a.productivity_level, + a.source, + a.last_manual_action_time AS activity_last_manual_action_time, + e.id AS event_id, + e.event_time, + e.event_type, + e.aggregation_id, + e.last_manual_action_time AS event_last_manual_action_time + FROM activities a + JOIN activity_events e ON a.id = e.activity_id + WHERE a.id IN ({placeholders}); + """ + + async with conn.execute(specs_query, activity_ids) as cursor: + rows = await cursor.fetchall() + + grouped = defaultdict(list) + for row in rows: + activity_id = row["activity_id"] + grouped[activity_id].append(row) + + activity_specs = [] + for activity_id, activity_rows in grouped.items(): + first_row = activity_rows[0] + + db_activity = DBActivity( + id=activity_id, + name=first_row["name"], + description=first_row["description"], + productivity_level=first_row["productivity_level"], + source=first_row["source"], + last_manual_action_time=first_row["activity_last_manual_action_time"], + ) + + events = [] + for row in activity_rows: + db_activity_event = DBActivityEvent( + event_time=row["event_time"], + event_type=row["event_type"], + id=row["event_id"], + activity_id=activity_id, + aggregation_id=row["aggregation_id"], + last_manual_action_time=row["event_last_manual_action_time"], + ) + events.append(db_activity_event) + + db_activity_spec = DBActivitySpec(activity=db_activity, events=events) + activity_specs.append(db_activity_spec) + + return activity_specs + + +async def insert_candidate_sessions( + conn: aiosqlite.Connection, + *, + sessionization_run_id: int, + sessions: Sequence[CandidateSession], +) -> list[int]: + if not sessions: + return [] + + placeholders = ", ".join("(?, ?, ?)" for _ in sessions) + values: list[Any] = [] + for session in sessions: + values.extend([session.name, session.llm_id, sessionization_run_id]) + + sql = f""" + INSERT INTO candidate_sessions ( + name, + llm_id, + sessionization_run_id + ) + VALUES {placeholders} + RETURNING + id + """ + + async with conn.execute(sql, values) as cursor: + rows = await cursor.fetchall() + + return [int(row["id"]) for row in rows] + + +async def insert_candidate_session_to_activity( + conn: aiosqlite.Connection, + *, + mappings: Sequence[CandidateSessionToActivity], +) -> None: + if not mappings: + return + + await conn.executemany( + """ + INSERT INTO candidate_session_to_activity ( + session_id, + activity_id + ) + VALUES (?, ?) + """, + [(mapping.candidate_session_id, mapping.activity_id) for mapping in mappings], + ) + + +async def delete_candidate_session_to_activity_by_activity_ids( + conn: aiosqlite.Connection, + *, + activity_ids: Sequence[int], +) -> None: + # Delete candidate session mappings for the given activity ids in a single query. + if not activity_ids: + return + + placeholders = ",".join("?" for _ in activity_ids) + await conn.execute( + f""" + DELETE FROM candidate_session_to_activity + WHERE activity_id IN ({placeholders}) + """, + list(activity_ids), + ) + + +async def delete_candidate_sessions_by_ids( + conn: aiosqlite.Connection, + *, + candidate_session_ids: Sequence[int], +) -> None: + if not candidate_session_ids: + return + + placeholders = ",".join("?" for _ in candidate_session_ids) + await conn.execute( + f""" + DELETE FROM candidate_sessions + WHERE id IN ({placeholders}) + """, + list(candidate_session_ids), + ) + + +async def delete_candidate_sessions_without_activities( + conn: aiosqlite.Connection, +) -> int: + # Delete all candidate sessions that have no activity mappings. Returns the number of rows deleted. + cursor = await conn.execute( + """ + DELETE FROM candidate_sessions + WHERE id NOT IN ( + SELECT DISTINCT session_id + FROM candidate_session_to_activity + ) + """ + ) + return cursor.rowcount or 0 + + +async def select_candidate_session_specs( + conn: aiosqlite.Connection, + *, + sessionization_run_id: int | None = None, +) -> list[DBCandidateSessionSpec]: + where_clause = "" + params: list[Any] = [] + if sessionization_run_id is not None: + where_clause = "WHERE cs.sessionization_run_id = ?" + params.append(sessionization_run_id) + + async with conn.execute( + f""" + SELECT + cs.id, + cs.name, + cs.llm_id, + cs.sessionization_run_id, + cs.created_at, + ( + SELECT GROUP_CONCAT(sub.activity_id, ',') + FROM ( + SELECT activity_id + FROM candidate_session_to_activity csta + WHERE csta.session_id = cs.id + ORDER BY datetime(csta.created_at) ASC, csta.activity_id ASC + ) AS sub + ) AS activity_ids_csv + FROM candidate_sessions cs + {where_clause} + ORDER BY datetime(cs.created_at) ASC, cs.id ASC + """, + params, + ) as cursor: + rows = await cursor.fetchall() + + specs: list[DBCandidateSessionSpec] = [] + for row in rows: + activity_ids_csv = row["activity_ids_csv"] + activity_ids = ( + [int(value) for value in activity_ids_csv.split(",") if value] + if activity_ids_csv + else [] + ) + + specs.append( + DBCandidateSessionSpec( + session=DBCandidateSession( + id=row["id"], + name=row["name"], + llm_id=row["llm_id"], + sessionization_run_id=row["sessionization_run_id"], + created_at=row["created_at"], + ), + activity_ids=activity_ids, + ) + ) + + return specs + + +async def insert_sessionization_run( + conn: aiosqlite.Connection, + *, + sessionization_run: SessionizationRun, +) -> int: + cursor = await conn.execute( + """ + INSERT INTO sessionization_run ( + candidate_creation_start, + candidate_creation_end, + overlap_start, + right_tail_end, + finalized_horizon + ) + VALUES (?, ?, ?, ?, ?) + RETURNING + id + """, + ( + sessionization_run.candidate_creation_start, + sessionization_run.candidate_creation_end, + sessionization_run.overlap_start, + sessionization_run.right_tail_end, + sessionization_run.finalized_horizon, + ), + ) + + row = await cursor.fetchone() + assert row is not None, "sessionization_run insert did not return a row" + + return row["id"] + + +async def select_latest_sessionization_run( + conn: aiosqlite.Connection, +) -> DBSessionizationRun | None: + async with conn.execute( + """ + SELECT + id, + created_at, + candidate_creation_start, + candidate_creation_end, + overlap_start, + right_tail_end, + finalized_horizon + FROM sessionization_run + ORDER BY created_at DESC, id DESC + LIMIT 1 + """ + ) as cursor: + row = await cursor.fetchone() + + if row is None: + return None + + return DBSessionizationRun( + id=row["id"], + created_at=row["created_at"], + candidate_creation_start=row["candidate_creation_start"], + candidate_creation_end=row["candidate_creation_end"], + overlap_start=row["overlap_start"], + right_tail_end=row["right_tail_end"], + finalized_horizon=row["finalized_horizon"], + ) + + +async def insert_sessions( + conn: aiosqlite.Connection, + *, + sessionization_run_id: int, + sessions: Sequence[Session], +) -> list[int]: + if not sessions: + return [] + + insert_sql = """ + INSERT INTO sessions ( + name, + llm_id, + sessionization_run_id + ) + VALUES (?, ?, ?) + """ + + inserted_ids: list[int] = [] + for session in sessions: + cursor = await conn.execute( + insert_sql, + ( + session.name, + session.llm_id, + sessionization_run_id, + ), + ) + last_row_id = cursor.lastrowid + assert last_row_id is not None, "sessions insert did not return an id" + inserted_ids.append(last_row_id) + + return inserted_ids + + +async def insert_session_to_activity( + conn: aiosqlite.Connection, + mappings: Sequence[tuple[int, int]], +) -> None: + if not mappings: + return + + await conn.executemany( + """ + INSERT INTO session_to_activity (session_id, activity_id) + VALUES (?, ?) + """, + mappings, + ) + + +async def select_specs_with_tags_and_sessions_in_time_range( + conn: aiosqlite.Connection, start: datetime, end: datetime +) -> list[DBActivitySpecWithTagsAndSessions]: + """Get activity specs with tags, finalized sessions, and candidate sessions in the given time range. + + Uses the same activity filtering criteria as select_specs_with_tags_in_time_range. + """ + # First get regular activity specs + activity_specs = await select_specs_in_time_range(conn, start, end) + + if not activity_specs: + return [] + + # Get all activity IDs + activity_ids = [ + spec.activity.id for spec in activity_specs if spec.activity.id is not None + ] + + if not activity_ids: + return [] + + placeholders = ",".join("?" for _ in activity_ids) + + # Query to get all tags, sessions, and candidate sessions for these activities + query = f""" + SELECT + a.id AS activity_id, + -- Tags + t.id AS tag_id, + t.name AS tag_name, + t.description AS tag_description, + -- Finalized sessions (can only be one per activity due to UNIQUE constraint) + s.id AS session_id, + s.name AS session_name, + s.llm_id AS session_llm_id, + s.created_at AS session_created_at, + s.sessionization_run_id AS session_run_id, + -- Candidate sessions (can be multiple) + cs.id AS candidate_session_id, + cs.name AS candidate_session_name, + cs.llm_id AS candidate_session_llm_id, + cs.created_at AS candidate_session_created_at, + cs.sessionization_run_id AS candidate_session_run_id + FROM activities a + LEFT JOIN tag_mappings tm ON a.id = tm.activity_id + LEFT JOIN tags t ON tm.tag_id = t.id AND t.deleted_at IS NULL + LEFT JOIN session_to_activity sta ON a.id = sta.activity_id + LEFT JOIN sessions s ON sta.session_id = s.id + LEFT JOIN candidate_session_to_activity csta ON a.id = csta.activity_id + LEFT JOIN candidate_sessions cs ON csta.session_id = cs.id + WHERE a.id IN ({placeholders}) + """ + + async with conn.execute(query, activity_ids) as cursor: + rows = await cursor.fetchall() + + # Group data by activity + tags_by_activity: dict[int, list[DBTag]] = defaultdict(list) + session_by_activity: dict[int, DBSession] = {} + candidate_sessions_by_activity: dict[int, list[DBCandidateSession]] = defaultdict( + list + ) + + for row in rows: + activity_id = row["activity_id"] + + # Collect tags + tag_id = row["tag_id"] + if tag_id is not None: + tag = DBTag( + id=tag_id, name=row["tag_name"], description=row["tag_description"] + ) + # Avoid duplicates + if tag not in tags_by_activity[activity_id]: + tags_by_activity[activity_id].append(tag) + + # Collect finalized session (only one per activity) + session_id = row["session_id"] + if session_id is not None and activity_id not in session_by_activity: + session_by_activity[activity_id] = DBSession( + id=session_id, + name=row["session_name"], + llm_id=row["session_llm_id"], + created_at=row["session_created_at"], + sessionization_run_id=row["session_run_id"], + ) + + # Collect candidate sessions (can be multiple) + candidate_session_id = row["candidate_session_id"] + if candidate_session_id is not None: + candidate_session = DBCandidateSession( + id=candidate_session_id, + name=row["candidate_session_name"], + llm_id=row["candidate_session_llm_id"], + created_at=row["candidate_session_created_at"], + sessionization_run_id=row["candidate_session_run_id"], + ) + # Avoid duplicates + if candidate_session not in candidate_sessions_by_activity[activity_id]: + candidate_sessions_by_activity[activity_id].append(candidate_session) + + # Combine activities with their relationships + result = [] + for spec in activity_specs: + activity_id = spec.activity.id + tags = tags_by_activity.get(activity_id, []) + session = session_by_activity.get(activity_id, None) + candidate_sessions = candidate_sessions_by_activity.get(activity_id, []) + + # Create enhanced spec with tags and sessions + spec_with_relationships = DBActivitySpecWithTagsAndSessions( + **spec.model_dump(), + tags=tags, + session=session, + candidate_sessions=candidate_sessions, + ) + result.append(spec_with_relationships) + + return result diff --git a/src/clepsy/entities.py b/src/clepsy/entities.py index 976b7e8..23c94c0 100755 --- a/src/clepsy/entities.py +++ b/src/clepsy/entities.py @@ -498,6 +498,36 @@ class SourceStatus(StrEnum): REVOKED = "revoked" +class ScheduleStatus(StrEnum): + IDLE = "idle" + ERROR = "error" + DISABLED = "disabled" + + +class JobType(StrEnum): + GOAL_UPDATE_CURRENT_PROGRESS = "goals_update_current_progress" + GOAL_UPDATE_PREVIOUS_PERIOD = "goals_update_previous_period" + AGGREGATION_WINDOW = "aggregation_window" + SESSIONIZATION = "sessions_sessionization" + + +class ScheduledJob(BaseModel): + schedule_key: str + job_type: JobType + cron_expr: str | None = None + next_run_at: datetime + enabled: bool = True + payload: dict[str, Any] | None = None + running_count: int = 0 + max_concurrent: int = 1 + last_started_at: datetime | None = None + status: ScheduleStatus = ScheduleStatus.IDLE + + +class DBScheduledJob(ScheduledJob): + id: int + + class DeviceSource(BaseModel): name: str source_type: SourceType @@ -855,3 +885,8 @@ class DBCandidateSessionSpec(BaseModel): def __hash__(self): return hash((self.session.id, tuple(self.activity_ids))) + + +class ScheduleUpdate(BaseModel): + schedule_id: int + next_run_at: datetime diff --git a/src/clepsy/infra/streams.py b/src/clepsy/infra/streams.py index f94ae1e..4bef251 100755 --- a/src/clepsy/infra/streams.py +++ b/src/clepsy/infra/streams.py @@ -1,13 +1,56 @@ +# ruff: noqa: I001 + +"""Utilities for interacting with the Valkey source events stream.""" + from datetime import datetime, timezone from typing import Iterable from loguru import logger -from valkey.exceptions import ResponseError as StreamResponseError # type: ignore +from valkey.exceptions import ( + LockNotOwnedError, + ResponseError as StreamResponseError, +) # type: ignore from clepsy.infra.valkey_client import get_connection SOURCE_EVENTS_STREAM = "source:events" +DEV_STREAM_FLUSH_LOCK_KEY = "dev:source_stream:flush_lock" + + +def flush_valkey() -> None: + try: + conn = get_connection(decode_responses=True) + lock = conn.lock(DEV_STREAM_FLUSH_LOCK_KEY, timeout=10, blocking_timeout=0) # type: ignore[attr-defined] + except Exception: + logger.debug("[dev] Failed to acquire dev flush lock") + return + + acquired = False + try: + acquired = lock.acquire(blocking=False) + if not acquired: + logger.debug( + "[dev] Source events stream flush already handled by other worker", + ) + return + + conn.flushall() + logger.info( + "[dev] Flushed Valkey completely!", + ) + except Exception: + logger.exception("[dev] Failed to flush Valkey") + finally: + if acquired: + try: + lock.release() + except LockNotOwnedError: + logger.debug( + "[dev] Flush lock key already cleared; skipping release", + ) + except Exception: + logger.exception("[dev] Failed to release dev flush lock") def to_ms(ts: datetime) -> int: @@ -23,7 +66,7 @@ def xadd_source_event( Returns the message ID assigned by Valkey. """ - conn = get_connection() + conn = get_connection(decode_responses=True) entry = { # bytes-safe; valkey will handle encoding "type": event_type, "ts": str(to_ms(timestamp)), @@ -48,12 +91,30 @@ def xadd_source_event( raise +def get_oldest_source_event_timestamp() -> datetime | None: + """Get the timestamp of the oldest event in the source events stream. + + Returns None if the stream is empty. + """ + conn = get_connection(decode_responses=True) + # Get the first entry in the stream (oldest) + entries: Iterable = conn.xrange(SOURCE_EVENTS_STREAM, min="-", max="+", count=1) # type: ignore[attr-defined] + entries_list = list(entries) + if not entries_list: + return None + + # Extract timestamp from message ID (format: "{timestamp_ms}-{sequence}") + msg_id, _ = entries_list[0] + timestamp_ms = int(str(msg_id).split("-")[0]) + return datetime.fromtimestamp(timestamp_ms / 1000.0, tz=timezone.utc) + + def xrange_source_events(*, start: datetime, end: datetime) -> list[dict]: """Read events from the source stream within [start, end] by stream ID range. Returns a list of {"id": str, "event_type": str, "payload_json": str} dicts. """ - conn = get_connection() + conn = get_connection(decode_responses=True) start_id = f"{to_ms(start)}-0" end_id = f"{to_ms(end)}-999999" try: @@ -63,22 +124,15 @@ def xrange_source_events(*, start: datetime, end: datetime) -> list[dict]: raise out: list[dict] = [] for msg_id, fields in entries: - # fields is a dict-like of bytes->bytes (decode_responses=False) - etype = fields.get(b"type") - payload = fields.get(b"payload") + etype = fields.get("type") + payload = fields.get("payload") if etype is None or payload is None: continue out.append( { - "id": msg_id.decode() - if isinstance(msg_id, (bytes, bytearray)) - else str(msg_id), - "event_type": etype.decode() - if isinstance(etype, (bytes, bytearray)) - else str(etype), - "payload_json": payload.decode() - if isinstance(payload, (bytes, bytearray)) - else str(payload), + "id": str(msg_id), + "event_type": str(etype), + "payload_json": str(payload), } ) return out diff --git a/src/clepsy/infra/valkey_client.py b/src/clepsy/infra/valkey_client.py index 6562658..aff8b90 100755 --- a/src/clepsy/infra/valkey_client.py +++ b/src/clepsy/infra/valkey_client.py @@ -7,10 +7,8 @@ from clepsy.config import config -@lru_cache(maxsize=1) -def get_connection() -> "redis.Redis": - """Get a process-wide Valkey/Redis connection. +@lru_cache(maxsize=2) +def get_connection(*, decode_responses: bool) -> "redis.Redis": + """Get a process-wide Valkey/Redis connection for the given response mode.""" - Uses VALKEY_URL from config. Compatible with redis-py API. - """ - return redis.from_url(config.valkey_url, decode_responses=False) # type: ignore[attr-defined] + return redis.from_url(config.valkey_url, decode_responses=decode_responses) # type: ignore[attr-defined] diff --git a/src/clepsy/jobs/aggregation.py b/src/clepsy/jobs/aggregation.py index f76ad2f..6940194 100755 --- a/src/clepsy/jobs/aggregation.py +++ b/src/clepsy/jobs/aggregation.py @@ -1,26 +1,28 @@ # ruff: noqa: I001 -from datetime import datetime, timedelta, timezone -from typing import Optional +from datetime import datetime, timezone +from time import perf_counter -from dateutil.parser import isoparse import dramatiq from loguru import logger import clepsy.entities as E -from clepsy.aggregator_worker import do_aggregation, do_empty_aggregation +from clepsy.aggregator_worker import do_empty_aggregation, do_aggregation from clepsy.config import config from clepsy.entities import TimeSpan from clepsy.infra import dramatiq_setup as _dramatiq_setup # noqa: F401 -from clepsy.infra.streams import xrange_source_events +from clepsy.infra.streams import xrange_source_events, get_oldest_source_event_timestamp from clepsy.jobs.actor_init import actor_init +from clepsy.db import get_db_connection +from clepsy.db.queries import ( + select_latest_aggregation, + update_scheduled_job_next_run_at, +) -def current_window(now: Optional[datetime] = None) -> tuple[datetime, datetime]: - now = now or datetime.now(tz=timezone.utc) - interval = config.aggregation_interval - start = now - timedelta(seconds=now.timestamp() % interval.total_seconds()) - end = start + interval - return start, end +# Apply a patch to automatically prefix all logs in this module +logger = logger.patch( + lambda record: record.update(message=f"[aggregate_window] {record['message']}") +) def map_source_event(row) -> E.AggregationInputEvent: @@ -48,63 +50,155 @@ def map_source_event(row) -> E.AggregationInputEvent: raise ValueError(f"Unexpected etype {etype}") -@dramatiq.actor -async def aggregate_window( - start_iso: str | None = None, end_iso: str | None = None -) -> None: - """Dramatiq async actor: aggregate a specific time window. +async def aggregate_window(schedule_id: int) -> None: + current_time = datetime.now(tz=timezone.utc) + + oldest_source_event_timestamp = get_oldest_source_event_timestamp() + + async with get_db_connection() as conn: + previous_aggregation = await select_latest_aggregation(conn) + + if not previous_aggregation: + # Case: First run + if oldest_source_event_timestamp: + # Case 2 & 3: Events exist. + # If too recent (Case 2), we'll wait. + # If old enough (Case 3), we'll process [oldest, oldest + interval). + start = oldest_source_event_timestamp + else: + # Case 1: No events yet. + # Start from (now - interval - grace). + # This will result in a "current job" processing (empty aggregation) + # and scheduling next run at (now + interval). + start = ( + current_time + - config.aggregation_interval + - config.aggregation_grace_period + ) + else: + # Case: Subsequent runs + start = previous_aggregation.end_time + + # Case 6 & 7: Gap handling (e.g. server downtime) + # If oldest source event is newer than previous aggregation end, + # skip empty periods and start from where events actually begin. + if oldest_source_event_timestamp and oldest_source_event_timestamp > start: + logger.info( + "Skipping empty period: previous_aggregation.end_time={}, oldest_source_event={}", + start, + oldest_source_event_timestamp, + ) + start = oldest_source_event_timestamp + # Note: If oldest_source_event_timestamp < start, this means there are older events + # that should have been processed. This shouldn't happen in normal operation, but + # if it does (e.g., backfill), we'll process from previous_aggregation.end_time + # and those older events will remain unprocessed until manually handled. + + # Case 4 & 5: Normal sequential processing + # We continue from previous_aggregation.end_time. + + # Windows are always fixed length: [start, start + interval) + end = start + config.aggregation_interval + + # Check if we can safely process this window + # We need current_time >= start + interval + grace_period to ensure we're lagging by at least grace_period + if ( + current_time - start + < config.aggregation_interval + config.aggregation_grace_period + ): + # Case 2, 4, 6: Too close to real-time + # No aggregation in current job. + # Next run scheduled at: start + interval + grace_period + logger.info( + "Aggregation too close to real-time. start={}, end={}, current_time={}, required_lag={}", + start, + end, + current_time, + config.aggregation_interval + config.aggregation_grace_period, + ) + # Schedule next run at: start + interval + grace_period + # This ensures we're always lagging by interval + grace_period + next_aggregation_time = ( + start + config.aggregation_interval + config.aggregation_grace_period + ) + logger.info( + "Next aggregation scheduled at {}", + next_aggregation_time, + ) - - If start/end are not provided, compute the current window. - - Reads events from Valkey Streams and runs aggregation. - """ - # Ensure DB adapters/converters are registered in this worker process - await actor_init() + async with get_db_connection() as update_conn: + await update_scheduled_job_next_run_at( + update_conn, + schedule_id=schedule_id, + next_run_at=next_aggregation_time, + ) + return - start: datetime - end: datetime - if start_iso and end_iso: - start = isoparse(start_iso) - end = isoparse(end_iso) - else: - start, end = current_window() + # Case 1, 3, 5, 7: Ready to process + # Aggregation (or empty aggregation) will run in current job. + # Next run scheduled at: end + interval + grace_period + effective_end = end + config.aggregation_grace_period + schedule_update = E.ScheduleUpdate( + schedule_id=schedule_id, + next_run_at=end + config.aggregation_interval + config.aggregation_grace_period, + ) logger.info( - "[Dramatiq] aggregate_window start={} end={} (grace={})", + "aggregate_window start={} end={} (grace={})", start, end, config.aggregation_grace_period, ) + fetch_started = perf_counter() # Query durable source events with grace on the upper bound to capture late arrivals. # We'll filter strictly by event timestamp within [start, end) afterwards. - effective_end = end + config.aggregation_grace_period rows = xrange_source_events(start=start, end=effective_end) + logger.debug( + "fetched {row_count} rows in {duration:.2f}s", + row_count=len(rows), + duration=perf_counter() - fetch_started, + ) window_span = TimeSpan(start_time=start, end_time=end) - if not rows: - logger.info("[Dramatiq] No source events in window; running empty aggregation") - await do_empty_aggregation() - return input_logs: list[E.AggregationInputEvent] = [] + transform_started = perf_counter() for row in rows: mapped = map_source_event(row) - if mapped is not None: - # Filter strictly by event time within [start, end) - evt_ts = mapped.timestamp - if evt_ts.tzinfo is None: - evt_ts = evt_ts.replace(tzinfo=timezone.utc) - if start <= evt_ts < end: - input_logs.append(mapped) + evt_ts = mapped.timestamp + if evt_ts.tzinfo is None: + evt_ts = evt_ts.replace(tzinfo=timezone.utc) + if start <= evt_ts < end: + input_logs.append(mapped) if not input_logs: - logger.info( - "[Dramatiq] No mappable source events in window; running empty aggregation" - ) - await do_empty_aggregation() + logger.info("No mappable source events in window; running empty aggregation") + await do_empty_aggregation(schedule_update=schedule_update) return # Ensure deterministic ordering by event timestamp input_logs.sort(key=lambda e: e.timestamp) + logger.debug( + "prepared {log_count} logs in {duration:.2f}s", + log_count=len(input_logs), + duration=perf_counter() - transform_started, + ) - await do_aggregation(input_logs=input_logs, aggregation_time_span=window_span) + aggregation_started = perf_counter() + await do_aggregation( + input_logs=input_logs, + aggregation_time_span=window_span, + schedule_update=schedule_update, + ) + logger.info( + "aggregation completed in {duration:.2f}s", + duration=perf_counter() - aggregation_started, + schedule_update=schedule_update, + ) + + +@dramatiq.actor +async def aggregate_window_job(schedule_id: int) -> None: + await actor_init() + await aggregate_window(schedule_id=schedule_id) diff --git a/src/clepsy/jobs/goals.py b/src/clepsy/jobs/goals.py index 32a73de..a06ae38 100755 --- a/src/clepsy/jobs/goals.py +++ b/src/clepsy/jobs/goals.py @@ -9,30 +9,40 @@ from loguru import logger from clepsy.modules.goals.calculate_goals import ( - update_current_progress_job as _update_current_progress_job, - update_previous_full_period_goal_result_job as _update_previous_full_period_goal_result_job, + update_current_progress_job, + update_previous_full_period_goal_result_job, ) +async def run_update_current_progress(goal_id: int, ttl_seconds: float) -> None: + """Update current progress for a goal if stale.""" + + logger.info( + "[Goals] run_update_current_progress goal_id={} ttl_seconds={}", + goal_id, + ttl_seconds, + ) + await update_current_progress_job( + goal_id=goal_id, + ttl=timedelta(seconds=ttl_seconds), + ) + + +async def run_update_previous_full_period_result(goal_id: int) -> None: + """Compute and upsert the previous full period result for a goal.""" + + logger.info( + "[Goals] run_update_previous_full_period_result goal_id={}", + goal_id, + ) + await update_previous_full_period_goal_result_job(goal_id=goal_id) + + @dramatiq.actor async def run_update_current_progress_job(goal_id: int, ttl_seconds: float) -> None: - """Dramatiq async actor: update current progress for a goal if stale. - - Parameters: - - goal_id: The goal identifier - - ttl_seconds: freshness window in seconds; if the latest progress row is newer than this, skip - """ try: - # Ensure DB adapters/converters are registered in this worker process await actor_init() - logger.info( - "[Dramatiq] run_update_current_progress_job goal_id={} ttl_seconds={}", - goal_id, - ttl_seconds, - ) - await _update_current_progress_job( - goal_id=goal_id, ttl=timedelta(seconds=ttl_seconds) - ) + await run_update_current_progress(goal_id=goal_id, ttl_seconds=ttl_seconds) except Exception: logger.exception("[Dramatiq] run_update_current_progress_job failed") raise @@ -40,14 +50,9 @@ async def run_update_current_progress_job(goal_id: int, ttl_seconds: float) -> N @dramatiq.actor async def run_update_previous_full_period_result_job(goal_id: int) -> None: - """Dramatiq async actor: compute and upsert the previous full period result for a goal.""" try: - # Ensure DB adapters/converters are registered in this worker process await actor_init() - logger.info( - "[Dramatiq] run_update_previous_full_period_result_job goal_id={}", goal_id - ) - await _update_previous_full_period_goal_result_job(goal_id=goal_id) + await run_update_previous_full_period_result(goal_id=goal_id) except Exception: logger.exception("[Dramatiq] run_update_previous_full_period_result_job failed") raise diff --git a/src/clepsy/jobs/scheduled_job_dispatch.py b/src/clepsy/jobs/scheduled_job_dispatch.py new file mode 100755 index 0000000..6a4992c --- /dev/null +++ b/src/clepsy/jobs/scheduled_job_dispatch.py @@ -0,0 +1,104 @@ +from time import perf_counter + +import dramatiq +from loguru import logger + +from clepsy.db import get_db_connection, queries as db_queries +from clepsy.entities import DBScheduledJob, JobType, ScheduleStatus +from clepsy.jobs import scheduler_tick +from clepsy.jobs.actor_init import actor_init +from clepsy.jobs.aggregation import aggregate_window +from clepsy.jobs.goals import ( + run_update_current_progress, + run_update_previous_full_period_result, +) +from clepsy.jobs.sessions import run_sessionization + + +async def dispatch_job( + job_type: JobType, schedule_id: int, payload: dict | None = None +) -> None: + """Execute a scheduled job within the current Dramatiq worker. + + Uses structural pattern matching to ensure exhaustive handling of registered + job types. Raises a ValueError if the requested job type is unknown. + + Args: + job_type: The type of job to execute + schedule_id: The ID of the scheduled job (required for all jobs) + payload: Optional job-specific payload data + """ + + data = dict(payload or {}) + logger.debug( + "Dispatching {job_type} with schedule_id={schedule_id} payload={payload}", + job_type=job_type, + schedule_id=schedule_id, + payload=payload, + ) + started_at = perf_counter() + match job_type: + case JobType.GOAL_UPDATE_CURRENT_PROGRESS: + await run_update_current_progress(**data) + case JobType.GOAL_UPDATE_PREVIOUS_PERIOD: + await run_update_previous_full_period_result(**data) + case JobType.AGGREGATION_WINDOW: + await aggregate_window(schedule_id=schedule_id) + case JobType.SESSIONIZATION: + await run_sessionization(schedule_id=schedule_id) + case _ as unknown: + raise ValueError(f"Unhandled job type {unknown!r}") + + logger.debug( + "Dispatched {job_type} finished in {duration:.2f}s", + job_type=job_type, + duration=perf_counter() - started_at, + ) + + +@dramatiq.actor +async def run_scheduled_job(job_dict: dict) -> None: + job = DBScheduledJob.model_validate(job_dict) + await actor_init() + + status_on_completion: ScheduleStatus | None = None + started_at = perf_counter() + logger.info( + "[RunScheduledJob] job {job_id} ({job_type}) starting", + job_id=job.id, + job_type=job.job_type.value, + ) + + try: + # Pass schedule_id separately from payload - it's execution metadata, not job data + await dispatch_job(job.job_type, schedule_id=job.id, payload=job.payload) + except Exception: + status_on_completion = ScheduleStatus.ERROR + logger.exception( + "[RunScheduledJob] job {job_id} ({job_type}) failed after {duration:.2f}s", + job_id=job.id, + job_type=job.job_type.value, + duration=perf_counter() - started_at, + ) + raise + finally: + try: + async with get_db_connection() as conn: + await db_queries.decrement_scheduled_job_running_count( + conn, + schedule_id=job.id, + new_status=status_on_completion, + ) + except Exception: + logger.exception( + "[RunScheduledJob] failed to finalize schedule {}", + job.schedule_key, + ) + scheduler_tick.scheduler_tick.send() + + logger.info( + "[RunScheduledJob] job {job_id} ({job_type}) completed in {duration:.2f}s", + job_id=job.id, + job_type=job.job_type.value, + duration=perf_counter() - started_at, + ) diff --git a/src/clepsy/jobs/scheduler_tick.py b/src/clepsy/jobs/scheduler_tick.py new file mode 100755 index 0000000..6c59ce7 --- /dev/null +++ b/src/clepsy/jobs/scheduler_tick.py @@ -0,0 +1,209 @@ +from datetime import datetime, timedelta, timezone + +from croniter import croniter +from croniter.croniter import CroniterBadCronError +from dateutil.parser import isoparse +import dramatiq +from loguru import logger + +from clepsy.config import config +from clepsy.db import get_db_connection, queries as db_queries +from clepsy.entities import DBScheduledJob +from clepsy.jobs.actor_init import actor_init +from clepsy.jobs.scheduled_job_dispatch import run_scheduled_job +from clepsy.utils import datetime_to_eta, ensure_utc + + +logger = logger.patch( + lambda record: record.update(message=f"[SchedulerTick] {record['message']}") +) + + +def coerce_to_utc(now_iso: str | None) -> datetime: + if now_iso is None: + return datetime.now(tz=timezone.utc) + + parsed = isoparse(now_iso) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +MAX_CATCH_UP_STEPS = 256 +DEFAULT_POLL_INTERVAL = timedelta(seconds=30) + + +def compute_next_run(job: DBScheduledJob, *, now: datetime) -> datetime | None: + """Compute the next run time from cron expression. + + Returns None if the job has no cron expression (self-managed scheduling). + """ + if job.cron_expr is None: + return None + + base = ensure_utc(job.next_run_at) + try: + iterator = croniter(job.cron_expr, base) + next_run = ensure_utc(iterator.get_next(datetime)) + except CroniterBadCronError as exc: + raise ValueError( + f"Invalid cron expression '{job.cron_expr}' for schedule {job.schedule_key}" + ) from exc + + steps = 0 + while next_run <= now and steps < MAX_CATCH_UP_STEPS: + next_run = ensure_utc(iterator.get_next(datetime)) + steps += 1 + + if next_run <= now: + # If we're behind, schedule for the next poll interval + next_run = now + DEFAULT_POLL_INTERVAL + + return next_run + + +async def schedule_follow_up_tick(*, now: datetime) -> None: + """Schedule the next tick using a static polling interval. + + We use a simple polling approach rather than trying to be smart about + immediate retries, which can cause contention and rapid-fire retries. + Jobs manage their own next_run_at, so we just need to check periodically. + """ + # Always use the default poll interval for simplicity and reliability + next_tick_at = ensure_utc(now) + DEFAULT_POLL_INTERVAL + + scheduler_tick.send_with_options(eta=datetime_to_eta(next_tick_at)) + logger.debug("scheduled next tick at {}", next_tick_at.isoformat()) + + +@dramatiq.actor +async def scheduler_tick() -> None: + """Central scheduler tick. + + Reads the database for due schedules, safely advances their run cursor, and + dispatches them for execution. Afterwards schedules the next tick using a + static polling interval (30 seconds). Jobs manage their own next_run_at, + so we just need to check periodically. + """ + + await actor_init() + now = datetime.now(tz=timezone.utc) + logger.debug("evaluating schedules at {}", now.isoformat()) + + # Read entire table once (no lock needed for read) + try: + async with get_db_connection(start_transaction=False) as conn: + all_jobs: list[DBScheduledJob] = await db_queries.select_all_scheduled_jobs( + conn + ) + except Exception: + logger.exception("failed to fetch scheduled jobs") + await schedule_follow_up_tick(now=now) + return + + # Do all logic in Python + timeout_threshold = now - config.scheduled_job_timeout + timed_out_jobs: list[DBScheduledJob] = [] + due_jobs: list[DBScheduledJob] = [] + + for job in all_jobs: + # Check for timed out jobs + if ( + job.running_count > 0 + and job.last_started_at is not None + and ensure_utc(job.last_started_at) <= timeout_threshold + and job.status != "disabled" + ): + timed_out_jobs.append(job) + + # Check for due jobs + if ( + job.enabled + and ensure_utc(job.next_run_at) <= now + and job.running_count < job.max_concurrent + and job.status != "disabled" + ): + due_jobs.append(job) + + # Release timed out jobs (write transaction only if needed) + if timed_out_jobs: + try: + async with get_db_connection(start_transaction=True) as conn: + for job in timed_out_jobs: + last_started = ( + ensure_utc(job.last_started_at).isoformat() + if job.last_started_at + else "unknown" + ) + logger.warning( + "schedule {} timed out after {} (last_started={})", + job.schedule_key, + config.scheduled_job_timeout, + last_started, + ) + # Update all timed-out jobs in one query + await db_queries.release_timed_out_scheduled_jobs( + conn, + timeout_threshold=timeout_threshold, + now=now, + ) + except Exception: + logger.exception("failed to release timed-out schedules") + + # Process due jobs + if not due_jobs: + logger.debug("no due schedules found") + else: + for job in due_jobs: + logger.info( + "schedule {} ({}) due at {} (running={}/{})", + job.schedule_key, + job.job_type.value, + job.next_run_at.isoformat(), + job.running_count, + job.max_concurrent, + ) + + new_next_run: datetime | None = None + if job.cron_expr is not None: + try: + new_next_run = compute_next_run(job, now=now) + except ValueError: + logger.exception( + "failed to compute next run for {}", + job.schedule_key, + ) + continue + else: + # Job manages its own next_run_at, don't update it + logger.debug( + "schedule {} has no cron expression, will not update next_run_at", + job.schedule_key, + ) + + try: + async with get_db_connection(start_transaction=True) as conn: + started = await db_queries.mark_scheduled_job_started( + conn, + schedule_id=job.id, + expected_next_run_at=job.next_run_at, + started_at=now, + new_next_run_at=new_next_run, + ) + except Exception: + logger.exception( + "failed to mark schedule {} started", + job.schedule_key, + ) + continue + + if not started: + logger.debug( + "schedule {} was claimed by another worker", + job.schedule_key, + ) + continue + + run_scheduled_job.send(job.model_dump(mode="json")) + + await schedule_follow_up_tick(now=now) diff --git a/src/clepsy/jobs/sessions.py b/src/clepsy/jobs/sessions.py index 19a2931..4bbbdca 100755 --- a/src/clepsy/jobs/sessions.py +++ b/src/clepsy/jobs/sessions.py @@ -11,11 +11,9 @@ @dramatiq.actor async def run_sessionization_job() -> None: - """Dramatiq async actor for sessionization.""" logger.info("[Dramatiq] run_sessionization_job starting") try: - # Ensure DB adapters/converters are registered in this worker process await actor_init() await run_sessionization() except Exception: diff --git a/src/clepsy/main.py b/src/clepsy/main.py index 1b9f1de..210e2e5 100755 --- a/src/clepsy/main.py +++ b/src/clepsy/main.py @@ -11,7 +11,7 @@ import uvicorn from clepsy import bootstrap -from clepsy.scheduling import scheduler, init_schedules +from clepsy.scheduling import initialize_scheduler from clepsy.auth.auth_middleware import DeviceTokenMiddleware, JWTMiddleware from clepsy.modules.account_creation.router import router as account_creation_router from clepsy.modules.activities.router import router as activity_router @@ -25,6 +25,7 @@ from clepsy.modules.user_settings.router import router as user_settings_router from prometheus_fastapi_instrumentator import Instrumentator from clepsy.config import config +from clepsy.infra.streams import flush_valkey @asynccontextmanager @@ -32,15 +33,14 @@ async def lifespan(app_: FastAPI): # ---- startup ------------------------------------------------- logger.info("Starting Clepsy backend...") logger.info("Initializing bootstrap...") - await bootstrap.init() + if config.is_dev and not config.preserve_dev_streams: + logger.info("Flushing valkey...") + flush_valkey() - # Start persistent APScheduler (v4) with SQLAlchemy datastore + await bootstrap.init() - async with scheduler: - await init_schedules(scheduler) - await scheduler.start_in_background() - app_.state.scheduler = scheduler - yield + await initialize_scheduler() + yield app = FastAPI(title="Clepsy backend", lifespan=lifespan) diff --git a/src/clepsy/modules/goals/calculate_goals.py b/src/clepsy/modules/goals/calculate_goals.py index f1af621..9a1eb81 100755 --- a/src/clepsy/modules/goals/calculate_goals.py +++ b/src/clepsy/modules/goals/calculate_goals.py @@ -1,4 +1,5 @@ from datetime import date, datetime, time, timedelta, timezone +import sqlite3 from zoneinfo import ZoneInfo import aiosqlite @@ -526,13 +527,13 @@ async def update_current_progress_for_goal( include_tags: list[DBTag] | None, exclude_tags: list[DBTag] | None, now_utc: datetime, -) -> None: +) -> bool: # If goal is currently paused, skip computing current progress. if await is_goal_paused_at(conn, goal_id=goal.id, at_utc=now_utc): logger.info( f"Goal {goal.id} is paused at {now_utc.isoformat()}, skipping current progress update." ) - return + return False start_utc, end_utc = get_current_period_bounds(goal.period, goal.timezone, now_utc) res = await calculate_goal_progress_current( @@ -544,19 +545,34 @@ async def update_current_progress_for_goal( end_utc=end_utc, conn=conn, ) - await upsert_goal_progress_current( - conn, - goal_definition_id=definition.id, - period_start_utc=res.period_start, - period_end_utc=res.period_end, - metric_value=float(res.metric_value) + upsert_kwargs = { + "goal_definition_id": definition.id, + "period_start_utc": res.period_start, + "period_end_utc": res.period_end, + "metric_value": float(res.metric_value) if isinstance(res.metric_value, (int, float)) else 0.0, - success=res.success, - eval_state=res.eval_state, - eval_state_reason=res.eval_state_reason, - updated_at_utc=now_utc, - ) + "success": res.success, + "eval_state": res.eval_state, + "eval_state_reason": res.eval_state_reason, + "updated_at_utc": now_utc, + } + + try: + async with get_db_connection( + commit_on_exit=True, start_transaction=True, transaction_type="IMMEDIATE" + ) as write_conn: + await upsert_goal_progress_current(write_conn, **upsert_kwargs) + except (sqlite3.OperationalError, aiosqlite.OperationalError) as exc: + if "database is locked" in str(exc).lower(): + logger.warning( + "[goals] Skipping current progress upsert for goal_id={} due to locked database", + goal.id, + ) + return False + raise + + return True async def update_current_progress_job(goal_id: int, ttl: timedelta) -> None: @@ -568,12 +584,10 @@ async def update_current_progress_job(goal_id: int, ttl: timedelta) -> None: - Otherwise, compute current progress and upsert. """ logger.info( - "[goals] Starting DEFERRED transaction for updating current progress (goal_id={})", + "[goals] Preparing current progress refresh for goal_id={}", goal_id, ) - async with get_db_connection( - commit_on_exit=True, start_transaction=True, transaction_type="DEFERRED" - ) as conn: + async with get_db_connection(start_transaction=False) as conn: # Early-exit freshness check for the latest definition's current-progress row latest_updated_at = await select_latest_progress_updated_at_for_goal( conn, goal_id=goal_id @@ -598,7 +612,7 @@ async def update_current_progress_job(goal_id: int, ttl: timedelta) -> None: ) if gwr is None: return - await update_current_progress_for_goal( + updated = await update_current_progress_for_goal( conn=conn, goal=gwr.goal, definition=gwr.definition, @@ -606,8 +620,13 @@ async def update_current_progress_job(goal_id: int, ttl: timedelta) -> None: exclude_tags=gwr.exclude_tags, now_utc=now_utc, ) + if updated: + logger.info( + "[goals] Current progress refreshed for goal_id={}", + goal_id, + ) - logger.info("[goals] DEFERRED transaction committed for goal_id={}", goal_id) + logger.debug("[goals] Current progress job finished for goal_id={}", goal_id) async def update_previous_full_period_goal_result_job(goal_id: int) -> None: diff --git a/src/clepsy/modules/goals/pages/home.py b/src/clepsy/modules/goals/pages/home.py index 639da34..d3e54d8 100755 --- a/src/clepsy/modules/goals/pages/home.py +++ b/src/clepsy/modules/goals/pages/home.py @@ -162,7 +162,7 @@ async def render_goal_row( root_attrs.update( { "hx-get": f"/s/goals/row/{goal.id}", - "hx-trigger": "load, every 3s", + "hx-trigger": "load delay:5s, every 60s", "hx-target": f"#goal-row-{goal.id}", "hx-swap": "outerHTML", } diff --git a/src/clepsy/modules/goals/router.py b/src/clepsy/modules/goals/router.py index 955042e..3691427 100755 --- a/src/clepsy/modules/goals/router.py +++ b/src/clepsy/modules/goals/router.py @@ -70,14 +70,14 @@ async def delete_goal_endpoint( ) -> Response: try: async with get_db_connection( - commit_on_exit=True, start_transaction=False + commit_on_exit=True, start_transaction=True ) as conn: await delete_goal(conn, goal_id) + await unschedule_goal_previous_period_update(goal_id=goal_id, conn=conn) # Return 200 with empty body so hx-swap="outerHTML" clears the row element resp = HTMLResponse("", status_code=200) # Remove periodic schedule for this goal (best effort) - await unschedule_goal_previous_period_update(goal_id=goal_id) resp.headers["HX-Trigger"] = json.dumps( { @@ -166,9 +166,7 @@ async def get_goal_row(_: Request, goal_id: int) -> Response: @router.get("/row/{goal_id}/refresh") async def refresh_goal_row(_: Request, goal_id: int) -> Response: - async with get_db_connection( - commit_on_exit=True, start_transaction=True, transaction_type="DEFERRED" - ) as conn: + async with get_db_connection() as conn: gwrs = await select_goals_with_latest_definition(conn, last_successes_limit=8) gwr = next((g for g in gwrs if g.goal.id == goal_id), None) if not gwr: @@ -542,7 +540,11 @@ async def persist_created_goal( include_tag_ids: list[int], exclude_tag_ids: list[int], ) -> Response: - async with get_db_connection(commit_on_exit=True, start_transaction=True) as conn: + eff_from = datetime.now(dt_timezone.utc) + + async with get_db_connection( + commit_on_exit=True, start_transaction=True, transaction_type="IMMEDIATE" + ) as conn: goal_id = await insert_goal( conn, metric=metric, @@ -551,7 +553,6 @@ async def persist_created_goal( timezone=timezone_str, ) - eff_from = datetime.now(dt_timezone.utc) def_id = await insert_goal_definition( conn, goal_id=goal_id, @@ -570,13 +571,14 @@ async def persist_created_goal( include_tag_ids=include_tag_ids, exclude_tag_ids=exclude_tag_ids, ) - # Schedule periodic evaluation for previous full period result - await schedule_goal_previous_period_update( - goal_id=goal_id, - period=period, - timezone_str=timezone_str or None, - created_at=eff_from, - ) + # Schedule periodic evaluation for previous full period result within the same transaction + await schedule_goal_previous_period_update( + goal_id=goal_id, + period=period, + timezone_str=timezone_str or None, + created_at=eff_from, + conn=conn, + ) response = RedirectResponse(url="/s/goals", status_code=200) response.headers["HX-Trigger"] = json.dumps( diff --git a/src/clepsy/modules/sessions/tasks.py b/src/clepsy/modules/sessions/tasks.py index 91d6fcf..bd1bd69 100755 --- a/src/clepsy/modules/sessions/tasks.py +++ b/src/clepsy/modules/sessions/tasks.py @@ -1,11 +1,14 @@ import asyncio from collections import defaultdict from dataclasses import dataclass -from datetime import datetime, timedelta +from datetime import datetime, timedelta, timezone +import sqlite3 from typing import List, Optional +import aiosqlite from loguru import logger +# Apply patch to automatically prefix all logs in this module from baml_client import b from baml_client.type_builder import TypeBuilder import baml_client.types as baml_types @@ -26,6 +29,7 @@ select_latest_sessionization_run, select_specs_with_tags_in_time_range, select_user_settings, + update_scheduled_job_next_run_at, ) from clepsy.entities import ( CandidateSession, @@ -36,6 +40,7 @@ DBCandidateSession, DBCandidateSessionSpec, LLMConfig, + ScheduleUpdate, Session, SessionizationRun, SessionSpec, @@ -46,6 +51,11 @@ from clepsy.llm import create_client_registry +logger = logger.patch( + lambda record: record.update(message=f"[sessionization] {record['message']}") +) + + async def detect_sessions( specs_in_time_range: list[DBActivitySpecWithTags], preexisting_sessions: list[CandidateSession], @@ -828,44 +838,111 @@ async def finalize_carry_over_sessions_and_save( ) # Save sessionization run - logger.info( - "[sessionization] Starting DEFERRED transaction for saving sessionization results" + sessionization_run = SessionizationRun( + candidate_creation_start=candidate_creation_interval_start, + candidate_creation_end=candidate_creation_interval_end, + finalized_horizon=finalized_horizon, + overlap_start=overlap_start, + right_tail_end=right_tail_end, ) - async with get_db_connection( - start_transaction=True, transaction_type="DEFERRED" - ) as conn: - sessionization_run = SessionizationRun( - candidate_creation_start=candidate_creation_interval_start, - candidate_creation_end=candidate_creation_interval_end, - finalized_horizon=finalized_horizon, - overlap_start=overlap_start, - right_tail_end=right_tail_end, - ) - sessionization_id = await insert_sessionization_run( - conn, sessionization_run=sessionization_run - ) - if candidate_session_ids_to_delete: - await delete_candidate_sessions_by_ids( - conn, candidate_session_ids=candidate_session_ids_to_delete - ) + await persist_sessionization_run_results( + sessionization_run=sessionization_run, + sessions_to_create=sessions_to_create, + candidate_session_ids_to_delete=candidate_session_ids_to_delete, + ) - if sessions_to_create: - session_ids = await insert_sessions( - conn, - sessions=[spec.session for spec in sessions_to_create], - sessionization_run_id=sessionization_id, + +async def persist_sessionization_run_results( + *, + sessionization_run: SessionizationRun, + sessions_to_create: list[SessionSpec], + candidate_session_ids_to_delete: list[int], + candidate_session_specs_to_create: list[CandidateSessionSpec] | None = None, + new_mappings_existing_candidate_sessions: list[CandidateSessionToActivity] + | None = None, + activity_ids_to_delete_from_candidate_sessions: list[int] | None = None, + schedule_update: ScheduleUpdate | None = None, +) -> None: + logger.info("Starting IMMEDIATE transaction for saving sessionization results") + try: + async with get_db_connection( + start_transaction=True, commit_on_exit=True, transaction_type="IMMEDIATE" + ) as conn: + sessionization_id = await insert_sessionization_run( + conn, sessionization_run=sessionization_run ) - session_to_activities = [] - for spec, sid in zip(sessions_to_create, session_ids): - for aid in spec.activity_ids: - session_to_activities.append( - SessionToActivity(session_id=sid, activity_id=aid) + + if candidate_session_ids_to_delete: + await delete_candidate_sessions_by_ids( + conn, candidate_session_ids=candidate_session_ids_to_delete + ) + + if activity_ids_to_delete_from_candidate_sessions: + await delete_candidate_session_to_activity_by_activity_ids( + conn, + activity_ids=activity_ids_to_delete_from_candidate_sessions, + ) + + candidate_mappings: list[CandidateSessionToActivity] = [] + candidate_specs = candidate_session_specs_to_create or [] + if candidate_specs: + candidate_sessions = [spec.session for spec in candidate_specs] + candidate_session_ids = await insert_candidate_sessions( + conn, + sessions=candidate_sessions, + sessionization_run_id=sessionization_id, + ) + + for spec, cid in zip(candidate_specs, candidate_session_ids): + for aid in spec.activity_ids: + candidate_mappings.append( + CandidateSessionToActivity( + candidate_session_id=cid, activity_id=aid + ) + ) + + extra_mappings = new_mappings_existing_candidate_sessions or [] + combined_candidate_mappings = candidate_mappings + extra_mappings + if combined_candidate_mappings: + await insert_candidate_session_to_activity( + conn, mappings=combined_candidate_mappings + ) + + if sessions_to_create: + session_ids = await insert_sessions( + conn, + sessions=[spec.session for spec in sessions_to_create], + sessionization_run_id=sessionization_id, + ) + + session_to_activities: list[SessionToActivity] = [] + for spec, sid in zip(sessions_to_create, session_ids): + for aid in spec.activity_ids: + session_to_activities.append( + SessionToActivity(session_id=sid, activity_id=aid) + ) + + if session_to_activities: + await insert_session_to_activity( + conn, mappings=session_to_activities ) - if session_to_activities: - await insert_session_to_activity(conn, mappings=session_to_activities) - await delete_candidate_sessions_without_activities(conn) + await delete_candidate_sessions_without_activities(conn) + + # Update next_run_at in the same transaction for idempotency + if schedule_update: + await update_scheduled_job_next_run_at( + conn, + schedule_id=schedule_update.schedule_id, + next_run_at=schedule_update.next_run_at, + ) + except (sqlite3.OperationalError, aiosqlite.OperationalError) as exc: + if "database is locked" in str(exc).lower(): + logger.warning( + "Database locked while persisting sessionization run; will retry" + ) + raise async def deal_with_island( @@ -1049,12 +1126,25 @@ async def deal_with_island( ) -async def run_sessionization(): +async def run_sessionization(schedule_id: int): logger.info("Starting sessionization run") async with get_db_connection() as conn: user_settings = await select_user_settings(conn) if not user_settings: logger.warning("No user settings found. Exiting sessionization run.") + # Schedule next run to retry later + schedule_update = ScheduleUpdate( + schedule_id=schedule_id, + next_run_at=datetime.now(timezone.utc) + + config.session_window_length + + config.aggregation_grace_period, + ) + async with get_db_connection() as update_conn: + await update_scheduled_job_next_run_at( + update_conn, + schedule_id=schedule_update.schedule_id, + next_run_at=schedule_update.next_run_at, + ) return llm_config = user_settings.text_model_config previous_run = await select_latest_sessionization_run(conn) @@ -1065,6 +1155,19 @@ async def run_sessionization(): if not latest_aggregation: logger.warning("No aggregations found. Exiting sessionization run.") + # Schedule next run to retry later + schedule_update = ScheduleUpdate( + schedule_id=schedule_id, + next_run_at=datetime.now(timezone.utc) + + config.session_window_length + + config.aggregation_grace_period, + ) + async with get_db_connection() as update_conn: + await update_scheduled_job_next_run_at( + update_conn, + schedule_id=schedule_update.schedule_id, + next_run_at=schedule_update.next_run_at, + ) return latest_aggregation_interval_end_time = latest_aggregation.end_time carry_over_candidate_session_specs = await select_candidate_session_specs(conn) @@ -1088,18 +1191,7 @@ async def run_sessionization(): start_plus_window = ( candidate_creation_interval_start + config.session_window_length ) - logger.info( - "Sessionization interval debug | prev_end: {} (tzinfo={} type={}) | +window: {} (tzinfo={} type={}) | latest_agg_end: {} (tzinfo={} type={})", - candidate_creation_interval_start, - getattr(candidate_creation_interval_start, "tzinfo", None), - type(candidate_creation_interval_start), - start_plus_window, - getattr(start_plus_window, "tzinfo", None), - type(start_plus_window), - latest_aggregation_interval_end_time, - getattr(latest_aggregation_interval_end_time, "tzinfo", None), - type(latest_aggregation_interval_end_time), - ) + candidate_creation_interval_end = min( start_plus_window, latest_aggregation_interval_end_time, @@ -1116,6 +1208,22 @@ async def run_sessionization(): f"({candidate_creation_interval_start}) >= latest aggregation end " f"({latest_aggregation_interval_end_time}). Skipping sessionization." ) + # Schedule next run after next aggregation window completes + # Wait for aggregation to catch up: latest_agg_end + session_window + grace_period + schedule_update = ScheduleUpdate( + schedule_id=schedule_id, + next_run_at=( + latest_aggregation_interval_end_time + + config.session_window_length + + config.aggregation_grace_period + ), + ) + async with get_db_connection() as update_conn: + await update_scheduled_job_next_run_at( + update_conn, + schedule_id=schedule_update.schedule_id, + next_run_at=schedule_update.next_run_at, + ) return if candidate_creation_interval_start > candidate_creation_interval_end: @@ -1143,6 +1251,7 @@ async def run_sessionization(): previous_run.finalized_horizon, previous_run.candidate_creation_end, ) + async with get_db_connection() as conn: specs_not_finalized = await select_specs_with_tags_in_time_range( conn, @@ -1162,11 +1271,13 @@ async def run_sessionization(): ) # Fetch new activities in current window - new_activity_specs = await select_specs_with_tags_in_time_range( - conn, - start=candidate_creation_interval_start, - end=candidate_creation_interval_end, - ) + + async with get_db_connection() as conn: + new_activity_specs = await select_specs_with_tags_in_time_range( + conn, + start=candidate_creation_interval_start, + end=candidate_creation_interval_end, + ) logger.info( "Current window: {} -> {} | new activity specs: {} | unique activities: {}", @@ -1386,111 +1497,54 @@ async def run_sessionization(): case _: raise ValueError("Unexpected result from deal_with_island") - logger.info( - "[sessionization-isolated] Starting DEFERRED transaction for saving sessionization results" + sessionization_run = SessionizationRun( + candidate_creation_start=candidate_creation_interval_start, + candidate_creation_end=candidate_creation_interval_end, + finalized_horizon=finalized_horizon, + overlap_start=overlap_start, + right_tail_end=right_tail_end, ) - async with get_db_connection( - start_transaction=True, transaction_type="DEFERRED" - ) as conn: - sessionization_run = SessionizationRun( - candidate_creation_start=candidate_creation_interval_start, - candidate_creation_end=candidate_creation_interval_end, - finalized_horizon=finalized_horizon, - overlap_start=overlap_start, - right_tail_end=right_tail_end, - ) - - sessionization_id = await insert_sessionization_run( - conn, sessionization_run=sessionization_run - ) - - # Step 1: Delete candidate sessions (by ids) - if candidate_session_ids_to_delete: - await delete_candidate_sessions_by_ids( - conn, candidate_session_ids=candidate_session_ids_to_delete - ) - - # Step 2: Delete candidate activity mappings (left-side finalized ids) - if activity_ids_to_delete_from_candidate_sessions: - await delete_candidate_session_to_activity_by_activity_ids( - conn, - activity_ids=activity_ids_to_delete_from_candidate_sessions, - ) - - # Step 3: Insert new candidate sessions and their mappings - candidate_sessions_to_create = [] - candidate_session_mappings_to_create = [] - if candidate_session_specs_to_create: - candidate_sessions_to_create = [ - x.session for x in candidate_session_specs_to_create - ] - - candidate_session_ids = await insert_candidate_sessions( - conn, - sessions=candidate_sessions_to_create, - sessionization_run_id=sessionization_id, - ) + # Calculate next run time: always lag by session_window_length + grace_period + # Next window will start at candidate_creation_end, so schedule at candidate_creation_end + window_length + grace_period + schedule_update = ScheduleUpdate( + schedule_id=schedule_id, + next_run_at=( + candidate_creation_interval_end + + config.session_window_length + + config.aggregation_grace_period + ), + ) - for spec, cid in zip( - candidate_session_specs_to_create, candidate_session_ids - ): - for aid in spec.activity_ids: - candidate_session_mappings_to_create.append( - CandidateSessionToActivity( - candidate_session_id=cid, activity_id=aid - ) - ) + await persist_sessionization_run_results( + sessionization_run=sessionization_run, + sessions_to_create=sessions_to_create, + candidate_session_ids_to_delete=candidate_session_ids_to_delete, + candidate_session_specs_to_create=candidate_session_specs_to_create, + new_mappings_existing_candidate_sessions=new_mappings_existing_candidate_sessions, + activity_ids_to_delete_from_candidate_sessions=activity_ids_to_delete_from_candidate_sessions, + schedule_update=schedule_update, + ) - await insert_candidate_session_to_activity( - conn, mappings=candidate_session_mappings_to_create - ) + logger.info( + "Post-island summary | finalized sessions ready: {} | new candidate sessions: {} | existing candidate mappings: {} | candidate sessions to delete: {} | candidate activity ids to prune: {}", + len(sessions_to_create), + len(candidate_session_specs_to_create), + len(new_mappings_existing_candidate_sessions), + len(candidate_session_ids_to_delete), + len(activity_ids_to_delete_from_candidate_sessions), + ) + if ( + finalized_horizon is not None + or overlap_start is not None + or right_tail_end is not None + ): logger.info( - "Post-island summary | finalized sessions ready: {} | new candidate sessions: {} | existing candidate mappings: {} | candidate sessions to delete: {} | candidate activity ids to prune: {}", - len(sessions_to_create), - len(candidate_session_specs_to_create), - len(new_mappings_existing_candidate_sessions), - len(candidate_session_ids_to_delete), - len(activity_ids_to_delete_from_candidate_sessions), + "Window markers | finalized_horizon: {} | overlap_start: {} | right_tail_end: {}", + finalized_horizon, + overlap_start, + right_tail_end, ) - if ( - finalized_horizon is not None - or overlap_start is not None - or right_tail_end is not None - ): - logger.info( - "Window markers | finalized_horizon: {} | overlap_start: {} | right_tail_end: {}", - finalized_horizon, - overlap_start, - right_tail_end, - ) - - # Step 4: Insert mappings for existing candidates - if new_mappings_existing_candidate_sessions: - await insert_candidate_session_to_activity( - conn, mappings=new_mappings_existing_candidate_sessions - ) - - # Step 5: Insert finalized sessions + session_to_activity - if sessions_to_create: - session_ids = await insert_sessions( - conn, - sessions=[spec.session for spec in sessions_to_create], - sessionization_run_id=sessionization_id, - ) - - session_to_activities = [] - for spec, sid in zip(sessions_to_create, session_ids): - for aid in spec.activity_ids: - session_to_activities.append( - SessionToActivity(session_id=sid, activity_id=aid) - ) - if session_to_activities: - await insert_session_to_activity(conn, mappings=session_to_activities) - - # Step 6: Cleanup delete_candidate_sessions_without_activities - await delete_candidate_sessions_without_activities(conn) - - logger.info("[sessionization-isolated] DEFERRED transaction committed successfully") + logger.info("Sessionization results persisted") diff --git a/src/clepsy/scheduling.py b/src/clepsy/scheduling.py index 281b6e0..bf37351 100755 --- a/src/clepsy/scheduling.py +++ b/src/clepsy/scheduling.py @@ -1,119 +1,68 @@ from __future__ import annotations -# ruff: noqa: I001 -import logging -from datetime import datetime, timedelta, timezone as dt_timezone +from datetime import datetime, timedelta, timezone from zoneinfo import ZoneInfo, ZoneInfoNotFoundError -from apscheduler import AsyncScheduler, ConflictPolicy, TaskDefaults -from apscheduler.datastores.sqlalchemy import SQLAlchemyDataStore -from apscheduler.executors.async_ import AsyncJobExecutor -from apscheduler.triggers.cron import CronTrigger -from apscheduler.triggers.interval import IntervalTrigger +import aiosqlite +from loguru import logger from clepsy import utils from clepsy.config import config -from clepsy.entities import GoalPeriod -from clepsy.jobs.aggregation import aggregate_window -from clepsy.jobs.sessions import run_sessionization_job -from clepsy.jobs.goals import run_update_previous_full_period_result_job +from clepsy.db import get_db_connection +from clepsy.db.queries import delete_scheduled_job_by_key, upsert_scheduled_job +from clepsy.entities import GoalPeriod, JobType, ScheduledJob from clepsy.infra.valkey_client import get_connection +from clepsy.jobs.scheduler_tick import scheduler_tick -logger = logging.getLogger("apscheduler") -# Persistent datastore for APScheduler v4 -_data_store = SQLAlchemyDataStore( - engine_or_url=config.ap_scheduler_db_connection_string -) +AGGREGATION_SCHEDULE_KEY = "aggregation_window" +SESSIONIZATION_SCHEDULE_KEY = "sessions_sessionization" +GOAL_PREV_PERIOD_KEY_PREFIX = "goal_prev_period:" +SCHEDULER_INIT_LOCK_KEY = "scheduler:init_lock" -task_defaults = TaskDefaults( - job_executor="async", - misfire_grace_time=timedelta(days=365), - max_running_jobs=5, - metadata={}, -) +def utc_now() -> datetime: + return datetime.now(tz=timezone.utc) -# Module-level scheduler instance (use as async context manager in main.py) -scheduler = AsyncScheduler( - data_store=_data_store, - max_concurrent_jobs=10, - job_executors={"async": AsyncJobExecutor()}, - logger=logger, - task_defaults=task_defaults, -) +def align_interval_forward(now: datetime, interval: timedelta) -> datetime: + if interval.total_seconds() <= 0: + raise ValueError("Interval must be positive") + if now.tzinfo is None: + now = now.replace(tzinfo=timezone.utc) -def build_scheduler() -> AsyncScheduler: - """Return the module-level scheduler (kept for compatibility).""" - return scheduler - - -# Task IDs for persisted schedules (avoid pickling callables) -TASK_ID_SESSIONIZATION = "tasks.sessionization" -TASK_ID_AGGREGATION = "tasks.aggregation" -TASK_ID_GOAL_PREV_FULL_PERIOD = "tasks.goal_prev_full_period" - + seconds = int(interval.total_seconds()) + remainder = int(now.timestamp()) % seconds + if remainder == 0: + aligned = now + else: + aligned = now + timedelta(seconds=seconds - remainder) + return aligned.replace(microsecond=0) -async def init_schedules(sched: AsyncScheduler) -> None: - """Register core recurring schedules on an initialized scheduler. - Must be called after entering the scheduler's async context manager. - Uses a Redis lock to ensure only one worker initializes schedules. - """ +def cron_expr_for_interval(interval: timedelta) -> str: + total_seconds = int(interval.total_seconds()) + if total_seconds <= 0: + raise ValueError("Interval must be positive") - # Register task callables on ALL workers (must be done locally on each worker) - await sched.configure_task( # type: ignore[attr-defined] - func_or_task_id=TASK_ID_SESSIONIZATION, func=run_sessionization_job.send - ) - await sched.configure_task( # type: ignore[attr-defined] - func_or_task_id=TASK_ID_AGGREGATION, func=aggregate_window.send - ) - await sched.configure_task( # type: ignore[attr-defined] - func_or_task_id=TASK_ID_GOAL_PREV_FULL_PERIOD, - func=run_update_previous_full_period_result_job.send, - ) + total_minutes, remainder = divmod(total_seconds, 60) + if total_minutes <= 1 or remainder != 0: + return "* * * * *" + return f"*/{total_minutes} * * * *" - conn = get_connection() - lock_key = "scheduler:init_lock" - lock_acquired = False - - try: - # Try to acquire lock with 10 second timeout (first worker wins) - lock_acquired = conn.set(lock_key, "1", nx=True, ex=10) - - if not lock_acquired: - logger.info("Another worker is initializing schedules, skipping...") - return - logger.info("Acquired scheduler init lock, registering schedules...") - - # Only the lock holder creates the schedules in the database - await sched.add_schedule( # type: ignore[attr-defined] - TASK_ID_SESSIONIZATION, - trigger=IntervalTrigger( - seconds=int(config.session_window_length.total_seconds()) - ), - id="sessionization", - conflict_policy=ConflictPolicy.replace, - ) - await sched.add_schedule( # type: ignore[attr-defined] - TASK_ID_AGGREGATION, - trigger=IntervalTrigger( - seconds=int(config.aggregation_interval.total_seconds()) - ), - id="aggregation", - conflict_policy=ConflictPolicy.replace, - ) - - logger.info("Successfully registered schedules") - finally: - # Release lock if we acquired it - if lock_acquired: - conn.delete(lock_key) +def cron_expr_for_goal_period(period: GoalPeriod) -> str: + match period: + case GoalPeriod.DAY: + return "0 0 * * *" + case GoalPeriod.WEEK: + return "0 0 * * 1" + case GoalPeriod.MONTH: + return "0 0 1 * *" + case _: + raise ValueError(f"Unsupported GoalPeriod: {period}") -# ---- Goal cron helpers (reused from main branch with adjustments) ---- def get_next_period_range( *, relative_to: datetime, period: GoalPeriod ) -> tuple[datetime, datetime]: @@ -132,98 +81,117 @@ def get_next_period_range( return start, end -def cron_trigger_for_period( - *, period: GoalPeriod, min_first_start_time: datetime -) -> CronTrigger: - """Create a CronTrigger that fires when a period ends. +def resolve_timezone(timezone_str: str | None) -> ZoneInfo: + if timezone_str: + try: + return ZoneInfo(timezone_str) + except ZoneInfoNotFoundError: # noqa: BLE001 + logger.warning("Unknown timezone '{}'; defaulting to UTC", timezone_str) + return ZoneInfo("UTC") + + +async def ensure_core_schedules(now: datetime) -> None: + aggregation_job = ScheduledJob( + schedule_key=AGGREGATION_SCHEDULE_KEY, + job_type=JobType.AGGREGATION_WINDOW, + cron_expr=None, # Self-managed scheduling + next_run_at=align_interval_forward( + now, config.aggregation_interval + config.aggregation_grace_period + ), + ) + + session_job = ScheduledJob( + schedule_key=SESSIONIZATION_SCHEDULE_KEY, + job_type=JobType.SESSIONIZATION, + cron_expr=None, # Self-managed scheduling + next_run_at=align_interval_forward( + now, config.session_window_length + config.aggregation_grace_period + ), + ) - - DAY: fires daily at 00:00 (the moment the previous day ends) - - WEEK: fires Mondays at 00:00 (weeks start Monday, so previous week ends then) - - MONTH: fires on the 1st of each month at 00:00 (previous month just ended) + # Use longer busy_timeout for initialization to handle potential database locks from: + # - bootstrap.init() connections that may not be fully closed yet + # - WAL checkpoint operations + # - Concurrent uvicorn workers in production (entrypoint.sh uses --workers 2) + async with get_db_connection( + start_transaction=True, + transaction_type="IMMEDIATE", + busy_timeout=30000, # 30 seconds to wait for locks to clear + ) as conn: + await upsert_scheduled_job(conn, job=aggregation_job) + await upsert_scheduled_job(conn, job=session_job) + + +async def initialize_scheduler() -> None: + lock_conn = get_connection(decode_responses=True) + lock_acquired = False - The first occurrence will be strictly after `min_first_start_time`. - The cron's timezone is taken from `min_first_start_time` (or UTC if naive). - """ - tz = min_first_start_time.tzinfo or dt_timezone.utc + try: + lock_acquired = lock_conn.set(SCHEDULER_INIT_LOCK_KEY, "1", nx=True, ex=30) + if not lock_acquired: + logger.info("Another worker is initializing scheduler; skipping core setup") + return - start_after = ( - min_first_start_time - if min_first_start_time.tzinfo - else min_first_start_time.replace(tzinfo=tz) - ) - start_after = start_after + timedelta(seconds=1) + now = utc_now() + logger.info("Initializing scheduler core schedules at {}", now.isoformat()) + await ensure_core_schedules(now) - match period: - case GoalPeriod.DAY: - return CronTrigger( - hour=0, minute=0, second=0, timezone=tz, start_time=start_after - ) - case GoalPeriod.WEEK: - return CronTrigger( - day_of_week="mon", - hour=0, - minute=0, - second=0, - timezone=tz, - start_time=start_after, - ) - case GoalPeriod.MONTH: - return CronTrigger( - day=1, hour=0, minute=0, second=0, timezone=tz, start_time=start_after - ) - case _: - raise ValueError(f"Unsupported GoalPeriod: {period}") + # Kick the scheduler tick loop once; it will reschedule itself. + scheduler_tick.send() + finally: + if lock_acquired: + lock_conn.delete(SCHEDULER_INIT_LOCK_KEY) -def cron_trigger_given_period_and_created_at( - *, period: GoalPeriod, created_at: datetime -) -> CronTrigger: - next_period_range = get_next_period_range(relative_to=created_at, period=period) - return cron_trigger_for_period( - period=period, min_first_start_time=next_period_range[1] - timedelta(seconds=1) +async def schedule_goal_previous_period_update( + *, + goal_id: int, + period: GoalPeriod, + timezone_str: str | None, + created_at: datetime, + conn: aiosqlite.Connection, +) -> None: + tzinfo = resolve_timezone(timezone_str) + if created_at.tzinfo is None: + created_at = created_at.replace(tzinfo=timezone.utc) + + reference = datetime.now(tzinfo) + created_at_local = created_at.astimezone(tzinfo) + if created_at_local > reference: + reference = created_at_local + + next_boundary, _ = get_next_period_range(relative_to=reference, period=period) + next_run_utc = next_boundary.astimezone(timezone.utc) + + payload = { + "goal_id": goal_id, + } + + job = ScheduledJob( + schedule_key=f"{GOAL_PREV_PERIOD_KEY_PREFIX}{goal_id}", + job_type=JobType.GOAL_UPDATE_PREVIOUS_PERIOD, + cron_expr=cron_expr_for_goal_period(period), + next_run_at=next_run_utc, + payload=payload, ) + await upsert_scheduled_job(conn, job=job) -async def schedule_goal_previous_period_update( - *, goal_id: int, period: GoalPeriod, timezone_str: str | None, created_at: datetime + +async def unschedule_goal_previous_period_update( + *, goal_id: int, conn: aiosqlite.Connection | None = None ) -> None: - """Schedule periodic updates for a goal's previous full period result. + schedule_key = f"{GOAL_PREV_PERIOD_KEY_PREFIX}{goal_id}" - Uses the goal's period and timezone to align execution at period boundaries. - """ - tzinfo = None - if timezone_str: - try: - tzinfo = ZoneInfo(timezone_str) - except ZoneInfoNotFoundError: # noqa: BLE001 - tzinfo = dt_timezone.utc + if conn is None: + async with get_db_connection(start_transaction=True) as owned_conn: + await delete_scheduled_job_by_key(owned_conn, schedule_key=schedule_key) else: - tzinfo = dt_timezone.utc - - created_at_tz = created_at.astimezone(tzinfo) - trig = cron_trigger_given_period_and_created_at( - period=period, created_at=created_at_tz - ) - sched = build_scheduler() - # Ensure task is configured (idempotent) - await sched.configure_task( # type: ignore[attr-defined] - func_or_task_id=TASK_ID_GOAL_PREV_FULL_PERIOD, - func=run_update_previous_full_period_result_job.send, - ) - await sched.add_schedule( # type: ignore[attr-defined] - TASK_ID_GOAL_PREV_FULL_PERIOD, - trigger=trig, - id=f"goal_prev_full_period:{goal_id}", - args=(goal_id,), - conflict_policy=ConflictPolicy.replace, - ) + await delete_scheduled_job_by_key(conn, schedule_key=schedule_key) -async def unschedule_goal_previous_period_update(*, goal_id: int) -> None: - sched = build_scheduler() - schedule_id = f"goal_prev_full_period:{goal_id}" - # Best-effort removal; ignore if not found - try: - await sched.remove_schedule(schedule_id) # type: ignore[attr-defined] - except (LookupError, ValueError, AttributeError): # noqa: BLE001 - pass +__all__ = [ + "initialize_scheduler", + "schedule_goal_previous_period_update", + "unschedule_goal_previous_period_update", +] diff --git a/src/clepsy/utils.py b/src/clepsy/utils.py index dbce3ab..34e369b 100755 --- a/src/clepsy/utils.py +++ b/src/clepsy/utils.py @@ -4,9 +4,8 @@ import math import os import re -from typing import Callable, Sequence, TypeVar +from typing import Callable, Sequence, TypeVar, cast from uuid import UUID, uuid4 -import zlib from zoneinfo import ZoneInfo, ZoneInfoNotFoundError from baml_py import FunctionLog, Image as BamlImage @@ -409,7 +408,7 @@ def get_bootstrap_password() -> str: password = generate_bootstrap_password() write_bootstrap_password_file(password) logger.info( - "Generated new Clepsy bootstrap password at %s. Retrieve it from the container volume and rotate it after login.", + "Generated new Clepsy bootstrap password at {}. Retrieve it from the container volume and rotate it after login.", config.bootstrap_password_file_path, ) return password @@ -524,30 +523,25 @@ def store_image_in_redis( ttl_seconds: Time-to-live in seconds (default 10 minutes) """ - # Serialize image to PNG bytes + # Serialize image to PNG bytes and store directly in Redis buffered = io.BytesIO() image.save(buffered, format="PNG") image_bytes = buffered.getvalue() - # Compress with zlib - compressed = zlib.compress(image_bytes, level=6) - # Store in Redis with TTL - conn = get_connection() + conn = get_connection(decode_responses=False) key = f"image:{event_id}" - conn.setex(key, ttl_seconds, compressed) + conn.setex(key, ttl_seconds, image_bytes) logger.debug( - "Stored image {event_id} in Redis: {original} bytes -> {compressed} bytes ({ratio:.1f}% compression)", + "Stored image {event_id} in Redis: {size} bytes", event_id=event_id, - original=len(image_bytes), - compressed=len(compressed), - ratio=(1 - len(compressed) / len(image_bytes)) * 100, + size=len(image_bytes), ) def retrieve_image_from_redis(event_id: UUID) -> Image.Image | None: - """Retrieve and decompress an image from Redis. + """Retrieve and load an image from Redis. Args: event_id: UUID key to retrieve @@ -556,31 +550,22 @@ def retrieve_image_from_redis(event_id: UUID) -> Image.Image | None: PIL Image if found, None if expired or missing """ - conn = get_connection() + conn = get_connection(decode_responses=False) key = f"image:{event_id}" - compressed_data = conn.get(key) + image_bytes = cast(bytes | None, conn.get(key)) - if compressed_data is None: + if image_bytes is None: logger.warning( "Image {event_id} not found in Redis (expired or missing)", event_id=event_id, ) return None - # Type guard: ensure we have bytes (decode_responses=False ensures this) - assert isinstance( - compressed_data, bytes - ), f"Expected bytes from Redis, got {type(compressed_data)}" - - # Decompress and load image - image_bytes = zlib.decompress(compressed_data) - image = Image.open(io.BytesIO(image_bytes)) - image.load() # Force load into memory before BytesIO goes out of scope - - # Delete from Redis after successful retrieval - conn.delete(key) + with io.BytesIO(image_bytes) as f: + image = Image.open(f) + image.load() - logger.debug("Retrieved and deleted image {event_id} from Redis", event_id=event_id) + logger.debug("Retrieved {event_id} image from Redis", event_id=event_id) return image @@ -619,6 +604,18 @@ def parse_utc_naive_iso(s: str) -> datetime: return dt.astimezone(timezone.utc) +def ensure_utc(dt: datetime) -> datetime: + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + return dt.astimezone(timezone.utc) + + +def datetime_to_eta(dt: datetime) -> int: + """Convert a datetime to epoch milliseconds for Dramatiq ETA scheduling.""" + + return int(ensure_utc(dt).timestamp() * 1000) + + def custom_template(pattern: str) -> Callable[[str, dict], str]: compiled_pattern = re.compile(pattern) diff --git a/uv.lock b/uv.lock index f6dd4d5..f66f7af 100755 --- a/uv.lock +++ b/uv.lock @@ -40,6 +40,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f5/10/6c25ed6de94c49f88a91fa5018cb4c0f3625f31d5be9f771ebe5cc7cd506/aiosqlite-0.21.0-py3-none-any.whl", hash = "sha256:2549cf4057f95f53dcba16f2b64e8e2791d7e1adedb13197dd8ed77bb226d7d0", size = 15792, upload-time = "2025-02-03T07:30:13.6Z" }, ] +[[package]] +name = "aiosqlitepool" +version = "1.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c6/5a/f3184cdfd195a748bbb330894e34e5b274fec0e9b8dfac4c1fc71f36fc8b/aiosqlitepool-1.0.0.tar.gz", hash = "sha256:397f79993d7f34a5740939fb6e52ff29563fad5c400ef8b70990e64331957409", size = 16491, upload-time = "2025-07-11T10:15:43.029Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e0/65/4d9a7eb8a4cf6a586f14abcce9d774d5b4a986e3c3028a9c88801c9648d2/aiosqlitepool-1.0.0-py3-none-any.whl", hash = "sha256:832acb166bb9afef7f46b320d024b343083c90f4eb4bdc8c0d794a79e1fd1b4d", size = 12946, upload-time = "2025-07-11T10:15:41.953Z" }, +] + [[package]] name = "aistudio-sdk" version = "0.3.8" @@ -88,26 +97,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, ] -[[package]] -name = "apscheduler" -version = "4.0.0a6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "attrs" }, - { name = "tenacity" }, - { name = "tzlocal" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/40/7f/ca0404dac83438b5dcb75c797092381040320be84f3af098cf9a486529e0/apscheduler-4.0.0a6.tar.gz", hash = "sha256:5134617c028f097de4a09abbeefc42625cb0ce3adcb4ce49d74cc26054084761", size = 3116644, upload-time = "2025-04-27T08:36:22.252Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3d/bd/35b5691919859a906cd098d62e2a1399cb8ede3e05b6480cf15d472ad5b1/apscheduler-4.0.0a6-py3-none-any.whl", hash = "sha256:87031f7537aaa6ea2d3e69fdd9454b641f18938fc8cd0c589972006eceba8ee9", size = 88115, upload-time = "2025-04-27T08:36:20.315Z" }, -] - -[package.optional-dependencies] -sqlalchemy = [ - { name = "sqlalchemy", extra = ["asyncio"] }, -] - [[package]] name = "argon2-cffi" version = "25.1.0" @@ -163,15 +152,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/01/d9/782ffcb4c97b889bc12d8276637d2739b99520390ee8fec77c07416c5d12/asyncclick-8.3.0.7-py3-none-any.whl", hash = "sha256:7607046de39a3f315867cad818849f973e29d350c10d92f251db3ff7600c6c7d", size = 109925, upload-time = "2025-10-11T08:35:43.378Z" }, ] -[[package]] -name = "attrs" -version = "25.4.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6b/5c/685e6633917e101e5dcb62b9dd76946cbb57c26e133bae9e0cd36033c0a9/attrs-25.4.0.tar.gz", hash = "sha256:16d5969b87f0859ef33a48b35d55ac1be6e42ae49d5e853b597db70c35c57e11", size = 934251, upload-time = "2025-10-06T13:54:44.725Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/2a/7cc015f5b9f5db42b7d48157e23356022889fc354a2813c15934b7cb5c0e/attrs-25.4.0-py3-none-any.whl", hash = "sha256:adcf7e2a1fb3b36ac48d97835bb6d8ade15b8dcce26aba8bf1d14847b57a3373", size = 67615, upload-time = "2025-10-06T13:54:43.17Z" }, -] - [[package]] name = "baml-py" version = "0.213.0" @@ -350,10 +330,11 @@ dependencies = [ { name = "aiocache" }, { name = "aiomcache" }, { name = "aiosqlite" }, - { name = "apscheduler", extra = ["sqlalchemy"] }, + { name = "aiosqlitepool" }, { name = "argon2-cffi" }, { name = "asyncclick" }, { name = "baml-py" }, + { name = "croniter" }, { name = "cryptography" }, { name = "dramatiq", extra = ["redis", "watch"] }, { name = "fastapi" }, @@ -375,7 +356,6 @@ dependencies = [ { name = "pyjwt" }, { name = "python-dateutil" }, { name = "python-multipart" }, - { name = "sqlalchemy" }, { name = "tenacity" }, { name = "torch", version = "2.9.0", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform == 'darwin'" }, { name = "torch", version = "2.9.0+cpu", source = { registry = "https://download.pytorch.org/whl/cpu" }, marker = "sys_platform != 'darwin'" }, @@ -406,10 +386,11 @@ requires-dist = [ { name = "aiocache", specifier = ">=0.12.3" }, { name = "aiomcache", specifier = ">=0.8.2" }, { name = "aiosqlite", specifier = ">=0.20.0" }, - { name = "apscheduler", extras = ["sqlalchemy"], specifier = ">=4.0.0.0a6" }, + { name = "aiosqlitepool", specifier = ">=1.0.0" }, { name = "argon2-cffi", specifier = ">=25.1.0" }, { name = "asyncclick", specifier = ">=8.1.7.2" }, { name = "baml-py", specifier = "==0.213.0" }, + { name = "croniter", specifier = ">=6.0.0" }, { name = "cryptography", specifier = ">=44.0.2" }, { name = "dramatiq", extras = ["redis", "watch"], specifier = ">=1.18.0" }, { name = "fastapi", specifier = ">=0.111.1" }, @@ -431,7 +412,6 @@ requires-dist = [ { name = "pyjwt", specifier = ">=2.10.1" }, { name = "python-dateutil", specifier = ">=2.9.0" }, { name = "python-multipart", specifier = ">=0.0.20" }, - { name = "sqlalchemy", specifier = ">=2.0.43" }, { name = "tenacity", specifier = ">=9.0.0" }, { name = "torch", specifier = ">=2.0.0", index = "https://download.pytorch.org/whl/cpu" }, { name = "tzlocal", specifier = ">=5.3" }, @@ -575,6 +555,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/19/8f/92bdd27b067204b99f396a1414d6342122f3e2663459baf787108a6b8b84/coverage-7.11.3-py3-none-any.whl", hash = "sha256:351511ae28e2509c8d8cae5311577ea7dd511ab8e746ffc8814a0896c3d33fbe", size = 208478, upload-time = "2025-11-10T00:13:14.908Z" }, ] +[[package]] +name = "croniter" +version = "6.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "python-dateutil" }, + { name = "pytz" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/2f/44d1ae153a0e27be56be43465e5cb39b9650c781e001e7864389deb25090/croniter-6.0.0.tar.gz", hash = "sha256:37c504b313956114a983ece2c2b07790b1f1094fe9d81cc94739214748255577", size = 64481, upload-time = "2024-12-17T17:17:47.32Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/07/4b/290b4c3efd6417a8b0c284896de19b1d5855e6dbdb97d2a35e68fa42de85/croniter-6.0.0-py2.py3-none-any.whl", hash = "sha256:2f878c3856f17896979b2a4379ba1f09c83e374931ea15cc835c5dd2eee9b368", size = 25468, upload-time = "2024-12-17T17:17:45.359Z" }, +] + [[package]] name = "cryptography" version = "46.0.3" @@ -2460,6 +2453,8 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6b/fa/3234f913fe9a6525a7b97c6dad1f51e72b917e6872e051a5e2ffd8b16fbb/ruamel.yaml.clib-0.2.14-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:70eda7703b8126f5e52fcf276e6c0f40b0d314674f896fc58c47b0aef2b9ae83", size = 137970, upload-time = "2025-09-22T19:51:09.472Z" }, { url = "https://files.pythonhosted.org/packages/ef/ec/4edbf17ac2c87fa0845dd366ef8d5852b96eb58fcd65fc1ecf5fe27b4641/ruamel.yaml.clib-0.2.14-cp314-cp314-musllinux_1_2_i686.whl", hash = "sha256:a0cb71ccc6ef9ce36eecb6272c81afdc2f565950cdcec33ae8e6cd8f7fc86f27", size = 739639, upload-time = "2025-09-22T19:51:10.566Z" }, { url = "https://files.pythonhosted.org/packages/15/18/b0e1fafe59051de9e79cdd431863b03593ecfa8341c110affad7c8121efc/ruamel.yaml.clib-0.2.14-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:e7cb9ad1d525d40f7d87b6df7c0ff916a66bc52cb61b66ac1b2a16d0c1b07640", size = 764456, upload-time = "2025-09-22T19:51:11.736Z" }, + { url = "https://files.pythonhosted.org/packages/e7/cd/150fdb96b8fab27fe08d8a59fe67554568727981806e6bc2677a16081ec7/ruamel_yaml_clib-0.2.14-cp314-cp314-win32.whl", hash = "sha256:9b4104bf43ca0cd4e6f738cb86326a3b2f6eef00f417bd1e7efb7bdffe74c539", size = 102394, upload-time = "2025-11-14T21:57:36.703Z" }, + { url = "https://files.pythonhosted.org/packages/bd/e6/a3fa40084558c7e1dc9546385f22a93949c890a8b2e445b2ba43935f51da/ruamel_yaml_clib-0.2.14-cp314-cp314-win_amd64.whl", hash = "sha256:13997d7d354a9890ea1ec5937a219817464e5cc344805b37671562a401ca3008", size = 122673, upload-time = "2025-11-14T21:57:38.177Z" }, ] [[package]] @@ -2636,40 +2631,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, ] -[[package]] -name = "sqlalchemy" -version = "2.0.44" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet", marker = "platform_machine == 'AMD64' or platform_machine == 'WIN32' or platform_machine == 'aarch64' or platform_machine == 'amd64' or platform_machine == 'ppc64le' or platform_machine == 'win32' or platform_machine == 'x86_64'" }, - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f0/f2/840d7b9496825333f532d2e3976b8eadbf52034178aac53630d09fe6e1ef/sqlalchemy-2.0.44.tar.gz", hash = "sha256:0ae7454e1ab1d780aee69fd2aae7d6b8670a581d8847f2d1e0f7ddfbf47e5a22", size = 9819830, upload-time = "2025-10-10T14:39:12.935Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/62/c4/59c7c9b068e6813c898b771204aad36683c96318ed12d4233e1b18762164/sqlalchemy-2.0.44-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:72fea91746b5890f9e5e0997f16cbf3d53550580d76355ba2d998311b17b2250", size = 2139675, upload-time = "2025-10-10T16:03:31.064Z" }, - { url = "https://files.pythonhosted.org/packages/d6/ae/eeb0920537a6f9c5a3708e4a5fc55af25900216bdb4847ec29cfddf3bf3a/sqlalchemy-2.0.44-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:585c0c852a891450edbb1eaca8648408a3cc125f18cf433941fa6babcc359e29", size = 2127726, upload-time = "2025-10-10T16:03:35.934Z" }, - { url = "https://files.pythonhosted.org/packages/d8/d5/2ebbabe0379418eda8041c06b0b551f213576bfe4c2f09d77c06c07c8cc5/sqlalchemy-2.0.44-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9b94843a102efa9ac68a7a30cd46df3ff1ed9c658100d30a725d10d9c60a2f44", size = 3327603, upload-time = "2025-10-10T15:35:28.322Z" }, - { url = "https://files.pythonhosted.org/packages/45/e5/5aa65852dadc24b7d8ae75b7efb8d19303ed6ac93482e60c44a585930ea5/sqlalchemy-2.0.44-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:119dc41e7a7defcefc57189cfa0e61b1bf9c228211aba432b53fb71ef367fda1", size = 3337842, upload-time = "2025-10-10T15:43:45.431Z" }, - { url = "https://files.pythonhosted.org/packages/41/92/648f1afd3f20b71e880ca797a960f638d39d243e233a7082c93093c22378/sqlalchemy-2.0.44-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:0765e318ee9179b3718c4fd7ba35c434f4dd20332fbc6857a5e8df17719c24d7", size = 3264558, upload-time = "2025-10-10T15:35:29.93Z" }, - { url = "https://files.pythonhosted.org/packages/40/cf/e27d7ee61a10f74b17740918e23cbc5bc62011b48282170dc4c66da8ec0f/sqlalchemy-2.0.44-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:2e7b5b079055e02d06a4308d0481658e4f06bc7ef211567edc8f7d5dce52018d", size = 3301570, upload-time = "2025-10-10T15:43:48.407Z" }, - { url = "https://files.pythonhosted.org/packages/3b/3d/3116a9a7b63e780fb402799b6da227435be878b6846b192f076d2f838654/sqlalchemy-2.0.44-cp312-cp312-win32.whl", hash = "sha256:846541e58b9a81cce7dee8329f352c318de25aa2f2bbe1e31587eb1f057448b4", size = 2103447, upload-time = "2025-10-10T15:03:21.678Z" }, - { url = "https://files.pythonhosted.org/packages/25/83/24690e9dfc241e6ab062df82cc0df7f4231c79ba98b273fa496fb3dd78ed/sqlalchemy-2.0.44-cp312-cp312-win_amd64.whl", hash = "sha256:7cbcb47fd66ab294703e1644f78971f6f2f1126424d2b300678f419aa73c7b6e", size = 2130912, upload-time = "2025-10-10T15:03:24.656Z" }, - { url = "https://files.pythonhosted.org/packages/45/d3/c67077a2249fdb455246e6853166360054c331db4613cda3e31ab1cadbef/sqlalchemy-2.0.44-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ff486e183d151e51b1d694c7aa1695747599bb00b9f5f604092b54b74c64a8e1", size = 2135479, upload-time = "2025-10-10T16:03:37.671Z" }, - { url = "https://files.pythonhosted.org/packages/2b/91/eabd0688330d6fd114f5f12c4f89b0d02929f525e6bf7ff80aa17ca802af/sqlalchemy-2.0.44-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0b1af8392eb27b372ddb783b317dea0f650241cea5bd29199b22235299ca2e45", size = 2123212, upload-time = "2025-10-10T16:03:41.755Z" }, - { url = "https://files.pythonhosted.org/packages/b0/bb/43e246cfe0e81c018076a16036d9b548c4cc649de241fa27d8d9ca6f85ab/sqlalchemy-2.0.44-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b61188657e3a2b9ac4e8f04d6cf8e51046e28175f79464c67f2fd35bceb0976", size = 3255353, upload-time = "2025-10-10T15:35:31.221Z" }, - { url = "https://files.pythonhosted.org/packages/b9/96/c6105ed9a880abe346b64d3b6ddef269ddfcab04f7f3d90a0bf3c5a88e82/sqlalchemy-2.0.44-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b87e7b91a5d5973dda5f00cd61ef72ad75a1db73a386b62877d4875a8840959c", size = 3260222, upload-time = "2025-10-10T15:43:50.124Z" }, - { url = "https://files.pythonhosted.org/packages/44/16/1857e35a47155b5ad927272fee81ae49d398959cb749edca6eaa399b582f/sqlalchemy-2.0.44-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:15f3326f7f0b2bfe406ee562e17f43f36e16167af99c4c0df61db668de20002d", size = 3189614, upload-time = "2025-10-10T15:35:32.578Z" }, - { url = "https://files.pythonhosted.org/packages/88/ee/4afb39a8ee4fc786e2d716c20ab87b5b1fb33d4ac4129a1aaa574ae8a585/sqlalchemy-2.0.44-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1e77faf6ff919aa8cd63f1c4e561cac1d9a454a191bb864d5dd5e545935e5a40", size = 3226248, upload-time = "2025-10-10T15:43:51.862Z" }, - { url = "https://files.pythonhosted.org/packages/32/d5/0e66097fc64fa266f29a7963296b40a80d6a997b7ac13806183700676f86/sqlalchemy-2.0.44-cp313-cp313-win32.whl", hash = "sha256:ee51625c2d51f8baadf2829fae817ad0b66b140573939dd69284d2ba3553ae73", size = 2101275, upload-time = "2025-10-10T15:03:26.096Z" }, - { url = "https://files.pythonhosted.org/packages/03/51/665617fe4f8c6450f42a6d8d69243f9420f5677395572c2fe9d21b493b7b/sqlalchemy-2.0.44-cp313-cp313-win_amd64.whl", hash = "sha256:c1c80faaee1a6c3428cecf40d16a2365bcf56c424c92c2b6f0f9ad204b899e9e", size = 2127901, upload-time = "2025-10-10T15:03:27.548Z" }, - { url = "https://files.pythonhosted.org/packages/9c/5e/6a29fa884d9fb7ddadf6b69490a9d45fded3b38541713010dad16b77d015/sqlalchemy-2.0.44-py3-none-any.whl", hash = "sha256:19de7ca1246fbef9f9d1bff8f1ab25641569df226364a0e40457dc5457c54b05", size = 1928718, upload-time = "2025-10-10T15:29:45.32Z" }, -] - -[package.optional-dependencies] -asyncio = [ - { name = "greenlet" }, -] - [[package]] name = "starlette" version = "0.49.3"