diff --git a/.dockerignore b/.dockerignore index 97f8580d9..eca6c8fe8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,6 +9,17 @@ __pycache__/ dist/ build/ .env +.env.bak.* +# Secrets: keep plaintext and every transient secrets.env variant out of +# the build context. If an encrypted secrets.env is used, it is mounted +# at runtime — never baked into the image. Mirrored in .gitignore. +secrets.env +secrets.env.* +secrets.env~ +.secrets.env.swp +.secrets.env.swo +**/#secrets.env# +!secrets.env.example /data/ /logs/ .git/ @@ -18,6 +29,8 @@ build/ .vscode/ .idea/ dev-docs/ +docs/ +*.md *.db *.sqlite *.sqlite3 diff --git a/.env.example b/.env.example index 44a6e1559..e96dd151c 100644 --- a/.env.example +++ b/.env.example @@ -16,6 +16,10 @@ LLM_HOST=localhost # when started with OLLAMA_HOST=0.0.0.0:11434. # OLLAMA_BASE_URL=http://host.docker.internal:11434/v1 +# Optional LM Studio URL. In Docker, host LM Studio is reachable here +# when LM Studio is set to serve on all interfaces (0.0.0.0). +# LM_STUDIO_URL=http://host.docker.internal:1234 + # OpenAI API key (only needed if using OpenAI models). # Do not commit real keys. Keep this commented until needed. # OPENAI_API_KEY=your_openai_api_key_here @@ -23,6 +27,16 @@ LLM_HOST=localhost # Research service LLM endpoint # RESEARCH_LLM_ENDPOINT=http://localhost:8000/v1/chat/completions +# Extra CA bundle for LLM providers whose TLS chain isn't in the default +# trust store. Layered ON TOP of the system / certifi bundle — verification +# stays on for every host, the trust set just gets larger. Useful for: +# - GigaChat / Sber (Russian Trusted Root CA): without this the endpoint +# shows offline with CERTIFICATE_VERIFY_FAILED — self-signed certificate +# in certificate chain. +# - On-premise / corporate LLM gateways with an internal CA. +# Point at a PEM file containing the missing root(s). +# LLM_CA_BUNDLE=/etc/odysseus/ca/extra-roots.pem + # ============================================================ # Search & Web # ============================================================ @@ -42,6 +56,13 @@ SEARXNG_INSTANCE=http://localhost:8080 # SQLite database path (default: sqlite:///./data/app.db) # DATABASE_URL=sqlite:///./data/app.db +# ============================================================ +# Data directory +# ============================================================ +# Move everything that lives under data/ - settings, sessions, database, auth, +# cache, uploads, etc. - to another path: +# ODYSSEUS_DATA_DIR=C:\path\to\dir + # ============================================================ # Auth & Security # ============================================================ @@ -49,10 +70,20 @@ SEARXNG_INSTANCE=http://localhost:8080 # Enable authentication (default: true) # AUTH_ENABLED=true +# Host bind address and port for the Odysseus web UI in Docker Compose. +# Keep APP_BIND on loopback unless you intentionally want LAN/reverse-proxy access. +# APP_BIND=127.0.0.1 +# Change this if another local service already uses 7000 (macOS AirPlay often does). +# APP_PORT=7000 + # Development-only auth bypass for loopback requests. # Keep false for Docker, LAN, reverse proxy, and any shared deployment. # LOCALHOST_BYPASS=false +# Mark session cookies Secure. Set true when Odysseus is served through HTTPS +# by a trusted reverse proxy or private access gateway. +# SECURE_COOKIES=true + # Optional: pre-seed the first admin password during setup. # Do not commit a real password. # ODYSSEUS_ADMIN_PASSWORD=change_me_before_first_boot @@ -88,6 +119,9 @@ SEARXNG_INSTANCE=http://localhost:8080 # Default: http://{LLM_HOST}:11434/v1/embeddings (ollama) # EMBEDDING_URL=http://localhost:11434/v1/embeddings +# Embedding API key (if there's one) +# EMBEDDING_API_KEY=embedding_api_key_here + # Embedding model name (must be available at the endpoint above) # EMBEDDING_MODEL=all-minilm:l6-v2 @@ -119,3 +153,67 @@ SEARXNG_INSTANCE=http://localhost:8080 # Empty/local/localhost runs scripts on the app host. Set to an SSH host alias # if you intentionally want scheduled scripts to run remotely. # ODYSSEUS_SCRIPT_HOST=localhost + +# Chat / agent attachment size cap in bytes (default: 10 MB). +# Raise this for local installs that need larger PDFs or text documents. +# Example: 52428800 = 50 MB. +# ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=10485760 + +# Other per-feature upload size caps in bytes. All are validated and optional; +# defaults shown. An invalid value (non-integer or < 1) fails fast at startup. +# ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=104857600 # gallery image upload (100 MB) +# ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=26214400 # gallery transform input (25 MB) +# ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=10485760 # memory import file (10 MB) +# ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=26214400 # personal document upload (25 MB) +# ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=26214400 # email compose attachment (25 MB) +# ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400 # speech-to-text audio (25 MB) +# ODYSSEUS_ICS_MAX_BYTES=10485760 # calendar .ics import (10 MB) + +# ============================================================ +# Host Docker access (explicit opt-in) +# ============================================================ +# Default Docker Compose does not mount /var/run/docker.sock. Existing +# Ollama, vLLM, and other OpenAI-compatible endpoints remain usable without it. +# +# Enable this only for intentional Cookbook/local Docker-daemon management. +# Raw socket access is high-trust and can grant broad control over the host +# Docker daemon. Set DOCKER_GID to the host docker group's numeric GID. +# Put these values in .env, or export them before running docker compose. +# COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml +# DOCKER_GID=963 +# docker/host-docker.yml sets this inside the container. Keep it paired +# with the socket overlay; setting it alone is not sufficient. +# ODYSSEUS_ENABLE_HOST_DOCKER=true +# +# Host Docker access can be combined with one GPU overlay: +# COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml +# COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml + +# ============================================================ +# GPU support (Docker Compose) +# ============================================================ +# Pass the host GPU into the odysseus container. Default (unset) = CPU. +# COMPOSE_FILE is a native `docker compose` feature: a colon-separated +# list of files merged left-to-right. Pick ONE GPU line below, or leave +# all commented for CPU. +# +# NVIDIA (requires nvidia-container-toolkit + `nvidia-ctk runtime +# configure --runtime=docker` on the host): +# COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml +# COMPOSE_FILE=docker-compose.yml;docker/gpu.nvidia.yml #(Windows) +# +# AMD ROCm (requires ROCm drivers on the host and the GID of the render group): +# COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml +# Find the render GID with: getent group render | cut -d: -f3 +# RENDER_GID=989 +# +# These overlays only expose the GPU devices. The slim Odysseus image +# still needs CUDA/ROCm userspace via Cookbook -> Dependencies (vLLM, +# llama-cpp-python, etc.) before models can actually serve on GPU. + +# ============================================================ +# Storage Paths (Docker Compose) +# ============================================================ + +# APP_DATA_DIR=./data +# APP_LOGS_DIR=./logs diff --git a/.gitattributes b/.gitattributes index d62b6c338..2db234ba7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,27 @@ -*.sh text eol=lf +# Normalize line endings so a Windows checkout (git core.autocrlf=true) can't +# corrupt shell-script shebangs. A CRLF `#!/bin/sh\r` makes the kernel look for +# an interpreter literally named "/bin/sh\r", producing the Docker startup error +# "exec /usr/local/bin/entrypoint.sh: no such file or directory" (issues #150, #77). +* text=auto + +# Shell scripts must stay LF on every platform (run by sh/bash, incl. in Docker). +*.sh text eol=lf +*.bash text eol=lf +entrypoint.sh text eol=lf docker/entrypoint.sh text eol=lf + +# Windows-native scripts stay CRLF. +*.ps1 text eol=crlf +*.cmd text eol=crlf +*.bat text eol=crlf + +# Binary assets — never normalize. +*.png binary +*.jpg binary +*.jpeg binary +*.gif binary +*.webp binary +*.pdf binary +*.ico binary +*.woff binary +*.woff2 binary diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..fc7545ace --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,9 @@ +# Code owners. +# +# Intentionally empty for now. The catch-all rule that mapped every path to a +# single owner froze all merges the moment "Require review from Code Owners" +# was enabled, because no other maintainer's approval could satisfy the gate. +# A per-area ownership map (security/auth, CI, frontend, agent internals, with +# multiple named owners per line) is being worked out in issue #593; once +# agreed it replaces this file. Until then, required reviews and the security +# CI gate (docs/security-ci.md) remain in force via branch protection. diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..3834b79d6 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,114 @@ +name: Bug Report +description: Report a reproducible bug in Odysseus. +labels: ["bug"] + +body: + - type: markdown + attributes: + value: | + **Before submitting:** search [open issues](https://github.com/odysseus-dev/odysseus/issues) + and [discussions](https://github.com/odysseus-dev/odysseus/discussions) first. + Duplicate reports slow things down. + + For security vulnerabilities, **do not open a public issue** — + use [GitHub Security Advisories](https://github.com/odysseus-dev/odysseus/security/advisories/new) + and read [SECURITY.md](https://github.com/odysseus-dev/odysseus/blob/main/SECURITY.md) first. + + - type: checkboxes + id: prerequisites + attributes: + label: Prerequisites + options: + - label: I searched [open issues](https://github.com/odysseus-dev/odysseus/issues?q=is%3Aissue+is%3Aopen) and [discussions](https://github.com/odysseus-dev/odysseus/discussions) and did not find an existing report of this bug. + required: true + - label: This is **not** a security vulnerability. (Vulnerabilities go to [GitHub Security Advisories](https://github.com/odysseus-dev/odysseus/security/advisories/new) — see [SECURITY.md](https://github.com/odysseus-dev/odysseus/blob/main/SECURITY.md).) + required: true + - label: I am running the latest code from the `dev` branch (the default branch you get on clone, where fixes land first) and the bug still reproduces there. Please `git pull` the latest `dev` before filing. + required: true + + - type: dropdown + id: install-method + attributes: + label: Install Method + options: + - "-- Please Select --" + - Docker (docker compose up) + - Manual Python install (pip / venv) + - Windows native (launch-windows.ps1) + - macOS app (build-macos-app.sh / start-macos.sh) + - Other (describe in the reproduction steps below) + validations: + required: true + + - type: dropdown + id: os + attributes: + label: Operating System + options: + - "-- Please Select --" + - Linux + - macOS + - Windows + - Other + validations: + required: true + + - type: textarea + id: steps + attributes: + label: Steps to Reproduce + description: Exact steps that reliably trigger the bug. The more specific, the faster this gets fixed. + placeholder: | + 1. Go to ... + 2. Click / type ... + 3. Observe ... + validations: + required: true + + - type: textarea + id: expected + attributes: + label: Expected Behaviour + description: What should have happened? + validations: + required: true + + - type: textarea + id: actual + attributes: + label: Actual Behaviour + description: What actually happened? Include the full error message if there is one. + validations: + required: true + + - type: textarea + id: logs + attributes: + label: Logs / Screenshots + description: Paste relevant terminal output or attach screenshots. Remove API keys, passwords, and personal data before pasting. + render: text + + - type: input + id: model-backend + attributes: + label: Model / Backend (if relevant) + description: "e.g. Ollama + llama3.2:latest, vLLM + mistral-7b, OpenAI API, Anthropic API" + placeholder: "Ollama + llama3.2:latest" + + - type: dropdown + id: willing_to_fix + attributes: + label: Are you willing to submit a fix? + options: + - "-- Please Select --" + - "Yes — I can open a PR" + - "Partially — I can help but need guidance" + - "No — I am only filing the report" + validations: + required: true + + - type: textarea + id: additional-info + attributes: + label: Additional Information + description: Anything else that might help — browser console errors, related issues, things you already tried, or environment quirks. diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 000000000..aa8ceaf18 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,13 @@ +blank_issues_enabled: false +contact_links: + - name: Question / Need Help + url: https://github.com/odysseus-dev/odysseus/discussions/categories/q-a + about: Ask how-to questions, setup help, and model configuration questions here. Issues are for confirmed bugs and concrete proposals only. + + - name: Idea or Suggestion + url: https://github.com/odysseus-dev/odysseus/discussions/categories/ideas + about: Discuss ideas and gauge interest before opening a formal feature request. If there is already a discussion, link it in your feature request. + + - name: Security Vulnerability + url: https://github.com/odysseus-dev/odysseus/security/advisories/new + about: Report vulnerabilities privately via GitHub Security Advisories — never as a public issue. Read SECURITY.md before reporting. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..4ee603ee9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,92 @@ +name: Feature Request +description: Propose a new feature or a concrete improvement to Odysseus. +labels: ["enhancement"] + +body: + - type: markdown + attributes: + value: | + **Before submitting:** search [open issues](https://github.com/odysseus-dev/odysseus/issues) + and [discussions](https://github.com/odysseus-dev/odysseus/discussions) first. + Feature requests that duplicate [ROADMAP.md](https://github.com/odysseus-dev/odysseus/blob/main/ROADMAP.md) + or an existing open issue will be closed as duplicates. + + If your idea needs community input before it becomes a concrete proposal, + start a [discussion](https://github.com/odysseus-dev/odysseus/discussions/categories/ideas) instead. + + - type: checkboxes + id: prerequisites + attributes: + label: Prerequisites + options: + - label: I searched [open issues](https://github.com/odysseus-dev/odysseus/issues?q=is%3Aissue+is%3Aopen) and this has not already been proposed. + required: true + - label: I searched [discussions](https://github.com/odysseus-dev/odysseus/discussions) and this is not already being debated there. + required: true + - label: This is a concrete, actionable proposal — not a vague "it would be nice if..." request. + required: true + + - type: dropdown + id: area + attributes: + label: Area + description: Which part of the application does this affect? + options: + - "-- Please Select --" + - Chat / Agent + - Email + - Calendar + - Documents / RAG + - Memory + - Cookbook / Local Models / GPU + - Search + - Notes / Editor + - Auth / Security + - Docker / Deployment + - UI / Frontend + - API / Backend + - MCP + - Testing / CI + - Other + validations: + required: true + + - type: textarea + id: problem + attributes: + label: Problem or Motivation + description: What problem does this solve, or what use case does it enable? Be specific — "it would be better" is not enough. + validations: + required: true + + - type: textarea + id: solution + attributes: + label: Proposed Solution + description: Describe the behaviour or change you want to see. Include API shape, UI sketch, or code snippets if that helps make it concrete. + validations: + required: true + + - type: textarea + id: alternatives + attributes: + label: Alternatives Considered + description: What other approaches did you consider and why did you rule them out? If there is an existing workaround, describe it. + + - type: textarea + id: prior-art + attributes: + label: Prior Art / Related Issues + description: Link any related issues, discussions, or external references that informed this proposal. + + - type: dropdown + id: willing_to_implement + attributes: + label: Are you willing to implement this? + options: + - "-- Please Select --" + - "Yes — I can open a PR" + - "Partially — I can help but need guidance" + - "No — I am only filing the request" + validations: + required: true diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 000000000..e1e0bf13e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,48 @@ +# Dependabot keeps dependencies and pinned action versions current. +# +# Why this matters for security: every workflow in this repo pins its GitHub +# Actions to an exact commit (a SHA), which is safe but freezes them in time. +# Dependabot opens a small, reviewable pull request whenever a newer version +# exists -- for Python packages, npm packages, the Docker base image, and the +# pinned Actions themselves -- so staying patched does not require manual work. +# Updates are grouped so a week's bumps arrive as one PR per ecosystem, not a +# flood of separate ones. + +version: 2 +updates: + # Python dependencies (requirements.txt + requirements-optional.txt). + - package-ecosystem: pip + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 5 + groups: + python: + patterns: ["*"] + + # Frontend / tooling npm packages (package.json). + - package-ecosystem: npm + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 5 + groups: + npm: + patterns: ["*"] + + # The pinned action SHAs used across .github/workflows. + - package-ecosystem: github-actions + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 5 + groups: + actions: + patterns: ["*"] + + # The Docker base image in the Dockerfile. + - package-ecosystem: docker + directory: "/" + schedule: + interval: weekly + open-pull-requests-limit: 5 diff --git a/.github/pull_request_review_template.md b/.github/pull_request_review_template.md new file mode 100644 index 000000000..725138545 --- /dev/null +++ b/.github/pull_request_review_template.md @@ -0,0 +1,123 @@ +# Pull Request Review Template + +Use this shape as a copyable reference for substantive PR reviews; GitHub does +not auto-apply this file to review comments. Omit sections that do not add +useful signal. Lead with confirmed findings; keep speculative notes out of the +public review unless they are framed as a concrete open question. + +## Small PR Path + +For narrow docs, typo, test-only, or obvious local fixes, a short review is +enough: + +```md +LGTM after checking: +- scope: +- validation: +- residual risk: +``` + +Use the fuller structure below for larger, risky, multi-finding, or +security-sensitive reviews. + +## Findings + +**![P2 Badge](https://img.shields.io/badge/P2-yellow?style=flat) issue (test): Short issue title** + +- **Problem:** Concrete broken flow, contract, input, or risk. + +- **Impact:** Why this matters to users, CI, maintainers, data, security, or scale. + +- **Ask:** Smallest practical correction or decision the author should make. + +- **Location:** `path:line` + +## Open Questions + +- **question (scope, non-blocking): Short author question** Ask the concrete + intent, scope, or tradeoff question. + +## Validation + +- Ran: +- Not run: +- Residual risk: + +## PR Hygiene + +- Target/template/checks: +- Related, duplicate, or superseding context: + +## No Findings Variant + +```md +## Findings + +none confirmed + +## Validation + +- Ran: +- Not run: +- Residual risk: +``` + +## Legend + +- **Findings:** Verified, author-actionable issues that should be fixed or + consciously accepted before merge. +- **Priority badges:** The shields.io badges below are optional formatting for + priority labels. Plain `P0`, `P1`, `P2`, or `P3` text is also acceptable when + an external image dependency is undesirable or may not render. + - **P0:** `![P0 Badge](https://img.shields.io/badge/P0-red?style=flat)` - + release-blocking or actively dangerous. + - **P1:** `![P1 Badge](https://img.shields.io/badge/P1-orange?style=flat)` - + serious bug, security risk, data-loss risk, or broken primary flow. + - **P2:** `![P2 Badge](https://img.shields.io/badge/P2-yellow?style=flat)` - + meaningful correctness, test, maintainability, or edge-case issue. + - **P3:** `![P3 Badge](https://img.shields.io/badge/P3-lightgrey?style=flat)` - + minor polish or low-risk cleanup. +- **Intent labels:** + - **`issue`:** A confirmed defect, regression, broken contract, or concrete + risk. + - **`suggestion`:** A non-blocking improvement that would make the PR clearer, + safer, or easier to maintain. + - **`nit`:** A tiny, non-blocking cleanup or style note. Use it only when the + author can safely ignore it without changing the review outcome. + - **`question`:** A real author-facing clarification about intent, scope, or + tradeoffs. Do not use questions to hide an issue that should be stated + directly. + - **`LGTM`:** "Looks good to me." Use only when the review found no blocking + issues, or when any remaining notes are clearly optional. +- **Decorations:** Optional labels in parentheses that clarify the finding type, + scope, or merge impact. + - **`security`:** Auth, authorization, ownership, secrets, SSRF, injection, + unsafe external input, or other trust-boundary concerns. + - **`test`:** Missing, failing, misleading, brittle, or insufficient tests. + - **`scope`:** PR scope, feature boundaries, unrelated churn, or work that + should be split into a separate issue or PR. + - **`ci`:** CI configuration, workflow failures, flaky checks, or validation + signal quality. + - **`api`:** Route, request/response, public function, schema, persistence, or + integration contract changes. + - **`docs`:** User-facing docs, contributor docs, examples, or comments that + need to change with the code. + - **`non-blocking`:** Useful feedback that should not prevent merge by + itself. +- **Finding fields:** + - **Problem:** What is wrong, what contract is ambiguous, or what risk the PR + introduces. + - **Impact:** Why the problem matters in practical terms. + - **Ask:** The smallest concrete fix, test, or decision requested from the PR + author. + - **Location:** The most useful repo-relative file and line reference for the + finding, using `path:line`. +- **Optional sections:** + - **Open Questions:** Genuine scope or intent questions; omit when there are + no real questions. + - **Validation:** What the reviewer ran, what was intentionally not run, and + what risk remains after review. + - **PR Hygiene:** Target-branch, template, CI/check, duplicate, related-work, + or superseding-PR notes. +- **`none confirmed`:** Use only when no review-worthy findings were confirmed; + still list validation gaps or residual risk when relevant. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 000000000..822229b35 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,57 @@ +## Summary + + + +## Target branch + +- [ ] This PR targets **`dev`**, not `main`. All PRs land in `dev`; `main` is curated by the maintainer at each release. If your PR is on `main` by accident, click "Edit" on this PR and change the base. + +## Linked Issue + + + +Fixes # + +## Type of Change + +- [ ] Bug fix (non-breaking — fixes a confirmed issue) +- [ ] New feature (non-breaking — adds new behaviour) +- [ ] Breaking change (changes or removes existing behaviour) +- [ ] Refactor / cleanup (behaviour unchanged) +- [ ] Documentation only +- [ ] CI / tooling / configuration + +## Checklist + +- [ ] I searched [open issues](https://github.com/odysseus-dev/odysseus/issues) and [open PRs](https://github.com/odysseus-dev/odysseus/pulls) — this is not a duplicate. +- [ ] This PR targets `dev` +- [ ] My changes are limited to the scope described above — no unrelated refactors or whitespace changes mixed in. +- [ ] I actually ran the app (`docker compose up` or `uvicorn app:app`) and verified the change works end-to-end. Type-checks and unit tests are not enough. + +## How to Test + + + +1. +2. +3. + +## Visual / UI changes — REQUIRED if you touched anything that renders + +**Anything that changes what the UI looks like — buttons, icons, padding, colors, fonts, spacing, layout, CSS, HTML, SVG, or any `static/js/` module that draws to the DOM — needs all of the following. PRs that change rendering without these WILL be closed.** + +- [ ] **Screenshot or short clip** of the change in the running app, attached below. Mobile screenshot too if the change affects mobile. +- [ ] **Style match**: the change uses Odysseus's existing visual language. Specifically: + - Reuse existing CSS variables (`--red`, `--fg`, `--bg`, `--card`, `--border`, etc.) — do not introduce new color values, font sizes, or spacing units. + - Reuse existing button/input/card/border classes. Don't invent parallel styling. + - **No Unicode emoji in UI or code.** Use inline SVG (matching the monochrome icon style already in `static/index.html`) or plain text. + - Monospaced font (`Fira Code`) for primary UI text. Don't override. + - Dark theme is the default; any light-mode work must be wired through the existing theme system, not hard-coded. +- [ ] **No new component patterns.** If a similar widget already exists in the app, extend it instead of writing a parallel one. +- [ ] **I am not an LLM agent submitting a bulk PR.** If you are, please open an issue describing the problem first — bulk auto-generated PRs that don't match the project's visual style are closed on sight, even when the underlying fix is correct. + +### Screenshots / clips + + diff --git a/.github/scripts/check-issue-description.js b/.github/scripts/check-issue-description.js new file mode 100644 index 000000000..a76ca29ab --- /dev/null +++ b/.github/scripts/check-issue-description.js @@ -0,0 +1,196 @@ +// @ts-check +'use strict'; + +/** @param {{ github: import('@octokit/rest').Octokit, context: import('@actions/github').context, core: import('@actions/core') }} */ +module.exports = async ({ github, context, core }) => { + const issue = context.payload.issue; + const body = (issue.body || '').trim(); + const labels = issue.labels.map(l => l.name); + const owner = context.repo.owner; + const repo = context.repo.repo; + + const isBug = labels.includes('bug'); + const isFeature = labels.includes('enhancement'); + + // Extract a Section's text, stripping HTML comments. Matches any heading + // depth (#, ##, ###, …) so a manually-written body isn't penalised for + // using a different number of hashes than the issue form generates. + function section(heading) { + const re = new RegExp(`#+\\s+${heading}\\s*([\\s\\S]*?)(?=\\n#+\\s+|$)`, 'i'); + const m = body.match(re); + return m ? m[1].replace(//g, '').trim() : ''; + } + + const failures = []; + + // ── Common: body must exist ─────────────────────────────────────────────── + if (body.length < 50) { + failures.push( + '**Description** — body is empty or too short. ' + + 'Please open the issue using one of the provided templates.', + ); + } + + // An issue is one or the other — never both. Resolve to a single type so the + // validation can't run two conflicting blocks at once. + const type = isBug && isFeature ? 'conflict' : isBug ? 'bug' : isFeature ? 'feature' : 'untyped'; + + switch (type) { + case 'conflict': + failures.push('**Labels** — an issue cannot be both `bug` and `enhancement`. Remove one label.'); + break; + + case 'bug': { + if (!section('Install Method')) { + failures.push('**Install Method** — select how you installed Odysseus'); + } + + if (!section('Operating System')) { + failures.push('**Operating System** — select your OS'); + } + + const stepsText = section('Steps to Reproduce'); + if (!stepsText || !/\d+\.|[-*]/.test(stepsText)) { + failures.push('**Steps to Reproduce** — must include at least one numbered or bulleted step'); + } + + if (section('Expected Behaviour').length < 10) { + failures.push('**Expected Behaviour** — section is empty or too short'); + } + + if (section('Actual Behaviour').length < 10) { + failures.push('**Actual Behaviour** — section is empty or too short'); + } + break; + } + + case 'feature': + if (!section('Area')) { + failures.push('**Area** — select which part of the application this affects'); + } + + if (section('Problem or Motivation').length < 20) { + failures.push( + '**Problem or Motivation** — section is empty or too short ' + + '(explain the concrete problem this solves)', + ); + } + + if (section('Proposed Solution').length < 20) { + failures.push( + '**Proposed Solution** — section is empty or too short ' + + '(describe the change you want to see)', + ); + } + + if (!section('Are you willing to implement this\\?')) { + failures.push('**Are you willing to implement this?** — select an option'); + } + break; + + // 'untyped' → only the common body-length check applies. + } + + // ── Unfilled dropdowns ──────────────────────────────────────────────────── + // #2068 added a "-- Please Select --" default to every template dropdown, so + // a contributor who never opens the dropdown submits with that literal string + // as the section value. The per-section checks above only verify presence, so + // a placeholder value passes. Scan every section and flag the ones still + // showing the placeholder, as a single comma-separated line item. + const PLACEHOLDER = '-- Please Select --'; + const headingRe = /^#+\s+(.+?)\s*$/gm; + const headings = []; + let headingMatch; + while ((headingMatch = headingRe.exec(body)) !== null) { + headings.push({ + name: headingMatch[1].trim(), + headStart: headingMatch.index, + contentStart: headingMatch.index + headingMatch[0].length, + }); + } + const unfilled = []; + for (let i = 0; i < headings.length; i++) { + const end = i + 1 < headings.length ? headings[i + 1].headStart : body.length; + if (body.slice(headings[i].contentStart, end).includes(PLACEHOLDER)) { + unfilled.push(headings[i].name); + } + } + if (unfilled.length > 0) { + failures.push( + `**Unfilled dropdowns** — please choose a value; these sections still show ` + + `the \`${PLACEHOLDER}\` placeholder: ${unfilled.join(', ')}.`, + ); + } + + // ── Labels ──────────────────────────────────────────────────────────────── + // These labels are expected to already exist in the repo — managing the + // repo's label set is the maintainer's job, not this workflow's. We check a + // label exists before applying it (issues.addLabels would otherwise silently + // create a missing label) and fail soft — warn and skip — if it's absent. + async function labelExists(name) { + try { + await github.rest.issues.getLabel({ owner, repo, name }); + return true; + } catch (e) { + if (e.status === 404) return false; + throw e; + } + } + + async function addLabel(name) { + if (await labelExists(name)) { + await github.rest.issues.addLabels({ owner, repo, issue_number: issue.number, labels: [name] }); + } else { + core.warning(`Label "${name}" does not exist in the repo — skipping. Create it once to enable labelling.`); + } + } + + async function dropLabel(name) { + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number: issue.number, name }); + } catch (e) { + if (e.status !== 404 && e.status !== 410) throw e; + } + } + + // ── Find existing bot comment to update in-place ────────────────────────── + const MARKER = ''; + const { data: comments } = await github.rest.issues.listComments({ + owner, repo, issue_number: issue.number, + }); + const existing = comments.find(c => c.user.type === 'Bot' && c.body.includes(MARKER)); + + const LABEL_BAD = 'needs more info'; + const LABEL_GOOD = 'ready for review'; + + if (failures.length === 0) { + if (existing) { + await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id }); + } + + await dropLabel(LABEL_BAD); + await addLabel(LABEL_GOOD); + + } else { + const list = failures.map(f => `- ${f}`).join('\n'); + const commentBody = [ + MARKER, + '⚠️ **Issue description is incomplete.** Please update the following sections:', + '', + list, + '', + '_This comment is deleted automatically once all sections are complete._', + ].join('\n'); + + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: commentBody }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number: issue.number, body: commentBody }); + } + + await dropLabel(LABEL_GOOD); + await addLabel(LABEL_BAD); + + core.setFailed(`Issue description has ${failures.length} issue(s) — see bot comment for details.`); + } +}; diff --git a/.github/scripts/check-pr-description.js b/.github/scripts/check-pr-description.js new file mode 100644 index 000000000..f5dabea5d --- /dev/null +++ b/.github/scripts/check-pr-description.js @@ -0,0 +1,130 @@ +// @ts-check +'use strict'; + +/** @param {{ github: import('@octokit/rest').Octokit, context: import('@actions/github').context, core: import('@actions/core') }} */ +module.exports = async ({ github, context, core }) => { + const body = context.payload.pull_request.body || ''; + const prNum = context.payload.pull_request.number; + const MARKER = ''; + const owner = context.repo.owner; + const repo = context.repo.repo; + + // Strip HTML comments so placeholder text does not count as content. + function strip(text) { + return (text ?? '').replace(//g, '').trim(); + } + + // Extract the text content of a Section. Matches any heading depth (#, ##, + // ###, …) so the check doesn't break if the template's heading level changes. + function section(heading) { + const m = body.match(new RegExp(`#+\\s+${heading}[\\s\\S]*?(?=\\n#+\\s+|$)`, 'i')); + return strip(m?.[0].replace(new RegExp(`#+\\s+${heading}`, 'i'), '') ?? ''); + } + + const problems = []; + + // 1. Summary must be filled in. + if (section('Summary').length < 20) { + problems.push('**Summary** is empty or too short — describe what changed and why.'); + } + + // 2. Linked Issue must reference a real issue. Accept a bare #NNN, a closing + // keyword + #NNN, or a full issue URL (e.g. .../issues/123) — the strict + // keyword-prefixed form previously false-flagged correctly-linked PRs. + const linkedSection = section('Linked Issue'); + const hasIssueRef = /#\d+\b/.test(linkedSection) || /\/issues\/\d+/.test(linkedSection); + if (!linkedSection || !hasIssueRef) { + problems.push('**Linked Issue** — add a reference like `Fixes #NNN`, a bare `#NNN`, or a link to the issue.'); + } + + // 3. At least one Type of Change box must be checked. + const typeBlock = body.match(/##\s+Type of Change[\s\S]*?(?=\n##\s|$)/i)?.[0] ?? ''; + if (!/- \[x\]/i.test(typeBlock)) { + problems.push('**Type of Change** — check at least one box.'); + } + + // 4. Duplicate-search checklist item must be checked. + if (!/- \[x\] I searched/i.test(body)) { + problems.push('**Checklist** — check the duplicate-search box to confirm you searched existing issues and PRs.'); + } + + // 5. How to Test must contain enough real detail for a reviewer to act on. + // Any format is fine — numbered steps, prose, the commands you ran, or a + // code block — so we only require non-trivial content, not a specific shape. + const howTo = section('How to Test'); + if (howTo.length < 30) { + problems.push('**How to Test** — explain how a reviewer can verify this change. Numbered steps, the commands you ran, or a short code block all work — give a sentence or two of real detail (not just "tested locally").'); + } + + // ── Comment ────────────────────────────────────────────────────────────── + const comments = await github.paginate(github.rest.issues.listComments, { + owner, repo, issue_number: prNum, per_page: 100, + }); + const existing = comments.find(c => (c.body ?? '').includes(MARKER)); + + if (problems.length === 0) { + if (existing) { + await github.rest.issues.deleteComment({ owner, repo, comment_id: existing.id }); + } + } else { + const commentBody = [ + MARKER, + '⚠️ **PR description — action needed**', + '', + 'The following required sections are missing or incomplete. Please update the PR description to address them:', + '', + problems.map(p => `- ${p}`).join('\n'), + '', + '---', + '_This comment is deleted automatically once all sections are complete._', + ].join('\n'); + + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body: commentBody }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number: prNum, body: commentBody }); + } + } + + // ── Labels ──────────────────────────────────────────────────────────────── + // These labels are expected to already exist in the repo — managing the + // repo's label set is the maintainer's job, not this workflow's. We check a + // label exists before applying it (issues.addLabels would otherwise silently + // create a missing label) and fail soft — warn and skip — if it's absent. + async function labelExists(name) { + try { + await github.rest.issues.getLabel({ owner, repo, name }); + return true; + } catch (e) { + if (e.status === 404) return false; + throw e; + } + } + + async function swapLabel(num, add, remove) { + if (await labelExists(add)) { + try { + await github.rest.issues.addLabels({ owner, repo, issue_number: num, labels: [add] }); + } catch (e) { + // Fail soft on a token that can't write labels so a label permission + // problem never masks the actual description verdict. + if (e.status !== 403) throw e; + core.warning(`Could not add "${add}" — token lacks label write here; skipping.`); + } + } else { + core.warning(`Label "${add}" does not exist in the repo — skipping. Create it once to enable labelling.`); + } + try { + await github.rest.issues.removeLabel({ owner, repo, issue_number: num, name: remove }); + } catch (e) { + if (e.status !== 404 && e.status !== 410 && e.status !== 403) throw e; + } + } + + if (problems.length === 0) { + await swapLabel(prNum, 'ready for review', 'needs work'); + } else { + await swapLabel(prNum, 'needs work', 'ready for review'); + core.setFailed(`PR description has ${problems.length} issue(s) — see bot comment for details.`); + } +}; diff --git a/.github/scripts/focused_test_guidance.py b/.github/scripts/focused_test_guidance.py new file mode 100644 index 000000000..1426d35fa --- /dev/null +++ b/.github/scripts/focused_test_guidance.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Report focused pytest guidance for changed paths under tests/.""" + +from __future__ import annotations + +import argparse +import os +import shlex +import subprocess +import sys +from collections.abc import Iterable +from pathlib import PurePosixPath + + +def parse_paths(raw_paths: bytes) -> list[str]: + """Decode the NUL-delimited output of ``git diff --name-only -z``.""" + return [os.fsdecode(path) for path in raw_paths.split(b"\0") if path] + + +def changed_paths_from_merge_base(base_sha: str, head_sha: str) -> list[str]: + """Return changed ``tests/`` paths using GitHub PR three-dot semantics. + + GitHub PR changed files are based on the merge base and the PR head, not a + direct endpoint diff between the current base branch tip and the PR head. + Using the direct endpoint diff can include files changed only on the base + branch when the PR branch is stale. + """ + merge_base = subprocess.check_output( + ["git", "merge-base", base_sha, head_sha], + stderr=subprocess.DEVNULL, + ).strip() + raw_paths = subprocess.check_output( + [ + "git", + "diff", + "--name-only", + "--diff-filter=ACMRT", + "-z", + os.fsdecode(merge_base), + head_sha, + "--", + "tests/", + ], + ) + return parse_paths(raw_paths) + + +def select_test_paths(paths: Iterable[str]) -> list[str]: + """Return unique, repository-relative paths contained by tests/.""" + selected: set[str] = set() + for raw_path in paths: + path = PurePosixPath(raw_path) + if path.is_absolute() or ".." in path.parts: + continue + parts = tuple(part for part in path.parts if part != ".") + if len(parts) >= 2 and parts[0] == "tests": + selected.add(PurePosixPath(*parts).as_posix()) + return sorted(selected) + + +def is_pytest_file(path: str) -> bool: + """Return whether a changed path follows this repository's pytest naming.""" + name = PurePosixPath(path).name + return name.endswith(".py") and ( + name.startswith("test_") or name.endswith("_test.py") + ) + + +def pytest_command(paths: Iterable[str]) -> str: + """Build a copyable pytest command for changed runnable test files.""" + command = ["python3", "-m", "pytest", "-q", *paths] + return shlex.join(command) + + +def format_report(paths: Iterable[str]) -> str: + """Format focused guidance for CI logs and the workflow summary.""" + changed_paths = select_test_paths(paths) + runnable_paths = [path for path in changed_paths if is_pytest_file(path)] + lines = ["## Focused test guidance (report-only)", ""] + if not changed_paths: + lines.append("No changed paths under `tests/`.") + else: + lines.extend(["Changed paths under `tests/`:", ""]) + lines.extend(f"- `{path}`" for path in changed_paths) + lines.extend(["", "Suggested focused validation:", ""]) + if runnable_paths: + lines.append(f"```sh\n{pytest_command(runnable_paths)}\n```") + else: + lines.append("No directly runnable pytest files changed.") + lines.extend( + [ + "", + "This guidance does not infer tests from source changes. " + "Existing blocking CI remains the source of truth.", + ] + ) + return "\n".join(lines) + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Report focused pytest guidance for changed tests/ paths.", + ) + parser.add_argument("--base-sha", help="Pull request base commit SHA.") + parser.add_argument("--head-sha", help="Pull request head commit SHA.") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(sys.argv[1:] if argv is None else argv) + if bool(args.base_sha) != bool(args.head_sha): + raise SystemExit("--base-sha and --head-sha must be provided together") + + if args.base_sha and args.head_sha: + paths = changed_paths_from_merge_base(args.base_sha, args.head_sha) + else: + paths = parse_paths(sys.stdin.buffer.read()) + + print(format_report(paths)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..f7d3659e8 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,148 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + +# Least privilege: none of the jobs write to the repo. +permissions: + contents: read + +# Cancel superseded runs on the same ref to save Actions minutes. +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + focused-test-guidance: + name: Focused test guidance (report-only) + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + continue-on-error: true + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + - name: Report changed test paths + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + report_file="$RUNNER_TEMP/focused-test-guidance.md" + publish_report() { + cat "$report_file" + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "$report_file" >> "$GITHUB_STEP_SUMMARY" || true + fi + return 0 + } + + report_unavailable() { + { + printf '%s\n\n' '## Focused test guidance unavailable (report-only)' + printf '%s\n\n' "$1" + printf '%s\n' 'Existing blocking CI remains the source of truth.' + } > "$report_file" + publish_report + exit 0 + } + + if [ -z "$BASE_SHA" ] || [ -z "$HEAD_SHA" ]; then + report_unavailable "Pull request base/head metadata is missing." + fi + + if ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then + report_unavailable "The pull request base commit is unavailable locally." + fi + + if ! git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then + report_unavailable "The pull request head commit is unavailable locally." + fi + + if ! python3 .github/scripts/focused_test_guidance.py \ + --base-sha "$BASE_SHA" \ + --head-sha "$HEAD_SHA" > "$report_file"; then + report_unavailable "The focused test guidance helper could not produce a report." + fi + + publish_report + + python-syntax: + name: Python syntax (compileall) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.11" + # Byte-compile sources — catches syntax errors without installing deps. + - run: python -m compileall -q app.py core routes src services scripts tests + + node-syntax: + name: JS syntax (node --check) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "20" + # Syntax-check our own JS (skip vendored libs in static/lib). + - name: node --check + run: | + shopt -s globstar nullglob + for f in static/app.js static/js/**/*.js; do + node --check "$f" + done + + python-tests: + name: Python tests (pytest) + runs-on: ubuntu-latest + # Informational for now: the suite has known flaky / environment-dependent + # failures (test isolation + embedding-model assertions). Tracked under the + # ROADMAP "fresh install smoke tests" item; make this required once green. + continue-on-error: true + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + fetch-depth: 0 + persist-credentials: false + + # Detect whether this PR only touches documentation files. + # If so, skip the expensive pytest run while still reporting a passing check. + - name: Check for docs-only changes + id: docs-check + run: | + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + else + BASE="${{ github.event.before }}" + HEAD="${{ github.sha }}" + fi + # List all changed files; if every file matches docs/markdown patterns, skip pytest. + changed=$(git diff --name-only "$BASE" "$HEAD" 2>/dev/null || git diff --name-only HEAD~1 HEAD) + non_docs=$(echo "$changed" | grep -Ev '^(docs/|.*\.md$|\.github/[^/]+\.md$)' || true) + if [ -z "$non_docs" ]; then + echo "docs_only=true" >> "$GITHUB_OUTPUT" + echo "Docs-only change detected — skipping pytest." + else + echo "docs_only=false" >> "$GITHUB_OUTPUT" + fi + + - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + if: steps.docs-check.outputs.docs_only != 'true' + with: + python-version: "3.11" + cache: pip + - run: pip install -r requirements.txt + if: steps.docs-check.outputs.docs_only != 'true' + - run: mkdir -p data # sqlite DB lives at ./data/app.db + if: steps.docs-check.outputs.docs_only != 'true' + - run: python -m pytest -q + if: steps.docs-check.outputs.docs_only != 'true' diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..bb8a8c53e --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,41 @@ +name: CodeQL + +# Advanced setup so CodeQL also runs on pull requests (including from forks), +# surfacing findings before merge instead of only after a change lands on dev. +on: + push: + branches: [dev, main] + pull_request: + branches: [dev] + schedule: + - cron: "17 3 * * 1" + +permissions: + contents: read + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-latest + permissions: + security-events: write + actions: read + contents: read + strategy: + fail-fast: false + matrix: + language: [actions, javascript-typescript, python] + steps: + - name: Checkout + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Initialize CodeQL + uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 + with: + languages: ${{ matrix.language }} + build-mode: none + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/container-scan.yml b/.github/workflows/container-scan.yml new file mode 100644 index 000000000..f1c4b5bfd --- /dev/null +++ b/.github/workflows/container-scan.yml @@ -0,0 +1,52 @@ +# Container security: Dockerfile lint +# +# Purpose: the Docker image is how most people run Odysseus, so it is part of +# the attack surface. hadolint lints the Dockerfile for mistakes and insecure +# patterns (running as root longer than needed, unpinned base image, bad apt +# usage). Blocking. +# +# The image vulnerability scan (Trivy, advisory) lives in its own file, +# container-trivy.yml. Keeping it separate lets that advisory scan be +# path-filtered and held to a read-only token on pull requests without +# weakening this blocking gate, which must always report so a required check +# never hangs. +# +# Note: a separate open PR (#120) proposes a local `scripts/scan_image.py`. +# This job is complementary -- it is a CI gate, not a script a contributor has +# to remember to run. + +name: Container scan + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +permissions: {} + +concurrency: + group: container-scan-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + hadolint: + name: hadolint (Dockerfile lint) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Lint Dockerfile + uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0 + with: + dockerfile: Dockerfile + # DL3008: pinning apt package versions is impractical on a -slim base + # image. Debian purges old package versions from its repos, so a + # pinned version breaks future rebuilds. The base image itself is + # what should be pinned (tracked by Dependabot's docker ecosystem). + ignore: DL3008 diff --git a/.github/workflows/container-trivy.yml b/.github/workflows/container-trivy.yml new file mode 100644 index 000000000..2a482f067 --- /dev/null +++ b/.github/workflows/container-trivy.yml @@ -0,0 +1,125 @@ +# Container image vulnerability scan (advisory) +# +# Trivy builds the application image and scans it for known-vulnerable OS and +# Python packages. Advisory only -- it reports findings to the repo's Security +# tab without blocking a merge, because the image inevitably contains +# already-known CVEs in upstream packages that are not this project's bug. +# +# Split from the Dockerfile lint (container-scan.yml) for two reasons: +# +# - Least privilege. The image build runs Dockerfile instructions, which on a +# pull request are attacker-influenceable. That path (the `scan` job) is +# held to a read-only token and never publishes results. Only `publish`, +# which runs on push to main (curated, fast-forwarded from reviewed dev), +# gets security-events:write to upload SARIF. +# - Cost. Docs-only changes do not rebuild the image (paths-ignore below), +# matching docker-publish.yml. hadolint stays on the broad trigger in +# container-scan.yml so the blocking gate always reports. + +name: Container scan (Trivy) + +on: + pull_request: + paths-ignore: + - '**.md' + - 'docs/**' + - '.github/ISSUE_TEMPLATE/**' + push: + branches: [main] + paths-ignore: + - '**.md' + - 'docs/**' + - '.github/ISSUE_TEMPLATE/**' + workflow_dispatch: + +permissions: {} + +concurrency: + group: container-trivy-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + # Pull requests and manual runs: build and scan under a read-only token. + # The build executes PR-supplied Dockerfile instructions, so this job must + # not hold any write scope, and it does not upload to the Security tab. + scan: + name: Trivy (image scan, advisory) + if: github.event_name != 'push' + runs-on: ubuntu-latest + # Advisory: a CVE in an upstream package must not block a PR. + continue-on-error: true + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + # Build without pushing so a broken Dockerfile is caught here, and the + # exact image we ship is what gets scanned. + - name: Build image + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + push: false + load: true + tags: odysseus:ci + + - name: Scan image with Trivy + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: odysseus:ci + format: table + ignore-unfixed: true + env: + # Pin the vuln DB source to GHCR to avoid rate-limited Docker Hub + # mirrors that flake on shared runners. + TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2 + + # Push to main only: build, scan, and publish SARIF to the Security tab. + # This is the only path that runs trusted code, so it is the only one granted + # security-events:write. + publish: + name: Trivy (image scan + SARIF upload) + if: github.event_name == 'push' + runs-on: ubuntu-latest + continue-on-error: true + permissions: + contents: read + security-events: write # upload SARIF to the Security tab + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + + - name: Build image + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + push: false + load: true + tags: odysseus:ci + + - name: Scan image with Trivy + uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0 + with: + image-ref: odysseus:ci + format: sarif + output: trivy-results.sarif + ignore-unfixed: true + env: + TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2 + + - name: Upload Trivy results + uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 + with: + sarif_file: trivy-results.sarif + category: trivy-image diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml new file mode 100644 index 000000000..0a587de19 --- /dev/null +++ b/.github/workflows/dependency-review.yml @@ -0,0 +1,71 @@ +# Supply-chain review +# +# Purpose: defend against "side-chain" / supply-chain attacks -- a pull request +# that adds (or bumps) a dependency to a version with a known vulnerability or a +# disallowed license. Two layers: +# +# - dependency-review: runs ONLY on pull requests. It compares the +# dependencies before and after the PR and blocks the merge if the change +# pulls in a package with a known security advisory. This is the gate. +# - pip-audit: scans the project's current Python requirements against the +# advisory database. Advisory only (it never blocks a merge), because it can +# flag a pre-existing issue in an already-shipped dependency. + +name: Dependency review + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +# Default-deny token; jobs grant only read access. +permissions: {} + +concurrency: + group: dependency-review-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + dependency-review: + name: dependency-review (PR gate) + # Only meaningful on a pull request -- it needs a base..head diff to review. + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Review dependency changes + uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0 + with: + # Fail the PR on any newly introduced moderate-or-worse advisory. + fail-on-severity: moderate + + pip-audit: + name: pip-audit (advisory) + runs-on: ubuntu-latest + # Advisory: report known-vulnerable Python deps without blocking the merge. + continue-on-error: true + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + - name: Run pip-audit on requirements + run: | + set -euo pipefail + pip install pip-audit==2.10.0 + pip-audit -r requirements.txt -r requirements-optional.txt --strict diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml new file mode 100644 index 000000000..d52c0c4e8 --- /dev/null +++ b/.github/workflows/docker-publish.yml @@ -0,0 +1,140 @@ +name: ci / docker publish + +# Build the Odysseus image and publish to GHCR. +# push to main -> :latest, :X.Y.Z (curated release; main is fast-forwarded at releases) +# push to dev -> :dev, :X.Y.Z-dev. (rolling dev + an immutable, traceable pin) +# Multi-arch (linux/amd64 + linux/arm64): each arch builds on its own native +# runner and pushes by digest, then a merge job stitches the digests into one +# manifest list and applies the tags (faster + cleaner than QEMU emulation). +# Registry: ghcr.io//. + +on: + push: + branches: [dev, main] + paths-ignore: + - '**.md' + - 'docs/**' + - '.github/ISSUE_TEMPLATE/**' + +concurrency: + group: docker-publish-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build: + name: build (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - platform: linux/amd64 + arch: amd64 + runner: ubuntu-latest + - platform: linux/arm64 + arch: arm64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Set up Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + - name: Log in to GHCR + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push by digest + id: build + uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0 + with: + context: . + platforms: ${{ matrix.platform }} + outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true + cache-from: type=gha,scope=${{ matrix.arch }} + cache-to: type=gha,mode=max,scope=${{ matrix.arch }} + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + - name: Upload digest + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: digest-${{ matrix.arch }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge: + name: merge manifest + tag + runs-on: ubuntu-latest + needs: build + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + - name: Read APP_VERSION + short sha + id: ver + run: | + v=$(grep -E '^APP_VERSION' src/constants.py | head -1 | sed -E 's/.*"([^"]+)".*/\1/') + [ -n "$v" ] || { echo "APP_VERSION not found"; exit 1; } + echo "version=$v" >> "$GITHUB_OUTPUT" + echo "short=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT" + - name: Download digests + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + path: /tmp/digests + pattern: digest-* + merge-multiple: true + - name: Set up Buildx + uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0 + - name: Log in to GHCR + uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Compute tags + id: meta + uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=${{ steps.ver.outputs.version }},enable=${{ github.ref == 'refs/heads/main' }} + type=raw,value=dev,enable=${{ github.ref == 'refs/heads/dev' }} + type=raw,value=${{ steps.ver.outputs.version }}-dev.${{ steps.ver.outputs.short }},enable=${{ github.ref == 'refs/heads/dev' }} + - name: Create manifest list + push tags + working-directory: /tmp/digests + run: | + tags=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") + digests=$(printf "${REGISTRY}/${IMAGE_NAME}@sha256:%s " *) + # word-splitting is intended: $tags and $digests each expand to multiple args + # shellcheck disable=SC2086 + docker buildx imagetools create $tags $digests + env: + REGISTRY: ${{ env.REGISTRY }} + IMAGE_NAME: ${{ env.IMAGE_NAME }} + - name: Inspect + run: | + if [ "$GITHUB_REF" = "refs/heads/main" ]; then ref=latest; else ref=dev; fi + docker buildx imagetools inspect "${REGISTRY}/${IMAGE_NAME}:${ref}" + env: + REGISTRY: ${{ env.REGISTRY }} + IMAGE_NAME: ${{ env.IMAGE_NAME }} diff --git a/.github/workflows/issue-description-check.yml b/.github/workflows/issue-description-check.yml new file mode 100644 index 000000000..52e9dddae --- /dev/null +++ b/.github/workflows/issue-description-check.yml @@ -0,0 +1,24 @@ +name: ci / issue description check + +on: + issues: + types: [opened, edited, reopened] + +permissions: + issues: write + +jobs: + check: + name: Check issue description + runs-on: ubuntu-latest + # Skip bots (Dependabot, release-drafter, etc.) + if: ${{ github.event.issue.user.type != 'Bot' }} + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: return require('./.github/scripts/check-issue-description.js')({github, context, core}) diff --git a/.github/workflows/pr-description-check.yml b/.github/workflows/pr-description-check.yml new file mode 100644 index 000000000..53f0b5f50 --- /dev/null +++ b/.github/workflows/pr-description-check.yml @@ -0,0 +1,109 @@ +name: ci / PR checks + +on: + # pull_request_target runs in the base-repo context (has secrets) so the check + # works on fork PRs. Safe here: the checkout pins to the base branch (no fork + # code runs) and the scripts only read context.payload and call the GitHub API. + pull_request_target: # zizmor: ignore[dangerous-triggers] + types: [opened, edited, synchronize, reopened, ready_for_review] + +# Default-deny at the workflow level; each job opts into only the scopes it needs. +# Note: modifying a PR's labels/comments needs pull-requests:write even though the +# REST path is under /issues/{n}/...; issues:write alone returns 403 on PRs. +permissions: {} + +jobs: + check-description: + name: Check PR description + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + issues: write + # Skip bots: they open PRs programmatically and have their own process. + if: github.event.pull_request.user.type != 'Bot' + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ github.base_ref }} + sparse-checkout: .github/scripts + persist-credentials: false + + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: return require('./.github/scripts/check-pr-description.js')({github, context, core}) + + check-title: + name: Check PR title (Conventional Commits) + runs-on: ubuntu-latest + permissions: {} + # Skip bots: they open PRs programmatically and have their own process. + if: github.event.pull_request.user.type != 'Bot' + steps: + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const title = context.payload.pull_request.title || ""; + // Conventional Commits: type(optional-scope)(optional !): summary + const re = /^(feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert)(\([\w .\/-]+\))?!?: .+/; + if (!re.test(title)) { + core.setFailed( + `PR title is not in Conventional Commits format:\n "${title}"\n\n` + + `Expected: type(scope): summary\n` + + `Example: fix(search): handle empty query\n` + + `Types: feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert.` + ); + } else { + core.info(`PR title OK: ${title}`); + } + + check-mergeable: + name: Flag unmergeable PRs + runs-on: ubuntu-latest + permissions: + pull-requests: write + issues: write + # Skip bots: they open PRs programmatically and have their own process. + if: github.event.pull_request.user.type != 'Bot' + steps: + - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 + with: + script: | + const repo = { owner: context.repo.owner, repo: context.repo.repo }; + const number = context.payload.pull_request.number; + const READY = "ready for review"; + const CONFLICT = "merge conflict"; + + // Ensure the conflict label exists (red). Ignore if already present. + try { + await github.rest.issues.getLabel({ ...repo, name: CONFLICT }); + } catch { + await github.rest.issues.createLabel({ + ...repo, name: CONFLICT, color: "B60205", + description: "Conflicts with the base branch; needs a rebase before review.", + }).catch(() => {}); + } + + // mergeable is computed asynchronously and is often null right after + // an event, so poll a few times until GitHub has resolved it. + let pr = null; + for (let i = 0; i < 5; i++) { + const { data } = await github.rest.pulls.get({ ...repo, pull_number: number }); + if (data.mergeable !== null) { pr = data; break; } + await new Promise(r => setTimeout(r, 3000)); + } + if (!pr || pr.draft) return; + const labels = pr.labels.map(l => l.name); + + if (pr.mergeable === false) { + if (labels.includes(READY)) { + await github.rest.issues.removeLabel({ ...repo, issue_number: number, name: READY }).catch(() => {}); + } + if (!labels.includes(CONFLICT)) { + await github.rest.issues.addLabels({ ...repo, issue_number: number, labels: [CONFLICT] }); + } + } else if (pr.mergeable === true) { + if (labels.includes(CONFLICT)) { + await github.rest.issues.removeLabel({ ...repo, issue_number: number, name: CONFLICT }).catch(() => {}); + } + } diff --git a/.github/workflows/secret-scan.yml b/.github/workflows/secret-scan.yml new file mode 100644 index 000000000..02512204a --- /dev/null +++ b/.github/workflows/secret-scan.yml @@ -0,0 +1,60 @@ +# Secret scanning +# +# Purpose: stop credentials (API keys, tokens, passwords, private keys) from +# ever living in the Git history. Odysseus deliberately keeps real secrets in +# files that are gitignored (.env, data/), but a slip in a future commit -- or a +# malicious pull request that sneaks one in -- would otherwise go unnoticed. +# This job reads the repository and the full commit history and fails if it +# finds anything that looks like a secret. +# +# It runs the official gitleaks BINARY directly (pinned to an exact version and +# verified against the project's published SHA-256 checksum) rather than the +# gitleaks GitHub Action, because the Action asks for a paid license on +# organization-owned repos. The binary is free and behaves identically. + +name: Secret scan + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +# Start with zero permissions; the single job opts back in to read-only. +permissions: {} + +concurrency: + group: secret-scan-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + gitleaks: + name: gitleaks + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + # Full history so a secret committed in an earlier commit (and later + # deleted) is still caught -- deletion does not remove it from Git. + fetch-depth: 0 + persist-credentials: false + + # Pinned version + checksum so a tampered release binary cannot run here. + # Bump VERSION/SHA256 together; the checksum comes from the matching + # gitleaks__checksums.txt on the GitHub release. + - name: Run gitleaks (pinned, checksum-verified) + env: + GITLEAKS_VERSION: 8.30.1 + GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb + run: | + set -euo pipefail + TARBALL="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz" + curl -fsSL -o "${TARBALL}" \ + "https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/${TARBALL}" + echo "${GITLEAKS_SHA256} ${TARBALL}" | sha256sum -c - + tar -xzf "${TARBALL}" gitleaks + # Scan the whole history. Findings print to the log and fail the job. + ./gitleaks git --no-banner --redact --verbose . diff --git a/.github/workflows/workflow-security.yml b/.github/workflows/workflow-security.yml new file mode 100644 index 000000000..ee345333b --- /dev/null +++ b/.github/workflows/workflow-security.yml @@ -0,0 +1,80 @@ +# Workflow security (CI that audits the CI) +# +# Purpose: the GitHub Actions workflows themselves are an attack surface. A +# poorly written workflow can leak the repository token, run attacker-supplied +# code from a pull request, or pull in a tampered third-party action. These two +# tools check every workflow file in this repo for those mistakes: +# +# - actionlint: catches workflow syntax errors and shell-script bugs inside +# `run:` steps before they reach main. +# - zizmor: a security linter for Actions. Flags template-injection holes, +# unpinned actions, credential persistence, and over-broad token +# permissions -- exactly the patterns the rest of this CI is built to avoid. +# +# Add this early: it then audits every workflow added after it. + +name: Workflow security + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +# Default-deny token; each job grants only read access to the code. +permissions: {} + +concurrency: + group: workflow-security-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + actionlint: + name: actionlint + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + # Pinned version + checksum so a tampered binary cannot run here. + - name: Run actionlint (pinned, checksum-verified) + env: + ACTIONLINT_VERSION: 1.7.12 + ACTIONLINT_SHA256: 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8 + run: | + set -euo pipefail + TARBALL="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" + curl -fsSL -o "${TARBALL}" \ + "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/${TARBALL}" + echo "${ACTIONLINT_SHA256} ${TARBALL}" | sha256sum -c - + tar -xzf "${TARBALL}" actionlint + ./actionlint -color + + zizmor: + name: zizmor (Actions SAST) + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: '3.12' + + # Pinned zizmor release. --offline keeps the audit hermetic (no network + # calls about the actions it inspects); --min-severity=low surfaces + # everything so nothing slips through under the gate. + - name: Run zizmor + run: | + set -euo pipefail + pip install zizmor==1.25.2 + zizmor --offline --min-severity=low .github/workflows/ diff --git a/.gitignore b/.gitignore index 33499820c..77c364b8f 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,17 @@ venv/ # Environment .env +.env.bak.* !.env.example +# Local uv lockfile (optional, per-platform — see "Faster installs with uv" in README) +requirements.lock + +# SOPS workflow — encrypted `secrets.env` is intentionally committable, +# but every variant (plaintext, manual decrypt copy, editor backup) +# must stay out of git. Mirrored in .dockerignore so the same artifacts +# also cannot enter image build layers. +secrets.env.* +!secrets.env.example # Data — all user data stays local data/ @@ -60,12 +70,20 @@ output.txt.txt *.tiff *.pdf +# …except shipped static assets +!static/icons/*.png + # …except shipped demo assets in docs/ that the README links to. !docs/*.jpg !docs/*.jpeg !docs/*.png !docs/*.gif !docs/*.webp +# …and curated docs/ subfolder assets (e.g. accessibility before/after shots). +!docs/**/*.png +!docs/**/*.jpg +!docs/**/*.gif +!docs/**/*.webp # Reports and temp files reports/ @@ -76,8 +94,11 @@ research_data/ # Internal dev/review notes — not for public repo dev-docs/ +# Windows-port working docs (local only, not for public repo) +docs/windows-port/ # Local config compound.config.json *.error.log _scratch/ +/odysseus/ diff --git a/ACKNOWLEDGMENTS.md b/ACKNOWLEDGMENTS.md index c4079e6e5..94092c6ca 100644 --- a/ACKNOWLEDGMENTS.md +++ b/ACKNOWLEDGMENTS.md @@ -33,8 +33,8 @@ The full license texts are kept in [`licenses/`](licenses/). - **[Tongyi DeepResearch](https://github.com/Alibaba-NLP/DeepResearch)** by **Alibaba-NLP / Tongyi Lab** — the multi-step deep-research agent pipeline. Copyright © Alibaba-NLP / Tongyi Lab. **Apache-2.0.** Adapted for Odysseus's - Deep Research feature (`api/research_*.py`, `routes/research_routes.py`, - `services/search/`). Full text in + Deep Research feature (`services/research/`, `src/research_handler.py`, + `routes/research_routes.py`, `services/search/`). Full text in [`licenses/DeepResearch-Apache-2.0.txt`](licenses/DeepResearch-Apache-2.0.txt). --- @@ -47,7 +47,7 @@ just composed. | Service | Image | Purpose | License | |---|---|---|---| -| [SearXNG](https://github.com/searxng/searxng) | `searxng/searxng:latest` | Default metasearch backend | AGPL-3.0 | +| [SearXNG](https://github.com/searxng/searxng) | `searxng/searxng:2026.5.31-7159b8aed` (pinned tag; see compose) | Default metasearch backend | AGPL-3.0 | | [ChromaDB](https://github.com/chroma-core/chroma) | `chromadb/chroma:latest` | Vector store for memory / RAG | Apache-2.0 | | [ntfy](https://github.com/binwiederhier/ntfy) | `binwiederhier/ntfy` | Push notifications (self-hosted reminders) | Apache-2.0 / GPL-2.0 | @@ -86,6 +86,7 @@ Bundled in `static/fonts/`: | [Fira Code](https://github.com/tonsky/FiraCode) | SIL Open Font License 1.1 | Nikita Prokopov & contributors | | [Inter](https://github.com/rsms/inter) | SIL Open Font License 1.1 | Rasmus Andersson | | [GohuFont](https://font.gohu.org/) (`fonts/custom/GohuFont.ttf`) | WTFPL | Hugo Chargois | +| [OpenDyslexic](https://opendyslexic.org/) (`fonts/OpenDyslexic-{Regular,Bold}.woff2`) | SIL Open Font License 1.1 ([`licenses/OpenDyslexic-OFL.txt`](licenses/OpenDyslexic-OFL.txt)) | Abbie Gonzalez | ## Python dependencies @@ -118,6 +119,7 @@ Core (`requirements.txt`) and optional (`requirements-optional.txt`): | croniter | MIT | | pytest / pytest-asyncio | MIT / Apache-2.0 | | duckduckgo-search (optional) | MIT | +| markitdown (optional — Office/EPUB text extraction) | MIT | | **PyMuPDF** *(optional — form-filling only)* | **AGPL-3.0** — see note below | ## Companion services (interoperated with, not bundled) @@ -152,6 +154,9 @@ concerns from earlier are resolved: deployment (Artifex also sells a commercial PyMuPDF license that lifts this). - **`caldav`** (Python lib) is **dual-licensed GPL-3.0-or-later OR Apache-2.0**. Odysseus uses it under **Apache-2.0**, which is permissive and MIT-compatible. +- **`markitdown`** (Microsoft) is **MIT** and used only as an *optional* dependency for Office/EPUB text + extraction (`src/markitdown_runtime.py`), lazy-imported with graceful fallback — the MIT core runs without + it. The cloud `az-doc-intel` extra is deliberately **not** installed, keeping extraction fully local. --- diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 01ed77b71..38586845f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,17 @@ Thanks for helping. The project is moving quickly, so the best contributions are focused, easy to review, and easy to test. +## Branch model + +Odysseus has two branches: + +- **`dev`** — where all PRs land. Things can be in flux here; the merge button gets used freely. +- **`main`** — what users run. Curated and tested by the maintainer. Fast-forwarded to a stable `dev` commit at each release. + +**Open your PR against `dev`, not `main`.** The GitHub "base" dropdown defaults to `dev`. If you opened a PR against `main` by accident, click "Edit" on the PR and change the base — no rebase needed. + +End-users cloning the repo will land on `dev` by default. To run the curated/stable version: `git checkout main` after clone. + ## Before You Start - Search existing issues and pull requests before opening a new one. @@ -14,7 +25,7 @@ Thanks for helping. The project is moving quickly, so the best contributions are Docker is the recommended path for normal testing: ```bash -git clone https://github.com/pewdiepie-archdaemon/odysseus.git +git clone https://github.com/odysseus-dev/odysseus.git cd odysseus cp .env.example .env docker compose up -d --build @@ -26,7 +37,7 @@ Manual development uses Python 3.11+: python3 -m venv venv source venv/bin/activate pip install -r requirements.txt -python -m uvicorn app:app --host 0.0.0.0 --port 7000 +python -m uvicorn app:app --host 127.0.0.1 --port 7000 ``` Windows is not actively tested. Docker on Linux or a Linux/macOS manual install is the safer path for now. @@ -57,12 +68,44 @@ Good pull requests usually include: - A short explanation of the bug or feature. - The files or areas changed. -- Manual test steps or automated test results. +- Manual test steps or automated test results from running the actual app, not just the test suite. - Screenshots or short recordings for UI changes. - Links to related issues, for example `Fixes #123`. Please keep PRs small. Large PRs that mix unrelated cleanup, formatting, refactors, and behavior changes are much harder to review. +> **Auto-generated PRs.** If you are running an LLM agent (Devin, Cursor, OpenHands, Claude Code, etc.) against this repo: please open an issue describing the problem first instead of opening a PR directly. Bulk agent-generated PRs that don't match the project's visual style or contribution format will be closed without review, even when the underlying fix is correct. + +## Style and visual changes + +Odysseus has an intentional visual style. PRs that ignore it will be closed without merge, no matter how correct the underlying code is. + +Before submitting any change that affects what the app looks like — buttons, icons, fonts, colors, spacing, layout, CSS, HTML, SVG, or any `static/js/` module that draws to the DOM — please: + +1. **Run the app locally** and view the change in a browser. Type-checks and unit tests are not enough. +2. **Attach a screenshot or short clip** of the change in the running app. Add a mobile screenshot too if the change affects mobile. +3. **Match the existing visual language.** Specifically: + - Reuse existing CSS variables (`--red`, `--fg`, `--bg`, `--card`, `--border`, …). Do not introduce new color values, font sizes, or spacing units. + - Reuse existing button, input, card, and border classes. Don't invent parallel styling for similar widgets. + - **No Unicode emoji in UI or code.** Use inline SVG (matching the monochrome icon style already in `static/index.html`) or plain text. + - Monospaced font (`Fira Code`) for primary UI text. Don't override. + - Dark theme is the default; any light-mode work goes through the existing theme system, not hard-coded. +4. **Don't add parallel components.** If a similar widget already exists in the app, extend it instead of writing a new one. + +If you are unsure whether a change is "visual," it is. Default to attaching a screenshot. + +## Code conventions + +Don't hardcode values that the project already exposes through a constant or a helper. Hardcoded literals drift out of sync, break on non-default deployments, and reintroduce bugs we've already fixed. + +- **Filesystem paths:** never build writable paths from `Path(__file__)...` into the source tree, hardcode `/app/...`, or use a relative `"data/..."` string. Every persisted file and directory has a named constant in `src/constants.py` (for example `AUTH_FILE`, `USER_PREFS_FILE`, `SETTINGS_FILE`, `TTS_CACHE_DIR`, `CHROMA_DIR`). Import and use that named constant; do not re-derive the path locally with `os.path.join(DATA_DIR, "x.json")` or `DATA_DIR / "x.json"`. `DATA_DIR` is the single place that reads `ODYSSEUS_DATA_DIR`, so use it directly only for dynamic paths that have no fixed name (for example per-owner files). If a data file or directory has no constant yet, add one to `src/constants.py`. The source tree is read-only in Docker and `/app/...` does not exist on native runs; guard directory creation so an unwritable path degrades gracefully instead of crashing at import. +- **Internal API / loopback URLs:** don't hardcode `http://localhost:7000`. Use `internal_api_base()` from `src.constants` (it honors `ODYSSEUS_INTERNAL_BASE` / `APP_PORT`). +- **Ports, limits, model lists, and similar:** reuse the existing constant if one exists; if it doesn't and the value is used in more than one place, add a constant rather than copying the literal. + +If you need a value that has no constant or helper yet, add it to `src/constants.py` (the single source of truth for paths and config; `core/constants.py` only re-exports it for backward compatibility) and import it, rather than repeating a literal across files. + +**Commits:** use [Conventional Commits](https://www.conventionalcommits.org), `type(scope): summary` (e.g. `fix(search): ...`, `feat(notes): ...`, `docs(contributing): ...`). Common types: `fix`, `feat`, `refactor`, `docs`, `test`, `chore`, `ci`. Keep the subject short and imperative; put the "why" in the body when it isn't obvious. + ## Issue Reports For bugs, include: diff --git a/Dockerfile b/Dockerfile index ab0829122..4a11027fe 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,15 @@ -FROM python:3.12-slim +# ---- builder: patch + build wheels for Real-ESRGAN's broken-on-3.14 deps ---- +# basicsr/gfpgan/facexlib read their version via exec()+locals()['__version__'], +# which raises KeyError on Python 3.13+ (PEP 667). Build patched wheels here so +# the final image / Cookbook never has to compile the broken sdists. See +# docker/build-realesrgan-wheels.sh for the full rationale. +FROM python:3.14-slim AS realesrgan-wheels +RUN apt-get update && apt-get install -y --no-install-recommends curl \ + && rm -rf /var/lib/apt/lists/* +COPY docker/build-realesrgan-wheels.sh /usr/local/bin/build-realesrgan-wheels.sh +RUN bash /usr/local/bin/build-realesrgan-wheels.sh /wheels + +FROM python:3.14-slim # System deps. tmux is required by Cookbook for background downloads/serves. # openssh-client is required for Cookbook remote server tests, setup, probes, @@ -18,19 +29,72 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ tmux \ openssh-client \ gosu \ + libgl1 \ + libglib2.0-0t64 \ + libxcb1 \ + libmagic1 \ && rm -rf /var/lib/apt/lists/* +# libgl1/libglib2.0-0t64/libxcb1 are runtime shared libs (libGL.so.1, +# libglib-2.0/libgthread, libxcb.so.1) that opencv-python (cv2) loads. The +# slim base omits them, so the Cookbook "install realesrgan" path imports cv2 +# and dies with `libxcb.so.1: cannot open shared object file` despite a clean +# pip install. Using full opencv-python (not -headless) because basicsr/gfpgan/ +# facexlib/realesrgan all depend on the `opencv-python` distribution by name. +# +# libmagic1 is the shared lib (libmagic.so.1) that python-magic dlopens for +# content-based MIME sniffing in src/upload_handler.py. We install both here +# (libmagic1 + the python-magic wrapper, below) rather than in requirements.txt +# because python-magic resolves libmagic at import time: where the lib is +# absent the import can block or raise, so keeping it image-only avoids +# regressing pip/venv installs on hosts without libmagic. Debian always has the +# lib here, so the import is instant and detection actually works. + +# Docker CLI (client only — daemon stays on the host via the +# /var/run/docker.sock mount). The Debian `docker.io` package ships +# dockerd but not the client binary on slim, so grab the static client +# tarball from download.docker.com instead. +ARG DOCKER_CLI_VERSION=29.6.2 +RUN ARCH="$(dpkg --print-architecture)" \ + && case "$ARCH" in \ + amd64) DARCH=x86_64 ;; \ + arm64) DARCH=aarch64 ;; \ + *) echo "unsupported arch $ARCH"; exit 1 ;; \ + esac \ + && curl -fsSL "https://download.docker.com/linux/static/stable/${DARCH}/docker-${DOCKER_CLI_VERSION}.tgz" \ + -o /tmp/docker.tgz \ + && tar -xzf /tmp/docker.tgz -C /tmp \ + && install -m 0755 /tmp/docker/docker /usr/local/bin/docker \ + && rm -rf /tmp/docker /tmp/docker.tgz + WORKDIR /app -# Install Python deps first (layer cache) -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +# Install Python deps first (layer cache). Optional extras (PyMuPDF AGPL, etc.) +# are opt-in so the default image stays MIT-core; see requirements-optional.txt. +ARG INSTALL_OPTIONAL=false +COPY requirements.txt requirements-optional.txt ./ +RUN pip install --no-cache-dir -r requirements.txt \ + && if [ "$INSTALL_OPTIONAL" = "true" ]; then pip install --no-cache-dir -r requirements-optional.txt; fi + +# python-magic powers content-based MIME sniffing in src/upload_handler.py. +# Image-only (not in requirements.txt) because it needs the libmagic1 system +# lib installed above; see the apt note near the top of this stage. +RUN pip install --no-cache-dir python-magic==0.4.27 + +# Pre-install the patched basicsr/gfpgan/facexlib wheels built in the +# realesrgan-wheels stage (--no-deps keeps the image lean — torch & friends are +# pulled only when realesrgan is actually installed). With these dists already +# satisfied, the Cookbook's plain `pip install realesrgan` resolves them from +# wheels instead of rebuilding the sdists that fail on Python 3.14. +COPY --from=realesrgan-wheels /wheels/ /tmp/odysseus-wheels/ +RUN pip install --no-cache-dir --no-deps /tmp/odysseus-wheels/*.whl \ + && rm -rf /tmp/odysseus-wheels # Copy app code COPY . . # Create data directory (mount a volume here for persistence) -RUN mkdir -p data logs +RUN mkdir -p data logs services/cache/search # Entrypoint that drops to PUID/PGID (default 1000:1000) and repairs # ownership on the bind-mounted /app/data and /app/logs. Without this, diff --git a/LICENSE b/LICENSE index 7087e2d59..0c97efd25 100644 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,235 @@ -MIT License - -Copyright (c) 2025 Odysseus Contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. +GNU AFFERO GENERAL PUBLIC LICENSE +Version 3, 19 November 2007 + +Copyright (C) 2007 Free Software Foundation, Inc. + +Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. + + Preamble + +The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software. + +The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users. + +When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things. + +Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software. + +A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public. + +The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version. + +An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license. + +The precise terms and conditions for copying, distribution and modification follow. + + TERMS AND CONDITIONS + +0. Definitions. + +"This License" refers to version 3 of the GNU Affero General Public License. + +"Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks. + +"The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations. + +To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work. + +A "covered work" means either the unmodified Program or a work based on the Program. + +To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well. + +To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying. + +An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion. + +1. Source Code. +The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work. + +A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language. + +The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it. + +The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those +subprograms and other parts of the work. + +The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source. + +The Corresponding Source for a work in source code form is that same work. + +2. Basic Permissions. +All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law. + +You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you. + +Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary. + +3. Protecting Users' Legal Rights From Anti-Circumvention Law. +No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures. + +When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures. + +4. Conveying Verbatim Copies. +You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program. + +You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee. + +5. Conveying Modified Source Versions. +You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices". + + c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so. + +A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate. + +6. Conveying Non-Source Forms. +You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways: + + a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b. + + d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d. + +A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work. + +A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product. + +"Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made. + +If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM). + +The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network. + +Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying. + +7. Additional Terms. +"Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions. + +When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission. + +Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or authors of the material; or + + e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors. + +All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying. + +If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms. + +Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way. + +8. Termination. + +You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11). + +However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation. + +Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice. + +Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10. + +9. Acceptance Not Required for Having Copies. + +You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so. + +10. Automatic Licensing of Downstream Recipients. + +Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License. + +An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts. + +You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it. + +11. Patents. + +A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version". + +A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License. + +Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version. + +In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party. + +If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid. + +If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it. + +A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007. + +Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law. + +12. No Surrender of Others' Freedom. + +If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program. + +13. Remote Network Interaction; Use with the GNU General Public License. + +Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph. + +Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License. + +14. Revised Versions of this License. + +The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. + +Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation. + +If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program. + +Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version. + +15. Disclaimer of Warranty. + +THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + +16. Limitation of Liability. + +IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. + +17. Interpretation of Sections 15 and 16. + +If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee. + +END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + +If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. + +To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. + + This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + +If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements. + +You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see . diff --git a/Odysseus.spec b/Odysseus.spec new file mode 100644 index 000000000..547460c69 --- /dev/null +++ b/Odysseus.spec @@ -0,0 +1,45 @@ +# -*- mode: python ; coding: utf-8 -*- + + +a = Analysis( + ['launcher.py'], + pathex=[], + binaries=[], + datas=[('static', 'static'), ('scripts', 'scripts'), ('mcp_servers', 'mcp_servers'), ('services/hwfit/data', 'services/hwfit/data'), ('config', 'config'), ('.env.example', '.env.example')], + hiddenimports=[], + hookspath=[], + hooksconfig={}, + runtime_hooks=[], + excludes=[], + noarchive=False, + optimize=0, +) +pyz = PYZ(a.pure) + +exe = EXE( + pyz, + a.scripts, + [], + exclude_binaries=True, + name='Odysseus', + debug=False, + bootloader_ignore_signals=False, + strip=False, + upx=True, + console=False, + disable_windowed_traceback=False, + argv_emulation=False, + target_arch=None, + codesign_identity=None, + entitlements_file=None, + icon=['static\\icon.ico'], +) +coll = COLLECT( + exe, + a.binaries, + a.datas, + strip=False, + upx=True, + upx_exclude=[], + name='Odysseus', +) diff --git a/README.md b/README.md index ec62147f3..705ec6b68 100644 --- a/README.md +++ b/README.md @@ -1,246 +1,76 @@ -# Odysseus -─────────────────────────────────────────────── - ⊹ ࣪ ˖ ૮( ˶ᵔ ᵕ ᵔ˶ )っ Odysseus vers. 1.0 -─────────────────────────────────────────────── +

+ Odysseus +

-![Odysseus](docs/odysseus.jpg) +

+ A self-hosted AI workspace for chat, agents, research, documents, email, notes, calendar, and local model workflows. +

-A self-hosted AI workspace -- meant to be the self-hosted version of the UI experience you get from ChatGPT and Claude. But with more jank and fun. Running on your own hardware, with your own data -- local-first, privacy-first, and no trojan. +

+ Quick Start · + Setup Guide · + Contributing · + Roadmap +

-## Features - - **Chat** -- chat with any local model or API; adding them is super simple.
 vLLM · llama.cpp · Ollama · OpenRouter · OpenAI - - **Agent** -- hand it tools and let it run the whole task itself.
 built on [opencode](https://github.com/anomalyco/opencode) · MCP · web · files · shell · skills · memory - - **Cookbook** -- Scans your hardware, recommends models, click to download and serve.. easy!
 built on [llmfit](https://github.com/AlexsJones/llmfit) · VRAM-aware · GGUF / FP8 / AWQ · fit scoring · vLLM / llama.cpp serving - - **Deep Research** -- multi-step runs that gather, read, and synthesize sources into a nice visual report.
 adapted from [Tongyi DeepResearch](https://github.com/Alibaba-NLP/DeepResearch) - - **Compare** -- a fun tool to compare models side by side. Test completely blind, no bias!
 multi-model · blind test · synthesis - - **Documents** -- YOU write the text, AI is there to assist, not the opposite.
 multi-tab editor · markdown · HTML · CSV · syntax highlighting · AI edits · suggestions - - **Memory / Skills** -- Persistent memory and skills, your agent evolves over time as it better understands you and your tasks!
 ChromaDB · fastembed (ONNX) · vector + keyword retrieval · import/export - - **Email** -- IMAP/SMTP inbox with AI triage built in: urgency reminders, auto-tag, auto-summary, auto-reply drafts, auto-spam.
 IMAP · SMTP · per-account routing · CalDAV-aware - - **Notes & Tasks** -- Quick notes with reminders, a todo list, and scheduled tasks the agent can act on.
 note pings · checklist · cron-style tasks · ntfy / browser / email channels - - **Calendar** -- Local-first calendar with CalDAV sync to Radicale / Nextcloud / Apple / Fastmail.
 CalDAV pull · .ics import/export · per-calendar colors · agent-aware - - **Works on mobile** -- looks and runs great on your phone, not just desktop.
 responsive · installable (PWA) · touch gestures - - **Extras** -- more to explore, happy if you give it a go!
 image editor · theme editor · file uploads (vision + PDF) · web search · presets · sessions · 2FA +

+ Packaging status +

-## Demo -A full, hover-to-play tour lives on the landing page (`docs/index.html`). A few looks: - -### Chat & Agents -![Chat & Agents](docs/chat.gif) -### Deep Research -![Deep Research](docs/research.gif) -### Compare -![Compare](docs/compare.gif) -### Documents -![Documents](docs/document.gif) -### Notes & Tasks -![Notes & Tasks](docs/notes.gif) +

+ Odysseus interface +

-## Quick Start +--- -Defaults work out of the box — clone, run, configure inside the app. -Open the **Settings** panel after first login to point Odysseus at your LLM -server, search provider, email account, etc. Only touch `.env` if you need -to override deployment-level things like `AUTH_ENABLED`, `DATABASE_URL`, -or pre-seed `ODYSSEUS_ADMIN_PASSWORD` (otherwise an initial password is -generated and printed on first boot). +## Quick Start -Contributing? See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, testing, and -pull request guidelines. +> `dev` is the default branch and gets the newest changes first. Use [`main`](https://github.com/odysseus-dev/odysseus/tree/main) if you want the more curated branch. -### Option 1: Docker (recommended) ```bash -git clone https://github.com/pewdiepie-archdaemon/odysseus.git +git clone https://github.com/odysseus-dev/odysseus.git cd odysseus -cp .env.example .env # optional, but recommended for explicit defaults +cp .env.example .env docker compose up -d --build ``` -Compose starts Odysseus, ChromaDB, SearXNG, and ntfy. First run does a full -image build. Open `http://localhost:7000` after the containers are healthy. - -Cookbook remote servers use an Odysseus-owned SSH key from `./data/ssh` -inside Docker. In **Cookbook -> Settings -> Servers**, generate/copy the -public key and add it to the remote server's `~/.ssh/authorized_keys`. -After generating the key, you can also install it from the host with: -```bash -ssh-copy-id -i data/ssh/id_ed25519.pub user@server -``` -Cookbook local downloads are stored in `./data/huggingface`, mounted as -`~/.cache/huggingface` inside the Odysseus container. - -Useful checks: -```bash -docker compose ps -docker compose logs --tail=120 odysseus -docker compose logs odysseus | grep -E 'ChromaDB|MemoryVectorStore|DEGRADED' -docker compose exec odysseus python -c "from services.hwfit.models import get_models; print(len(get_models()))" -``` - -Expected vector-memory startup lines in Docker: -```text -ChromaDB connected: chromadb:8000 -MemoryVectorStore initialized -``` - -The Cookbook model catalog check should print a non-zero count. If it prints -`0`, rebuild the Odysseus image with `docker compose build --no-cache odysseus`. - -### Option 2: Manual install — Linux / macOS -**Requirements:** Python 3.11+. On Linux/Termux, Cookbook also requires `tmux` -for background model downloads and serves. - -Install system packages first: -```bash -# Debian/Ubuntu -sudo apt install tmux - -# Arch -sudo pacman -S tmux - -# Fedora -sudo dnf install tmux -``` - -Then install Odysseus: -```bash -git clone https://github.com/pewdiepie-archdaemon/odysseus.git -cd odysseus -python3 -m venv venv -source venv/bin/activate -pip install -r requirements.txt -python setup.py # creates data dirs and prints an initial admin password -python -m uvicorn app:app --host 0.0.0.0 --port 7000 -``` - -### Option 3: Manual install — Windows (PowerShell) -Windows support is not actively tested. Use it with caution; Docker on Linux -or a Linux/macOS manual install is the safer path for now. - -```powershell -git clone https://github.com/pewdiepie-archdaemon/odysseus.git -cd odysseus -python -m venv venv -venv\Scripts\Activate.ps1 -pip install -r requirements.txt -python setup.py -python -m uvicorn app:app --host 0.0.0.0 --port 7000 -``` -Open `http://localhost:7000`, log in with the generated admin password, -and configure everything else inside **Settings**. +Open `http://localhost:7000` when the containers are healthy. The first admin password is printed in `docker compose logs odysseus`. -## Security Notes -Odysseus is a self-hosted workspace with powerful local tools: shell access, file uploads, model downloads, web research, email/calendar integrations, and API tokens. Treat it like an admin console. +Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration live in the [setup guide](docs/setup.md). -- Keep `AUTH_ENABLED=true` for any network-accessible deployment. -- Do not expose it directly to the public internet without HTTPS and a trusted reverse proxy. -- Keep `data/`, `.env`, logs, databases, and uploaded/generated media out of Git. They are ignored by default. -- Review `data/auth.json` after first boot: disable open signup unless you intentionally want it, make only your own account admin, and keep demo/test accounts non-admin. -- Non-admin users do not get shell/Python/file read/write by default, and admin-only routes/tools such as MCP management, API tokens, webhooks, model/cookbook serving, backup/vault, and app settings are admin-gated. Other features are controlled by per-user privileges, so review each user's privileges before exposing a deployment. -- Rotate any API keys or tokens that were ever pasted into a shared chat, demo, screenshot, or log. -- If you enable API tokens or webhooks, create separate tokens per integration and delete unused ones. -- Prefer binding manual development runs to `127.0.0.1`; bind to `0.0.0.0` only when you intentionally want LAN/reverse-proxy access. -- Before publishing a fork, run `git status --short` and confirm no private files from `.env`, `data/`, `logs/`, uploads, backups, or local databases are staged. - -### Putting it behind HTTPS -Odysseus serves plain HTTP on its port. That's fine for `localhost` and trusted LAN/VPN use, but browsers will warn ("Password fields present on an insecure page") and the login + API tokens travel in cleartext. For anything reachable outside your machine — including a Tailscale IP shared with other devices — put a TLS-terminating reverse proxy in front. +## Features -Shortest path with [Caddy](https://caddyserver.com/) (auto-renews Let's Encrypt certs): +- **Chat + Agents** — local/API models, tools, MCP, files, shell, skills, and memory. +- **Cookbook** — hardware-aware model recommendations, downloads, and serving. +- **Deep Research** — multi-step web research with source reading and report generation. +- **Compare** — blind side-by-side model testing and synthesis. +- **Documents** — writing-first editor with AI edits, suggestions, Markdown, HTML, CSV, and syntax highlighting. +- **Email** — IMAP/SMTP inbox with triage, tags, summaries, reminders, and reply drafts. +- **Notes, Tasks + Calendar** — reminders, todos, scheduled agent tasks, and CalDAV sync. +- **Extras** — gallery/image editor, themes, uploads, web search, presets, sessions, and 2FA. -```caddy -odysseus.example.com { - reverse_proxy localhost:7000 -} -``` +## Demo -For a LAN-only Tailscale deployment, Caddy + [tailscale-cert](https://caddyserver.com/docs/caddyfile/options#auto-https) or the built-in MagicDNS HTTPS feature both work. nginx/Traefik configs are similar — proxy `localhost:7000`, terminate TLS at the proxy. Once that's in place, the browser warning goes away and your login is encrypted. +A full hover-to-play tour lives on the landing page: [`docs/index.html`](docs/index.html). ## Contributing -Help is welcome. The best entry points are fresh-install testing, provider setup -bugs, mobile/editor polish, docs, and small focused refactors. See -[ROADMAP.md](ROADMAP.md) for the current help-wanted list. - -## Configuration -Most setup is done inside the app with `/setup` or **Settings**. Use `.env` -for deployment-level defaults and secrets you want present before first boot. -Key settings: -| Variable | Default | Description | -|---|---|---| -| `LLM_HOST` | `localhost` | Your LLM server (e.g. `llm-host.local:8000`) | -| `LLM_HOSTS` | -- | Comma-separated list for model discovery | -| `OPENAI_API_KEY` | -- | Optional OpenAI key. Prefer adding providers in the app unless pre-seeding. | -| `SEARXNG_INSTANCE` | `http://localhost:8080` | SearXNG URL. Docker overrides this to `http://searxng:8080`. | -| `SEARXNG_SECRET` | generated on first Docker boot | Optional SearXNG cookie/CSRF secret. Leave blank unless you need to pin it. | -| `AUTH_ENABLED` | `true` | Enable/disable login | -| `LOCALHOST_BYPASS` | `false` | Development-only auth bypass for loopback requests. Keep false for shared/network deployments. | -| `DATABASE_URL` | `sqlite:///./data/app.db` | Database connection string | -| `CHROMADB_HOST` | `localhost` | ChromaDB host for vector memory. Docker overrides this to `chromadb`. | -| `CHROMADB_PORT` | `8100` | ChromaDB port for manual host runs. Docker overrides this to `8000`. | -| `EMBEDDING_URL` | -- | OpenAI-compatible embeddings endpoint | +Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, docs, and small focused refactors. See [CONTRIBUTING.md](CONTRIBUTING.md) and [ROADMAP.md](ROADMAP.md). -### Bundled services -Docker Compose includes these by default. The bundled service ports bind to `127.0.0.1` unless you opt in to a different bind address in `.env`, so they are reachable from the host machine but not from your LAN or the public internet by default: +## Security - - **ChromaDB** → vector store for semantic memory. In Docker, Odysseus connects to `chromadb:8000`; from the host it is exposed as `${CHROMADB_BIND:-127.0.0.1}:8100`. - - **SearXNG** → meta search for web search. In Docker, Odysseus connects to `searxng:8080`; from the host it is exposed as `127.0.0.1:8080`. - - **ntfy** → local notification service, exposed as `${NTFY_BIND:-127.0.0.1}:8091`. +Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model/service ports publicly. Deployment details are in the [setup guide](docs/setup.md#security-notes). -**Phone push notifications via ntfy:** A phone cannot subscribe to `127.0.0.1` on your server. To expose ntfy safely without opening it on every interface: +## Star History - - **Tailscale (recommended)** — set `NTFY_BIND=` and `NTFY_BASE_URL=http://:8091` in `.env`, recreate ntfy, then point the ntfy Android/iOS app at `http://:8091/`. - - **Enable ntfy auth and bind to LAN** — add `NTFY_AUTH_FILE` + `NTFY_AUTH_DEFAULT_ACCESS=deny-all` to the `ntfy` service, create a user with `docker compose exec ntfy ntfy user add ...`, then set `NTFY_BIND` to your LAN IP. See the [ntfy docs](https://docs.ntfy.sh/config/#access-control). - -### Optional external services - - **Ollama** → local LLM server -- [ollama.ai](https://ollama.ai) - -### Ollama with Docker -If Odysseus is running in Docker and Ollama is running on the host, add the endpoint in Settings as: - -`http://host.docker.internal:11434/v1` - -The default Compose file already maps `host.docker.internal` on Linux. Ollama also needs to listen outside its own loopback interface: - -```bash -OLLAMA_HOST=0.0.0.0:11434 ollama serve -``` - -For a systemd Ollama install, set that in the Ollama service override. If Odysseus can see Ollama but requests hang or fail, check that your host firewall allows Docker bridge traffic to port `11434`. - -First-token latency is usually Ollama/model/hardware, not Odysseus. To compare, test Ollama directly: - -```bash -curl http://127.0.0.1:11434/v1/models -``` - -## Architecture -``` -app.py # FastAPI entry point -core/ auth, database, middleware, constants -src/ llm_core, agent_loop, agent_tools, chat_processor, search/ -routes/ chat, session, document, memory, model … endpoints -services/ docs, memory, search, hwfit (Cookbook) … -static/ index.html + app.js + style.css + js/ (modular front-end) -docs/ landing page (index.html) + preview clips -``` - -## Data -All user data lives in `data/` (gitignored): `app.db` (sessions, messages, documents), -`memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`. + + + + + Star History Chart + + ## License -MIT -- see [LICENSE](LICENSE) and [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md). -``` - | - ||| - ||||| - | | | ||||||| - )_) )_) )_) ~|~ - )___))___))___)\ | - )____)____)_____)\\| - _____|____|____|_____\\\__ - \ / - ~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~ - ~^~ all aboard! ~^~ - ~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~ -``` +AGPL-3.0-or-later -- see [LICENSE](LICENSE) and [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md). diff --git a/ROADMAP.md b/ROADMAP.md index aa79c3088..d29ac5c75 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,6 +1,6 @@ # Roadmap / Help Wanted -Odysseus is on a voyage, but not home yet. It works great for me (lol), but this is ship is moving fast and feedback/help would be appreciated! (I dont know what I'm doing hlep). +Odysseus is on a voyage, but not home yet. It works great for me (lol), but this ship is moving fast and feedback/help would be appreciated! (I don't know what I'm doing, help). If you see weird CSS, strange layout behavior, or a suspiciously murky corner of the codebase, you are probably right to stay away. @@ -8,25 +8,59 @@ the codebase, you are probably right to stay away. ## High Priority - SQUASH BUGS -- Fresh Docker install smoke tests on Linux, macOS, and Windows!! +- Fresh install smoke tests on Linux, macOS, and Windows. Docker, native Python, + and WSL all need coverage. - Integration audit: do integrations even work? Confirm what works, what needs setup docs, and what should be removed or hidden. -- Self-host troubleshooting cookbook. Document the weird 30-second fixes that otherwise become 30-minute searches: Dovecot cleartext auth for local stacks, ntfy Android Instant Delivery for non-ntfy.sh servers, clipboard limits on plain-HTTP Tailscale URLs, Radicale collection URLs, and similar traps. - Cookbook reliability on other computers. This is probably the area most likely to need work across different machines, GPUs, drivers, shells, and Python environments. -- Tile/window management correctness. I had to brute force my way a bit here, I'm aware, popups, dropdowns, and fixed-position UI inside transformed modals can land in the wrong place. -- Esc button, it's small but a lot of windows that arent still close on esc and alot of them doesnt. -- Skill audit, how does your model respond to skill injection, does it follow? Does its parsing miss? +- Cookbook SGLang support across platforms. Make sure SGLang setup/serve works + predictably on Linux, Windows/WSL, macOS where possible, Docker, and common + NVIDIA/AMD hardware paths. +- Deep Research model presets by hardware. Recommend approved model/parameter + profiles for small, medium, and large local setups so people with different + hardware can use Deep Research without guessing. Surface this either in Deep + Research settings or as a Cookbook scan/dropdown suggestion. +- Cookbook model scan/download ranking. Prioritize newer architectures and + better hardware-fit models instead of scoring everything almost the same. + Ranking should account for architecture age, quant format, VRAM/RAM fit, + backend support, vision/mmproj requirements, and likely serve reliability. +- Cookbook error feedback and logging. Failed downloads, dependency installs, + preflights, and serve jobs should show the actual command/output/error in the + UI, with copyable logs and clear next steps instead of just "crashed". +- Agent prompt/context bloat. Agent mode is too heavy for smaller local models: + tool schemas, skills, memory, documents, and instructions can eat the context + before the user request really starts. We need slimmer prompts, better tool + selection, smaller default tool sets, and clearer guidance for models with + 4k/8k/16k context windows. +- Skill/tool prompt-injection audit. User-editable skills, notes, documents, + fetched pages, and memories should be treated as untrusted data. Keep testing + whether models follow malicious instructions from those surfaces. - Better degraded-state reporting for ChromaDB, SearXNG, email, ntfy, and provider probes. +- Email performance audit. Fetching, searching, opening, deleting, and sending + email can feel slow, especially over IMAP/SMTP providers with high latency. + Need someone who knows mail performance to profile the current flow, identify + whether the bottleneck is IMAP folder select/fetch, cache invalidation, + attachment/body loading, SMTP handshakes, or frontend refresh behavior, then + propose safer caching/prefetch/batching without breaking multi-account state. - Provider setup/probing audit for Anthropic, Gemini, Groq, xAI, OpenRouter, OpenAI, and DeepSeek. ## Refactor Targets - CSS cleanup. `static/style.css` basically Calypso's island atm. - Tour core helper. The onboarding tours have too much copy-pasted scaffolding; promote a shared `tour-core.js` helper before adding more tours. +- Modal/window positioning cleanup. Some window controls have improved, but the + underlying popup/dropdown/fixed-position behavior is still too fragile. - Mobile media override discoverability. A lot of "CSS did not move" bugs are mobile `@media` overrides of the same selector; comments or linting around desktop/mobile paired rules would help. - Dead code pass for old routes, stale feature flags, and unused UI states. ## Frontend +- Expand the Editor for quicker, more robust everyday use. Better file/document + handling, smoother window behavior, clearer save/export flows, stronger image + editing affordances, and fewer brittle edge cases. +- Better AI integration for Notes and Todos. Notes should be easier for the + agent to read, update, summarize, and turn into actions. Todos should be + assignable to an agent from the UI, possibly through a button, task action, + or dedicated skill/tool flow. - Mobile gallery/editor polish. Easier to launch/download inpaint model or any missing pieces. - Accessibility pass: keyboard navigation, focus states, contrast, reduced motion. - Improve empty states and error messages on fresh installs. diff --git a/SECURITY.md b/SECURITY.md index 2cca34be9..1fa5b0b3b 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -8,16 +8,20 @@ Security fixes are handled on the default branch until formal releases are cut. ## Deployment Guidance -- Keep `AUTH_ENABLED=true`. +- Keep `AUTH_ENABLED=true` for any network-accessible deployment. +- Keep `LOCALHOST_BYPASS=false` outside local development. +- Set `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway. - Use HTTPS when exposing the app beyond localhost. -- Put the app behind a trusted reverse proxy or private network. -- Protect `.env`, `data/`, logs, uploaded files, generated media, and database files. +- Put the authenticated Odysseus web/API entrypoint behind a trusted reverse proxy or private access layer such as Cloudflare Access, Tailscale, or a VPN. +- Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only. +- Protect `.env`, `data/`, `logs/`, uploads, generated media, backups, auth/session files, database files, API keys, and model/provider tokens. - Disable open signup unless you intentionally want new accounts. - Keep demo/test users non-admin, and remove them entirely on serious deployments. - Give admin accounts strong passwords and enable 2FA where possible. - Leave high-risk agent tools restricted to admins: shell, Python, file read/write, email send/read, MCP, app API, task/skill/memory management, settings, tokens, and model serving. - Rotate API keys, webhook secrets, and Odysseus API tokens if they appear in logs, screenshots, demos, or shared chats. - Treat shell, model-serving, MCP, email, calendar, and vault features as privileged admin functionality. +- Common internal-only ports are Odysseus `7000`, SearXNG `8080`, ntfy `8091`, ChromaDB `8100`, Ollama `11434`, and local model/provider APIs such as `8000-8020`. ## Publishing A Fork @@ -29,7 +33,7 @@ git check-ignore -v .env data/auth.json data/app.db logs/compound.log odysseus.d git grep -n -I -E "(sk-[A-Za-z0-9_-]{20,}|xox[baprs]-|AIza[0-9A-Za-z_-]{20,}|Bearer [A-Za-z0-9._~+/-]{20,})" -- . ':!static/lib/**' ':!package-lock.json' ``` -Only `.env.example`, docs, source, tests, and static assets should be committed. Never commit live `data/` contents, local databases, uploaded files, generated media, logs, backups, API keys, password hashes, or personal documents. +Only `.env.example`, docs, source, tests, and static assets should be committed. Never commit live `.env` values, `data/` contents, local databases, uploaded files, generated media, logs, backups, auth/session files, API keys, model/provider tokens, password hashes, or personal documents. ## Reporting diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md new file mode 100644 index 000000000..48665a61d --- /dev/null +++ b/THREAT_MODEL.md @@ -0,0 +1,81 @@ +# Threat Model + +Odysseus is a **self-hosted AI workspace with privileged local access**. This document states the trust boundary so contributors can reason about security decisions without reading through the full auth and middleware stack. + +## Trust Boundary + +Odysseus is designed for **trusted users on a private network**, not public exposure. The README describes it as "treat it like an admin console" — that framing is accurate. A logged-in admin can execute shell commands, read and write files, send email, and control model serving. This is intentional. The threat model does not try to prevent admins from doing these things. It does try to prevent: + +- Unauthenticated access +- Non-admins reaching admin-only capabilities +- The AI agent acting on instructions injected through untrusted content (web results, emails, fetched pages, memories) +- Internal services (ChromaDB, Ollama, SearXNG, etc.) being reachable from outside the host + +## Roles and Capabilities + +| Capability | Admin | Non-admin (default) | +|---|---|---| +| Chat with agent | ✓ | ✓ | +| Browser tool | ✓ | ✓ | +| Documents | ✓ | ✓ | +| Research mode | ✓ | ✓ | +| Image generation | ✓ | ✓ | +| Memory management | ✓ | ✓ | +| Shell / Python execution | ✓ | ✗ | +| File read / write | ✓ | ✗ | +| Email send / read | ✓ | ✗ | +| MCP tools | ✓ | ✗ | +| Calendar management | ✓ | ✗ | +| Token / webhook management | ✓ | ✗ | +| Model serving | ✓ | ✗ | +| Vault | ✓ | ✗ | +| Settings | ✓ | ✗ | + +Non-admin defaults are in `core/auth.py:DEFAULT_PRIVILEGES`. Tool enforcement is in `src/tool_security.py:NON_ADMIN_BLOCKED_TOOLS`. Any tool whose name starts with `mcp__` is also blocked for non-admins. Admins always get full access regardless of stored privilege values. + +## Authentication + +- **Sessions:** bcrypt passwords, 7-day session tokens stored atomically in `data/sessions.json` via `core/atomic_io.py`. +- **2FA:** TOTP with 8 single-use backup codes. Verified after password check, before session issuance. +- **Reserved usernames:** `internal-tool`, `api`, `demo`, `system` cannot be registered or renamed into. Defined in `core/auth.py:RESERVED_USERNAMES`. + - `internal-tool` is security-critical: `core/middleware.py:require_admin` treats any request where `request.state.current_user == "internal-tool"` as the in-process tool loopback and grants admin unconditionally. A real account with that name would silently pass every `require_admin` check. +- **Orphan sessions:** `validate_token` re-checks that the user record still exists on every call. A deleted user's cookie is dropped on next request rather than continuing to authenticate. + +## Internal Tool Loopback + +Agent tool calls reach admin-gated HTTP routes over an in-process HTTP loopback. The mechanism: + +1. At app startup, `core/middleware.py` generates a random `INTERNAL_TOOL_TOKEN` via `secrets.token_hex(32)`. It is never persisted and never sent to clients. +2. Loopback requests carry `X-Odysseus-Internal-Token: ` or have `request.state.current_user` already set to `"internal-tool"` by the auth middleware. +3. `require_admin` recognises either signal and grants access without checking the session user. + +The agent may be running in a non-admin user's session, but tool dispatch first calls `src/tool_security.py:owner_is_admin_or_single_user` to verify the session owner is an admin before issuing any loopback call. Non-admin users cannot invoke admin tools even via the agent. + +## Prompt-Injection Hardening + +External content that reaches the LLM is treated as untrusted via `src/prompt_security.py`: + +- `untrusted_context_message(label, content)` wraps the content in a `user`-role message with a header block instructing the model not to follow instructions inside it. Content goes in as data, not as a system instruction. +- `UNTRUSTED_CONTEXT_POLICY` is a system-prompt preamble that states the same policy at the top of every session where untrusted data may appear. + +**Untrusted surfaces that must go through this wrapper:** web search results, fetched URLs, emails (read), saved memories, skill text, notes, and any tool output sourced from outside the server. Injecting untrusted content directly into the system role is a security bug. + +## Security Headers + +`core/middleware.py:SecurityHeadersMiddleware` sets headers on every response: + +- `X-Frame-Options: DENY` + `frame-ancestors 'none'` on all routes except tool-render iframes (which are sandboxed at the HTML level). +- `X-Content-Type-Options: nosniff` and `Referrer-Policy: no-referrer` everywhere. +- **CSP:** nonce-based `script-src 'self' 'nonce-{nonce}' https://cdn.jsdelivr.net`. `style-src 'unsafe-inline'` is intentionally kept — `static/index.html` ships inline ` +
+

Pair a device

+

Generate a one-time pairing code (a chat-scoped API token) for a LAN client.

+
+ +
+

Admin only. Each code mints a new token, shown once. Manage or revoke under Settings → API tokens.

+
""" + return HTMLResponse(page) + + @router.post("/pair") + def pair_create(request: Request): + """Mint a pairing code. Admin-cookie only; CSRF-safe because the + SameSite=Lax session cookie is not sent on a cross-site POST (same + protection as POST /api/tokens). Minting invalidates the token cache so + the code works immediately, no restart. `?format=json` returns the + payload for an in-app pairing screen.""" + require_admin(request) + owner = get_current_user(request) + invalidate = getattr(request.app.state, "invalidate_token_cache", None) + token_id, raw_token = mint_pairing_token(owner, invalidate) + + hosts = _pairing.lan_ip_candidates() + host = hosts[0] if hosts else "127.0.0.1" + port = request.url.port or _pairing.default_port() + payload = _pairing.pairing_payload(host, port, raw_token) + qr = _pairing.pairing_qr_png_data_uri(payload) + qr_ok = bool(qr and qr.startswith("data:image/png;base64,")) + + if (request.query_params.get("format") or "").lower() == "json": + return { + "host": host, + "port": port, + "token": raw_token, + "token_id": token_id, + "hosts": hosts, + "payload": payload, + "qr": qr if qr_ok else None, + } + + import json as _json + payload_json = _json.dumps(payload, separators=(",", ":")) + # Only ever emit a known PNG data-URI into the src; every other value is + # html.escaped. + qr_block = ( + f'Pairing QR' + if qr_ok else "

QR rendering unavailable -- enter the details manually.

" + ) + page = f""" + +Pairing code + +
+

Pairing code

+ {qr_block} +
Host: {html.escape(host)}
+
Port: {html.escape(str(port))}
+
Token: {html.escape(raw_token)}
+
Payload: {html.escape(payload_json)}
+

Shown once. This grants chat access to your Odysseus; revoke it + in Settings → API tokens (id {html.escape(token_id)}). The + device must be on the same network, and the server must bind to your LAN.

+
""" + return HTMLResponse(page) + + return router diff --git a/core/atomic_io.py b/core/atomic_io.py index b7801ecb9..81c640d8a 100644 --- a/core/atomic_io.py +++ b/core/atomic_io.py @@ -26,7 +26,7 @@ def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) -> """ os.makedirs(os.path.dirname(path) or ".", exist_ok=True) tmp = f"{path}.tmp.{os.getpid()}" - with open(tmp, "w") as f: + with open(tmp, "w", encoding="utf-8") as f: json.dump(data, f, indent=indent) f.flush() os.fsync(f.fileno()) @@ -34,9 +34,11 @@ def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) -> def atomic_write_text(path: str, text: str) -> None: + if not isinstance(text, str): + raise TypeError("atomic_write_text expects a string") os.makedirs(os.path.dirname(path) or ".", exist_ok=True) tmp = f"{path}.tmp.{os.getpid()}" - with open(tmp, "w") as f: + with open(tmp, "w", encoding="utf-8") as f: f.write(text) f.flush() os.fsync(f.fileno()) diff --git a/core/auth.py b/core/auth.py index ded0f866a..4bc9a70dd 100644 --- a/core/auth.py +++ b/core/auth.py @@ -3,6 +3,7 @@ Config stored in data/auth.json. Uses bcrypt directly. """ +import enum import json import os import secrets @@ -19,6 +20,7 @@ from core.atomic_io import atomic_write_json as _atomic_write_json # noqa: E402 +from core.middleware import INTERNAL_TOOL_USER # noqa: E402 DEFAULT_PRIVILEGES = { "can_use_agent": True, @@ -30,16 +32,50 @@ "can_manage_memory": True, "max_messages_per_day": 0, "allowed_models": [], + "allowed_models_restricted": False, + # Explicit "block every model" sentinel. An empty `allowed_models` list is + # ambiguous — it's also what gets sent when the admin clicks "[All]" — so + # we need a dedicated flag to express "this user may use no models at all" + # distinctly from "this user has no restriction". + "block_all_models": False, } # Admins get everything ADMIN_PRIVILEGES = {k: (True if isinstance(v, bool) else (0 if isinstance(v, int) else [])) for k, v in DEFAULT_PRIVILEGES.items()} - -DEFAULT_AUTH_PATH = os.path.join( - Path(__file__).parent.parent, "data", "auth.json" -) +ADMIN_PRIVILEGES["allowed_models_restricted"] = False +# Admins must never be blocked from using models — the generic dict +# comprehension above flips every boolean default to True, which would be +# backwards for this sentinel. +ADMIN_PRIVILEGES["block_all_models"] = False + +from src.constants import AUTH_FILE, PASSWORD_MIN_LENGTH +DEFAULT_AUTH_PATH = AUTH_FILE TOKEN_TTL = 60 * 60 * 24 * 7 # 7 days +# Usernames the auth + middleware layer reserve as internal "synthetic owner" +# sentinels; they must never belong to a real account. The most dangerous is +# "internal-tool": `core.middleware.require_admin` treats any request whose +# `current_user == "internal-tool"` as the in-process tool loopback and grants +# admin, and because the cookie auth path sets `current_user` to the raw +# username, an account literally named "internal-tool" would be silently +# treated as an admin by every `require_admin`-gated route. "api" collides with +# the bearer-token owner-attribution sentinel. "demo"/"system" round out the +# synthetic-owner set the rest of the codebase already special-cases (see +# `_SYNTHETIC_OWNERS` in routes/assistant_routes.py and the matching guards in +# src/task_scheduler.py / routes/research_routes.py) — a real account with one +# of those names would be denied an assistant and inconsistently owner-scoped. +# Refuse to create or rename into any of them so the sentinels can't be +# impersonated. (Keep this in sync with that synthetic-owner set.) +RESERVED_USERNAMES = frozenset({INTERNAL_TOOL_USER, "api", "demo", "system"}) + + +def normalize_known_username(users: Dict[str, Any], username: str | None) -> Optional[str]: + """Return a normalized username only when it exists in the auth user map.""" + key = str(username or "").strip().lower() + if not key or key not in users: + return None + return key + def _hash_password(password: str) -> str: return bcrypt.hashpw(password.encode("utf-8"), bcrypt.gensalt()).decode("utf-8") @@ -49,6 +85,15 @@ def _verify_password(password: str, hashed: str) -> bool: return bcrypt.checkpw(password.encode("utf-8"), hashed.encode("utf-8")) +class SetAdminResult(enum.Enum): + """Outcome of AuthManager.set_admin, so callers can map each case to a + precise response instead of guessing from a bare bool.""" + OK = "ok" + USER_NOT_FOUND = "user_not_found" + NOT_AUTHORIZED = "not_authorized" # requester is not an admin + LAST_ADMIN = "last_admin" # would remove the last remaining admin + + class AuthManager: """Manages multi-user password + session-token auth system.""" @@ -60,16 +105,33 @@ def __init__(self, auth_path: str = DEFAULT_AUTH_PATH): # Guards mutations of self._sessions and the on-disk sessions.json. # Validate/create/revoke run concurrently from the FastAPI threadpool. self._sessions_lock = threading.RLock() + # Guards all mutations of self._config and the on-disk auth.json so + # concurrent create/delete/rename/privilege operations don't interleave + # and corrupt the user database. + self._config_lock = threading.Lock() + # Guards the first-run setup check-and-write so concurrent requests + # cannot both observe is_configured==False and both create admin accounts. + self._setup_lock = threading.Lock() self._load() self._load_sessions() self._migrate_single_user() + self._drop_reserved_loaded_users() self._migrate_legacy_admin_role() def _load(self): try: if os.path.exists(self.auth_path): - with open(self.auth_path, "r") as f: + with open(self.auth_path, "r", encoding="utf-8") as f: self._config = json.load(f) + # Normalize all stored usernames to lowercase so they match + # the .strip().lower() applied at login/verify time. Fixes + # "Invalid credentials" when auth.json was written with + # mixed-case keys (e.g. via manual edit or a future migration). + if "users" in self._config: + self._config["users"] = { + k.strip().lower(): v + for k, v in self._config["users"].items() + } logger.info("Auth config loaded") else: self._config = {} @@ -82,7 +144,7 @@ def _load_sessions(self): """Load persisted session tokens from disk, pruning expired ones.""" try: if os.path.exists(self._sessions_path): - with open(self._sessions_path, "r") as f: + with open(self._sessions_path, "r", encoding="utf-8") as f: data = json.load(f) now = time.time() self._sessions = {k: v for k, v in data.items() if v.get("expiry", 0) > now} @@ -106,20 +168,52 @@ def _save_sessions(self): def _migrate_single_user(self): """Migrate old single-user format to multi-user format.""" if "password_hash" in self._config and "users" not in self._config: - old_user = self._config.get("username", "admin") + old_user = str(self._config.get("username", "admin") or "admin").strip().lower() + if old_user in RESERVED_USERNAMES: + logger.warning( + "Migrating legacy single-user reserved username '%s' to 'admin'", + old_user, + ) + old_user = "admin" old_hash = self._config["password_hash"] - self._config = { - "users": { - old_user: { - "password_hash": old_hash, - "created": time.time(), - "is_admin": True, + with self._config_lock: + self._config = { + "users": { + old_user: { + "password_hash": old_hash, + "created": time.time(), + "is_admin": True, + } } } - } - self._save() + self._save() logger.info(f"Migrated single-user auth to multi-user (admin: {old_user})") + def _drop_reserved_loaded_users(self): + """Fail closed for legacy/manual auth rows that collide with sentinels.""" + users = self._config.get("users") + if not isinstance(users, dict): + return + normalized = {} + removed = [] + for username, data in users.items(): + key = str(username or "").strip().lower() + if not key: + continue + if key in RESERVED_USERNAMES: + removed.append(key) + continue + normalized[key] = data + if removed or normalized != users: + with self._config_lock: + self._config["users"] = normalized + self._save() + if removed: + logger.warning( + "Removed reserved username(s) from auth config: %s", + ", ".join(sorted(set(removed))), + ) + def _migrate_legacy_admin_role(self): """Normalize setup.py's old role='admin' marker to is_admin=True.""" changed = False @@ -144,37 +238,54 @@ def signup_enabled(self) -> bool: @signup_enabled.setter def signup_enabled(self, value: bool): - self._config["signup_enabled"] = value - self._save() + with self._config_lock: + self._config["signup_enabled"] = value + self._save() @property def is_configured(self) -> bool: return len(self.users) > 0 + def policy(self) -> dict: + """Return public auth policy constants for the frontend.""" + return { + "password_min_length": PASSWORD_MIN_LENGTH, + "reserved_usernames": sorted(RESERVED_USERNAMES), + "signup_enabled": self.signup_enabled, + "session_days": TOKEN_TTL // 86400, + } + # ------------------------------------------------------------------ # Account management # ------------------------------------------------------------------ def setup(self, username: str, password: str) -> bool: """First-run admin setup. Only works if no users exist.""" - if self.is_configured: - return False - return self.create_user(username, password, is_admin=True) + with self._setup_lock: + if self.is_configured: + return False + return self.create_user(username, password, is_admin=True) def create_user(self, username: str, password: str, is_admin: bool = False) -> bool: """Create a new user account.""" username = username.strip().lower() - if username in self.users: + if not username: return False - if "users" not in self._config: - self._config["users"] = {} - self._config["users"][username] = { - "password_hash": _hash_password(password), - "created": time.time(), - "is_admin": is_admin, - "privileges": dict(ADMIN_PRIVILEGES if is_admin else DEFAULT_PRIVILEGES), - } - self._save() + if username in RESERVED_USERNAMES: + logger.warning("Refused to create reserved username '%s'", username) + return False + with self._config_lock: + if username in self.users: + return False + if "users" not in self._config: + self._config["users"] = {} + self._config["users"][username] = { + "password_hash": _hash_password(password), + "created": time.time(), + "is_admin": is_admin, + "privileges": dict(ADMIN_PRIVILEGES if is_admin else DEFAULT_PRIVILEGES), + } + self._save() logger.info(f"Created user '{username}' (admin={is_admin})") return True @@ -187,14 +298,31 @@ def delete_user(self, username: str, requesting_user: str) -> bool: their cookie expired naturally (default ~30 days). """ username = username.strip().lower() - if username not in self.users: - return False - if username == requesting_user: - return False - if not self.users.get(requesting_user, {}).get("is_admin"): - return False - del self._config["users"][username] - self._save() + with self._config_lock: + if username not in self.users: + return False + if username == requesting_user: + return False + if not self.users.get(requesting_user, {}).get("is_admin"): + return False + # Revoke API bearer tokens before removing the auth row. The bearer + # path authenticates from ApiToken rows and does not require the + # owner to still exist, so a successful delete must not leave active + # rows behind. If the token store is unavailable, fail closed and + # keep the user/session state intact so the admin can retry. + try: + from core.database import get_db_session, ApiToken + with get_db_session() as db: + removed_tokens = db.query(ApiToken).filter(ApiToken.owner == username).delete() + if removed_tokens: + logger.info( + f"Revoked {removed_tokens} API token(s) owned by deleted user '{username}'" + ) + except Exception: + logger.warning(f"Failed to revoke API tokens for deleted user '{username}'") + return False + del self._config["users"][username] + self._save() # Purge all sessions belonging to this user. validate_token doesn't # cross-check `self.users`, so without this step a deleted user's # cookie keeps authenticating. @@ -210,6 +338,41 @@ def delete_user(self, username: str, requesting_user: str) -> bool: logger.info(f"Deleted user '{username}' (by {requesting_user}); revoked {revoked} active session(s)") return True + def rename_user(self, old_username: str, new_username: str, requesting_user: str) -> bool: + """Rename a user in auth config and active sessions. Admin only.""" + old_username = old_username.strip().lower() + new_username = new_username.strip().lower() + requesting_user = (requesting_user or "").strip().lower() + if not old_username or not new_username: + return False + if new_username in RESERVED_USERNAMES: + logger.warning("Refused to rename '%s' into reserved username '%s'", old_username, new_username) + return False + with self._config_lock: + if old_username not in self.users: + return False + if new_username in self.users: + return False + if not self.users.get(requesting_user, {}).get("is_admin"): + return False + self._config.setdefault("users", {})[new_username] = self._config["users"].pop(old_username) + self._save() + + renamed_sessions = 0 + with self._sessions_lock: + for sess in self._sessions.values(): + sess_user = str((sess or {}).get("username") or "").strip().lower() + if sess_user == old_username: + sess["username"] = new_username + renamed_sessions += 1 + if renamed_sessions: + self._save_sessions() + logger.info( + "Renamed user '%s' -> '%s' (by %s); updated %d active session(s)", + old_username, new_username, requesting_user, renamed_sessions, + ) + return True + def is_admin(self, username: str) -> bool: return self.users.get(username, {}).get("is_admin", False) @@ -231,28 +394,93 @@ def get_privileges(self, username: str) -> Dict[str, Any]: def set_privileges(self, username: str, privileges: Dict[str, Any]) -> bool: """Update privileges for a user. Can't modify admin privileges.""" username = username.strip().lower() - if username not in self.users: - return False - if self.users[username].get("is_admin"): - return False # admins always have full access - # Only allow known privilege keys - current = self.get_privileges(username) - for k, v in privileges.items(): - if k in DEFAULT_PRIVILEGES: - current[k] = v - self._config["users"][username]["privileges"] = current - self._save() + with self._config_lock: + if username not in self.users: + return False + if self.users[username].get("is_admin"): + return False # admins always have full access + # Only allow known privilege keys + current = self.get_privileges(username) + for k, v in privileges.items(): + if k in DEFAULT_PRIVILEGES: + current[k] = v + self._config["users"][username]["privileges"] = current + self._save() logger.info(f"Updated privileges for '{username}': {current}") return True + def set_admin(self, username: str, is_admin: bool, + requesting_user: str) -> SetAdminResult: + """Promote/demote an existing user to/from admin. Admin only. + + Refuses to remove the last remaining admin so the instance can never + be locked out of admin access; self-demotion is allowed as long as + another admin remains. Admin status is re-checked live on every + request, so unlike delete/rename no session or token revocation is + needed — a demoted admin simply fails the next is_admin() gate. + + Promotion stashes the user's current privilege map and demotion + restores it, so a temporary admin stint can't silently broaden a + user's non-admin access; users without a stash (created as admin, + or promoted before stashing existed) demote to DEFAULT_PRIVILEGES. + + Counting admins and flipping the flag happen in one critical section + so two concurrent demotions can't race the admin count to zero. + """ + username = (username or "").strip().lower() + requesting_user = (requesting_user or "").strip().lower() + is_admin = bool(is_admin) + with self._config_lock: + target = self._config.get("users", {}).get(username) + if target is None: + return SetAdminResult.USER_NOT_FOUND + if not self.users.get(requesting_user, {}).get("is_admin"): + return SetAdminResult.NOT_AUTHORIZED + currently_admin = bool(target.get("is_admin")) + if currently_admin == is_admin: + return SetAdminResult.OK # no-op; leave privileges untouched + if currently_admin and not is_admin: + admin_count = sum(1 for d in self.users.values() if d.get("is_admin")) + if admin_count <= 1: + return SetAdminResult.LAST_ADMIN + # Write order matters for lock-free readers: get_privileges() + # reads without _config_lock and trusts is_admin, so the admin + # flag must be flipped while the stored map is safe to expose — + # before writing admin privileges on promote, after restoring + # the pre-admin map on demote. + if is_admin: + target["is_admin"] = True + # Stash the pre-admin map so a later demotion can restore it. + # While is_admin is set the stored map is inert: get_privileges + # short-circuits to ADMIN_PRIVILEGES and set_privileges refuses + # admins, so only set_admin ever touches the stash. + target["privileges_before_admin"] = dict( + target.get("privileges") or DEFAULT_PRIVILEGES + ) + target["privileges"] = dict(ADMIN_PRIVILEGES) + else: + # Restore the stashed pre-admin map. Fall back to defaults for + # users created as admins (their stored map is ADMIN_PRIVILEGES, + # which must not leak past demotion — e.g. can_use_bash) and + # for admins promoted before the stash existed. + target["privileges"] = dict( + target.pop("privileges_before_admin", None) + or DEFAULT_PRIVILEGES + ) + target["is_admin"] = False + self._save() + logger.info("Set is_admin=%s for '%s' (by '%s')", is_admin, username, requesting_user) + return SetAdminResult.OK + def change_password(self, username: str, current_password: str, new_password: str) -> bool: username = username.strip().lower() if username not in self.users: return False if not _verify_password(current_password, self.users[username]["password_hash"]): return False - self._config["users"][username]["password_hash"] = _hash_password(new_password) - self._save() + with self._config_lock: + self._config["users"][username]["password_hash"] = _hash_password(new_password) + self._save() return True # ------------------------------------------------------------------ @@ -270,8 +498,9 @@ def totp_generate_secret(self, username: str) -> Optional[str]: if username not in self.users: return None secret = pyotp.random_base32() - self._config["users"][username]["totp_secret_pending"] = secret - self._save() + with self._config_lock: + self._config["users"][username]["totp_secret_pending"] = secret + self._save() return secret def totp_get_provisioning_uri(self, username: str, secret: str) -> str: @@ -290,13 +519,14 @@ def totp_confirm_enable(self, username: str, code: str) -> bool: if not totp.verify(code, valid_window=1): return False # Enable 2FA - self._config["users"][username]["totp_secret"] = secret - self._config["users"][username]["totp_enabled"] = True - self._config["users"][username].pop("totp_secret_pending", None) - # Generate backup codes - backup = [secrets.token_hex(4) for _ in range(8)] - self._config["users"][username]["totp_backup_codes"] = backup - self._save() + with self._config_lock: + self._config["users"][username]["totp_secret"] = secret + self._config["users"][username]["totp_enabled"] = True + self._config["users"][username].pop("totp_secret_pending", None) + # Generate backup codes + backup = [secrets.token_hex(4) for _ in range(8)] + self._config["users"][username]["totp_backup_codes"] = backup + self._save() logger.info(f"2FA enabled for '{username}'") return True @@ -308,13 +538,17 @@ def totp_verify(self, username: str, code: str) -> bool: return True # 2FA not enabled, always pass secret = user.get("totp_secret") if not secret: - return True + # 2FA is enabled but no secret is stored (corrupt/partially-written + # auth.json). Fail closed — returning True here bypassed the second + # factor entirely. + return False # Check backup codes first backup = user.get("totp_backup_codes", []) if code in backup: - backup.remove(code) - self._config["users"][username]["totp_backup_codes"] = backup - self._save() + with self._config_lock: + backup.remove(code) + self._config["users"][username]["totp_backup_codes"] = backup + self._save() logger.info(f"Backup code used for '{username}' ({len(backup)} remaining)") return True totp = pyotp.TOTP(secret) @@ -325,11 +559,12 @@ def totp_disable(self, username: str, password: str) -> bool: username = username.strip().lower() if not self.verify_password(username, password): return False - self._config["users"][username].pop("totp_secret", None) - self._config["users"][username].pop("totp_secret_pending", None) - self._config["users"][username].pop("totp_backup_codes", None) - self._config["users"][username]["totp_enabled"] = False - self._save() + with self._config_lock: + self._config["users"][username].pop("totp_secret", None) + self._config["users"][username].pop("totp_secret_pending", None) + self._config["users"][username].pop("totp_backup_codes", None) + self._config["users"][username]["totp_enabled"] = False + self._save() logger.info(f"2FA disabled for '{username}'") return True @@ -348,12 +583,22 @@ def create_session(self, username: str, password: str) -> Optional[str]: username = username.strip().lower() if not self.verify_password(username, password): return None + return self.create_session_trusted(username) + + def create_session_trusted(self, username: str) -> Optional[str]: + """Issue a session token for an already-verified user. + Call only after verify_password (and TOTP if enabled) have passed.""" + username = username.strip().lower() token = secrets.token_hex(32) - with self._sessions_lock: - self._sessions[token] = { - "username": username, - "expiry": time.time() + TOKEN_TTL, - } + with self._config_lock: + if username not in self.users: + logger.warning("Refused to issue session for missing user '%s'", username) + return None + with self._sessions_lock: + self._sessions[token] = { + "username": username, + "expiry": time.time() + TOKEN_TTL, + } self._save_sessions() return token @@ -412,6 +657,22 @@ def revoke_token(self, token: str): self._sessions.pop(token, None) self._save_sessions() + def revoke_user_sessions(self, username: str, except_token: Optional[str] = None) -> int: + """Revoke active browser sessions for a user, optionally preserving one.""" + username = username.strip().lower() + revoked = 0 + with self._sessions_lock: + to_drop = [ + token for token, session in self._sessions.items() + if token != except_token and (session or {}).get("username") == username + ] + for token in to_drop: + self._sessions.pop(token, None) + revoked += 1 + if revoked: + self._save_sessions() + return revoked + def status(self, token: Optional[str]) -> Dict[str, Any]: username = self.get_username_for_token(token) authenticated = username is not None diff --git a/core/constants.py b/core/constants.py index 5dcf9e91e..d71bb0aed 100644 --- a/core/constants.py +++ b/core/constants.py @@ -1,40 +1,12 @@ -# src/constants.py -"""Application-wide constants and configuration values.""" -import os - -APP_VERSION = "0.9.1" - -# Base paths -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/" -STATIC_DIR = os.path.join(BASE_DIR, "static") -DATA_DIR = os.path.join(BASE_DIR, "data") - -# Data file paths -SESSIONS_FILE = os.path.join(DATA_DIR, "sessions.json") -MEMORY_FILE = os.path.join(DATA_DIR, "memory.json") -MEMORY_DOC = os.path.join(DATA_DIR, "memory_doc.md") -PERSONAL_DIR = os.path.join(DATA_DIR, "personal_docs") -RUNBOOK_DIR = os.path.join(PERSONAL_DIR, "runbook") -UPLOAD_DIR = os.path.join(DATA_DIR, "uploads") -FEATURES_FILE = os.path.join(DATA_DIR, "features.json") -SETTINGS_FILE = os.path.join(DATA_DIR, "settings.json") - -# API Configuration -MAX_CONTEXT_MESSAGES = 90 -REQUEST_TIMEOUT = 20 -OPENAI_COMPAT_PATH = "/v1/chat/completions" - -# Environment variables with defaults -DEFAULT_HOST = os.getenv("LLM_HOST", "localhost") -LLM_HOSTS = [h.strip() for h in os.getenv("LLM_HOSTS", "").split(",") if h.strip()] -OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") -SEARXNG_INSTANCE = os.getenv('SEARXNG_INSTANCE', 'http://localhost:8080') - - -# Cleanup configuration -CLEANUP_ENABLED = os.getenv("CLEANUP_ENABLED", "True").lower() == "true" -CLEANUP_INTERVAL_HOURS = int(os.getenv("CLEANUP_INTERVAL_HOURS", "24")) - -# Default parameters -DEFAULT_TEMPERATURE = 1.0 -DEFAULT_MAX_TOKENS = 0 +# core/constants.py +"""Backward-compatible shim — the single source of truth is src/constants.py. + +Historically there were two copies of this module (this one lagged behind at +APP_VERSION 0.9.1 and was missing the consolidated tool-output constants). To +kill the drift, this now simply re-exports everything from src.constants so +there is exactly one place that defines paths and reads ODYSSEUS_DATA_DIR. +internal_api_base() also lives in src.constants now and is re-exported here so +existing `from core.constants import internal_api_base` callers keep working. +""" +from src.constants import * # noqa: F401,F403 +from src.constants import internal_api_base # noqa: F401 (explicit: functions aren't covered by some linters' * checks) diff --git a/core/database.py b/core/database.py index 10d99f50f..a9ad90b8b 100644 --- a/core/database.py +++ b/core/database.py @@ -1,28 +1,76 @@ import os import logging -from datetime import datetime -from sqlalchemy import create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text +import sqlite3 +from datetime import datetime, timezone +from pathlib import Path +from typing import Optional +from urllib.parse import unquote, urlparse +from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text +from sqlalchemy.engine import Engine, make_url from sqlalchemy.types import TypeDecorator from sqlalchemy.ext.declarative import declarative_base, declared_attr from sqlalchemy.orm import relationship, sessionmaker, backref +from src.runtime_paths import get_app_root +from core.platform_compat import safe_chmod, IS_WINDOWS + logger = logging.getLogger(__name__) # Create base class for declarative models Base = declarative_base() + +def utcnow_naive() -> datetime: + """Return naive UTC for existing DateTime columns.""" + return datetime.now(timezone.utc).replace(tzinfo=None) + + class TimestampMixin: """Mixin that adds timestamp fields to models""" @declared_attr def created_at(cls): - return Column(DateTime, default=datetime.utcnow, nullable=False) + return Column(DateTime, default=utcnow_naive, nullable=False) @declared_attr def updated_at(cls): - return Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False) + return Column(DateTime, default=utcnow_naive, onupdate=utcnow_naive, nullable=False) + +# Ensure the writable data directory exists before SQLite connects. +from src.constants import DATA_DIR, AUTH_FILE, MEMORY_FILE, USER_PREFS_FILE, SETTINGS_FILE +Path(DATA_DIR).mkdir(parents=True, exist_ok=True) + + +def _default_database_url() -> str: + return f"sqlite:///{Path(DATA_DIR) / 'app.db'}" + + +def _normalize_sqlite_url(url: str) -> str: + """Resolve relative ordinary SQLite paths without rewriting URI filenames.""" + try: + parsed = make_url(url) + except Exception: + return url + + if parsed.get_backend_name() != "sqlite": + return url + + db_path = parsed.database + if ( + not db_path + or db_path == ":memory:" + or str(db_path).lower().startswith("file:") + or os.path.isabs(str(db_path)) + ): + return url + + absolute_path = (Path(get_app_root()) / str(db_path)).resolve().as_posix() + return parsed.set(database=absolute_path).render_as_string( + hide_password=False + ) + -# Get database URL from environment, default to SQLite -DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./data/app.db") +# Get database URL from environment, default to SQLite in DATA_DIR +DATABASE_URL = _normalize_sqlite_url(os.getenv("DATABASE_URL", _default_database_url())) # Create engine engine = create_engine( @@ -30,10 +78,75 @@ def updated_at(cls): connect_args={"check_same_thread": False} if "sqlite" in DATABASE_URL else {} ) + +# Sidecar files SQLite can create next to the main DB. -journal is the default +# rollback journal; -wal/-shm appear once WAL is enabled. Each can hold copies of +# secret-bearing pages, so they get the same 0o600 lockdown as the DB itself. +_SQLITE_SIDECARS = ("-journal", "-wal", "-shm") + + +def _sqlite_db_path(url) -> Optional[str]: + """Return the filesystem path for a file-backed SQLite URL. + + SQLite query parameters such as ``mode=memory`` only affect filename + semantics when SQLAlchemy enables URI handling with ``uri=true``. Ordinary + file URLs must therefore remain file-backed even when they contain a query + parameter named ``mode``. + + For SQLite ``file:`` URIs, an empty authority or ``localhost`` identifies a + local path. Other authorities are retained as UNC-style paths. + """ + if url.get_backend_name() != "sqlite": + return None + + db_path = url.database + if not db_path or db_path == ":memory:": + return None + + db_path = str(db_path) + query = { + str(key).lower(): str(value).strip().lower() + for key, value in dict(getattr(url, "query", {}) or {}).items() + } + uri_enabled = query.get("uri") in {"1", "true", "yes", "on"} + is_file_uri = db_path.lower().startswith("file:") + + if not uri_enabled or not is_file_uri: + return db_path + + if ( + db_path.lower().startswith("file::memory:") + or query.get("mode") == "memory" + ): + return None + + parsed = urlparse(db_path) + fs_path = parsed.path or "" + if not fs_path or fs_path == ":memory:": + return None + + authority = parsed.netloc + if authority and authority.lower() != "localhost": + fs_path = f"//{authority}{fs_path}" + + return unquote(fs_path) + # Create session factory SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +# Listening on the Engine class ensures this listener fires for all Engine +# instances created within the process, not just the primary application engine. +# The isinstance(sqlite3.Connection) check ensures that this PRAGMA foreign_keys=ON +# configuration remains a no-op when using non-SQLite database backends. +@event.listens_for(Engine, "connect") +def set_sqlite_pragma(dbapi_connection, connection_record): + if isinstance(dbapi_connection, sqlite3.Connection): + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + class EncryptedText(TypeDecorator): """Text column transparently encrypted at rest via src.secret_storage. @@ -157,7 +270,7 @@ class ChatMessage(Base): meta_data = Column("metadata", Text, nullable=True) # JSON string for metrics etc. # Timestamp - timestamp = Column(DateTime, default=datetime.utcnow) + timestamp = Column(DateTime, default=utcnow_naive) # Relationship to Session session = relationship("Session", back_populates="messages") @@ -210,7 +323,7 @@ class DocumentVersion(Base): content = Column(Text, nullable=False) summary = Column(String, nullable=True) # Edit description source = Column(String, default="ai") # "ai" or "user" - created_at = Column(DateTime, default=datetime.utcnow) + created_at = Column(DateTime, default=utcnow_naive) document = relationship("Document", back_populates="versions") @@ -235,6 +348,7 @@ class GalleryImage(TimestampMixin, Base): id = Column(String, primary_key=True, index=True) filename = Column(String, nullable=False, unique=True) prompt = Column(Text, nullable=False, default="") + caption = Column(Text, nullable=True, default="") model = Column(String, nullable=True) size = Column(String, nullable=True) quality = Column(String, nullable=True) @@ -298,10 +412,18 @@ class EmailAccount(TimestampMixin, Base): # SMTP (sending) smtp_host = Column(String, default="") smtp_port = Column(Integer, default=465) + smtp_security = Column(String, default="ssl") # ssl | starttls | none smtp_user = Column(String, default="") smtp_password = Column(String, default="") from_address = Column(String, default="") + display_name = Column(String, nullable=True) # "Hriday Ranka" — used in From: header + + # OAuth2 (Google / Google Workspace). Tokens stored encrypted via secret_storage. + oauth_provider = Column(String, nullable=True) # "google" or None + oauth_access_token = Column(String, nullable=True) # encrypted + oauth_refresh_token = Column(String, nullable=True) # encrypted + oauth_token_expiry = Column(String, nullable=True) # unix timestamp string __table_args__ = ( Index('ix_email_accounts_owner_default', 'owner', 'is_default'), @@ -319,7 +441,16 @@ class ModelEndpoint(TimestampMixin, Base): is_enabled = Column(Boolean, default=True) hidden_models = Column(Text, nullable=True) # JSON list of model IDs that failed probing cached_models = Column(Text, nullable=True) # JSON list of last-known model IDs (avoids probe on list) + pinned_models = Column(Text, nullable=True) # JSON list of admin-pinned model IDs (manual, may not appear in /v1/models) model_type = Column(String, nullable=True, default="llm") # "llm" or "image" + # auto = classify by URL; local = self-hosted server; api/proxy = external + # OpenAI-compatible API even when reachable through a private/tailnet IP. + endpoint_kind = Column(String, nullable=True, default="auto") + # auto = background refresh with TTL/backoff; manual/disabled = cached-first + # only unless an explicit endpoint probe is requested. + model_refresh_mode = Column(String, nullable=True, default="auto") + model_refresh_interval = Column(Integer, nullable=True, default=None) + model_refresh_timeout = Column(Integer, nullable=True, default=None) # Whether models on this endpoint accept OpenAI-style function # schemas + emit `tool_calls`. Auto-detected at Cookbook auto- # register time from `--enable-auto-tool-choice` in the serve cmd; @@ -330,6 +461,24 @@ class ModelEndpoint(TimestampMixin, Base): # is the historical default. When non-null, the model picker only shows # the endpoint to that user (admins always see everything). owner = Column(String, nullable=True, index=True) + # Optional OAuth/session-backed credential row. Used by subscription-backed + # providers that need refresh tokens instead of a static API key. + provider_auth_id = Column(String, nullable=True, index=True) + + +class ProviderAuthSession(TimestampMixin, Base): + """Encrypted OAuth/session credentials for refresh-aware model providers.""" + __tablename__ = "provider_auth_sessions" + + id = Column(String, primary_key=True, index=True) + provider = Column(String, nullable=False, index=True) + owner = Column(String, nullable=True, index=True) + label = Column(String, nullable=True) + base_url = Column(String, nullable=False) + access_token = Column(EncryptedText, nullable=True) + refresh_token = Column(EncryptedText, nullable=True) + last_refresh = Column(DateTime, nullable=True) + auth_mode = Column(String, nullable=True) class McpServer(TimestampMixin, Base): """Admin-configured MCP (Model Context Protocol) tool servers.""" @@ -345,6 +494,7 @@ class McpServer(TimestampMixin, Base): is_enabled = Column(Boolean, default=True) oauth_config = Column(Text, nullable=True) # JSON: provider, keys_file, token_file, scopes disabled_tools = Column(Text, nullable=True) # JSON array of tool names to hide from LLM + oauth_tokens = Column(EncryptedText, nullable=True) # JSON {tokens, client_info} for generic MCP OAuth, encrypted at rest class Comparison(TimestampMixin, Base): @@ -456,8 +606,8 @@ class UserToolData(Base): tool_id = Column(String, ForeignKey("user_tools.id", ondelete="CASCADE"), nullable=False) key = Column(String, nullable=False) value = Column(Text, nullable=True) - created_at = Column(DateTime, default=datetime.utcnow) - updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) + created_at = Column(DateTime, default=utcnow_naive) + updated_at = Column(DateTime, default=utcnow_naive, onupdate=utcnow_naive) tool = relationship("UserTool", backref=backref("data_entries", cascade="all, delete-orphan")) @@ -576,7 +726,7 @@ class TaskRun(Base): id = Column(String, primary_key=True, index=True) task_id = Column(String, ForeignKey("scheduled_tasks.id", ondelete="CASCADE"), nullable=False) - started_at = Column(DateTime, nullable=False, default=datetime.utcnow) + started_at = Column(DateTime, nullable=False, default=utcnow_naive) finished_at = Column(DateTime, nullable=True) status = Column(String, default="running") # "running", "success", "error" result = Column(Text, nullable=True) @@ -617,7 +767,7 @@ class Memory(Base): session_id = Column(String, ForeignKey("sessions.id", ondelete="SET NULL"), nullable=True, index=True) # Timestamp as Unix timestamp - timestamp = Column(Integer, default=lambda: int(datetime.utcnow().timestamp())) + timestamp = Column(Integer, default=lambda: int(utcnow_naive().timestamp())) # Relationship to Session session = relationship("Session", backref="memories") @@ -638,6 +788,7 @@ def _migrate_add_last_message_at_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(sessions)") @@ -663,10 +814,14 @@ def _migrate_add_last_message_at_column(): "ON sessions(archived, last_message_at)" ) conn.commit() - conn.close() logging.getLogger(__name__).info("Migrated: added + backfilled 'last_message_at' on sessions") except Exception as e: logging.getLogger(__name__).warning(f"last_message_at migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_document_archived_column(): """Add `archived` to documents (soft-archive flag). Guarded + idempotent.""" @@ -674,6 +829,7 @@ def _migrate_add_document_archived_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(documents)") @@ -682,9 +838,13 @@ def _migrate_add_document_archived_column(): conn.execute("ALTER TABLE documents ADD COLUMN archived BOOLEAN DEFAULT 0") conn.commit() logging.getLogger(__name__).info("Migrated: added 'archived' to documents") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"documents.archived migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_owner_column(): @@ -693,6 +853,7 @@ def _migrate_add_owner_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(sessions)") @@ -702,9 +863,13 @@ def _migrate_add_owner_column(): conn.execute("CREATE INDEX IF NOT EXISTS ix_sessions_owner ON sessions(owner)") conn.commit() logging.getLogger(__name__).info("Migrated: added 'owner' column to sessions") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"Migration check failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_model_endpoints(): """Recreate model_endpoints table if schema changed (url->base_url).""" @@ -712,6 +877,7 @@ def _migrate_model_endpoints(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(model_endpoints)") @@ -720,9 +886,13 @@ def _migrate_model_endpoints(): conn.execute("DROP TABLE IF EXISTS model_endpoints") conn.commit() logging.getLogger(__name__).info("Migrated: dropped old model_endpoints table (schema change)") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"model_endpoints migration check failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_hidden_models_column(): """Add hidden_models column to model_endpoints if it doesn't exist.""" @@ -730,6 +900,7 @@ def _migrate_add_hidden_models_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(model_endpoints)") @@ -738,9 +909,13 @@ def _migrate_add_hidden_models_column(): conn.execute("ALTER TABLE model_endpoints ADD COLUMN hidden_models TEXT") conn.commit() logging.getLogger(__name__).info("Migrated: added 'hidden_models' column to model_endpoints") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"hidden_models migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_model_endpoint_owner_column(): """Add owner column to model_endpoints if it doesn't exist. @@ -755,6 +930,7 @@ def _migrate_add_model_endpoint_owner_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(model_endpoints)") @@ -764,9 +940,38 @@ def _migrate_add_model_endpoint_owner_column(): conn.execute("CREATE INDEX IF NOT EXISTS ix_model_endpoints_owner ON model_endpoints(owner)") conn.commit() logging.getLogger(__name__).info("Migrated: added 'owner' column + index to model_endpoints") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"model_endpoints.owner migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass + + +def _migrate_add_provider_auth_id_column(): + """Add provider_auth_id column to model_endpoints if it doesn't exist.""" + import sqlite3 + db_path = DATABASE_URL.replace("sqlite:///", "") + if not os.path.exists(db_path): + return + conn = None + try: + conn = sqlite3.connect(db_path) + cursor = conn.execute("PRAGMA table_info(model_endpoints)") + columns = [row[1] for row in cursor.fetchall()] + if columns and "provider_auth_id" not in columns: + conn.execute("ALTER TABLE model_endpoints ADD COLUMN provider_auth_id VARCHAR") + conn.execute("CREATE INDEX IF NOT EXISTS ix_model_endpoints_provider_auth_id ON model_endpoints(provider_auth_id)") + conn.commit() + logging.getLogger(__name__).info("Migrated: added 'provider_auth_id' column + index to model_endpoints") + except Exception as e: + logging.getLogger(__name__).warning(f"model_endpoints.provider_auth_id migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_model_type_column(): @@ -775,6 +980,7 @@ def _migrate_add_model_type_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(model_endpoints)") @@ -783,9 +989,41 @@ def _migrate_add_model_type_column(): conn.execute("ALTER TABLE model_endpoints ADD COLUMN model_type TEXT DEFAULT 'llm'") conn.commit() logging.getLogger(__name__).info("Migrated: added 'model_type' column to model_endpoints") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"model_type migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass + +def _migrate_add_model_endpoint_refresh_columns(): + """Add endpoint classification / refresh policy columns if missing.""" + import sqlite3 + db_path = DATABASE_URL.replace("sqlite:///", "") + if not os.path.exists(db_path): + return + conn = None + try: + conn = sqlite3.connect(db_path) + cursor = conn.execute("PRAGMA table_info(model_endpoints)") + columns = [row[1] for row in cursor.fetchall()] + if columns and "endpoint_kind" not in columns: + conn.execute("ALTER TABLE model_endpoints ADD COLUMN endpoint_kind TEXT DEFAULT 'auto'") + if columns and "model_refresh_mode" not in columns: + conn.execute("ALTER TABLE model_endpoints ADD COLUMN model_refresh_mode TEXT DEFAULT 'auto'") + if columns and "model_refresh_interval" not in columns: + conn.execute("ALTER TABLE model_endpoints ADD COLUMN model_refresh_interval INTEGER") + if columns and "model_refresh_timeout" not in columns: + conn.execute("ALTER TABLE model_endpoints ADD COLUMN model_refresh_timeout INTEGER") + conn.commit() + except Exception as e: + logging.getLogger(__name__).warning(f"model_endpoints refresh-policy migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_task_run_model_column(): """Add model column to task_runs if it doesn't exist (records which model ran).""" @@ -793,6 +1031,7 @@ def _migrate_add_task_run_model_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(task_runs)") @@ -801,9 +1040,13 @@ def _migrate_add_task_run_model_column(): conn.execute("ALTER TABLE task_runs ADD COLUMN model TEXT") conn.commit() logging.getLogger(__name__).info("Migrated: added 'model' column to task_runs") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"task_runs model migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_supports_tools_column(): """Add supports_tools column to model_endpoints if it doesn't exist.""" @@ -811,6 +1054,7 @@ def _migrate_add_supports_tools_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(model_endpoints)") @@ -819,9 +1063,13 @@ def _migrate_add_supports_tools_column(): conn.execute("ALTER TABLE model_endpoints ADD COLUMN supports_tools BOOLEAN") conn.commit() logging.getLogger(__name__).info("Migrated: added 'supports_tools' column to model_endpoints") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"supports_tools migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_cached_models_column(): @@ -830,6 +1078,7 @@ def _migrate_add_cached_models_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(model_endpoints)") @@ -837,9 +1086,36 @@ def _migrate_add_cached_models_column(): if columns and "cached_models" not in columns: conn.execute("ALTER TABLE model_endpoints ADD COLUMN cached_models TEXT") conn.commit() - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"cached_models migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass + +def _migrate_add_pinned_models_column(): + """Add pinned_models column to model_endpoints if it doesn't exist.""" + import sqlite3 + db_path = DATABASE_URL.replace("sqlite:///", "") + if not os.path.exists(db_path): + return + conn = None + try: + conn = sqlite3.connect(db_path) + cursor = conn.execute("PRAGMA table_info(model_endpoints)") + columns = [row[1] for row in cursor.fetchall()] + if columns and "pinned_models" not in columns: + conn.execute("ALTER TABLE model_endpoints ADD COLUMN pinned_models TEXT") + conn.commit() + logging.getLogger(__name__).info("Migrated: added 'pinned_models' column to model_endpoints") + except Exception as e: + logging.getLogger(__name__).warning(f"pinned_models migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_notes_sort_order(): """Add sort_order, image_url, repeat columns to notes if they don't exist.""" @@ -847,6 +1123,7 @@ def _migrate_add_notes_sort_order(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(notes)") @@ -864,9 +1141,13 @@ def _migrate_add_notes_sort_order(): if columns and "agent_session_id" not in columns: conn.execute("ALTER TABLE notes ADD COLUMN agent_session_id TEXT") conn.commit() - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"notes migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_mode_column(): """Add mode column to sessions table if it doesn't exist.""" @@ -874,6 +1155,7 @@ def _migrate_add_mode_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(sessions)") @@ -882,9 +1164,13 @@ def _migrate_add_mode_column(): conn.execute("ALTER TABLE sessions ADD COLUMN mode TEXT") conn.commit() logging.getLogger(__name__).info("Migrated: added 'mode' column to sessions") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"Migration check for mode failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_folder_column(): """Add folder column to sessions table if it doesn't exist.""" @@ -892,6 +1178,7 @@ def _migrate_add_folder_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(sessions)") @@ -900,9 +1187,13 @@ def _migrate_add_folder_column(): conn.execute("ALTER TABLE sessions ADD COLUMN folder TEXT") conn.commit() logging.getLogger(__name__).info("Migrated: added 'folder' column to sessions") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"Migration check for folder failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_token_columns(): """Add cumulative token tracking columns to sessions table.""" @@ -910,6 +1201,7 @@ def _migrate_add_token_columns(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(sessions)") @@ -919,9 +1211,13 @@ def _migrate_add_token_columns(): conn.execute("ALTER TABLE sessions ADD COLUMN total_output_tokens INTEGER DEFAULT 0") conn.commit() logging.getLogger(__name__).info("Migrated: added token tracking columns to sessions") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"Migration check for token columns failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_owner_to_table(table_name: str, index_name: str): """Generic helper: add owner TEXT column + index to a table if missing.""" @@ -929,6 +1225,7 @@ def _migrate_add_owner_to_table(table_name: str, index_name: str): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute(f"PRAGMA table_info({table_name})") @@ -938,9 +1235,13 @@ def _migrate_add_owner_to_table(table_name: str, index_name: str): conn.execute(f"CREATE INDEX IF NOT EXISTS {index_name} ON {table_name}(owner)") conn.commit() logging.getLogger(__name__).info(f"Migrated: added 'owner' column to {table_name}") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"Migration owner column for {table_name} failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_add_multiuser_owner_columns(): """Add owner column to memories, gallery_images, user_tools, comparisons.""" @@ -954,6 +1255,29 @@ def _migrate_add_multiuser_owner_columns(): _migrate_add_owner_to_table("documents", "ix_documents_owner") +def _migrate_add_gallery_caption_column(): + """Add OCR/vision caption storage for gallery images.""" + import sqlite3 + db_path = DATABASE_URL.replace("sqlite:///", "") + if not os.path.exists(db_path): + return + conn = None + try: + conn = sqlite3.connect(db_path) + columns = [row[1] for row in conn.execute("PRAGMA table_info(gallery_images)").fetchall()] + if columns and "caption" not in columns: + conn.execute("ALTER TABLE gallery_images ADD COLUMN caption TEXT DEFAULT ''") + conn.commit() + logging.getLogger(__name__).info("Migrated: added caption column to gallery_images") + except Exception as e: + logging.getLogger(__name__).warning(f"Migration gallery caption column failed: {e}") + finally: + try: + conn.close() + except Exception: + pass + + def _migrate_add_api_token_scopes_column(): """Add API token scopes for existing installs. @@ -965,6 +1289,7 @@ def _migrate_add_api_token_scopes_column(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) columns = [row[1] for row in conn.execute("PRAGMA table_info(api_tokens)").fetchall()] @@ -973,9 +1298,13 @@ def _migrate_add_api_token_scopes_column(): conn.execute("UPDATE api_tokens SET scopes = 'chat' WHERE scopes IS NULL OR scopes = ''") conn.commit() logging.getLogger(__name__).info("Migrated: added scopes column to api_tokens") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"api_tokens.scopes migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_assign_legacy_owner(): """Assign all null-owner data to the first (admin) user. @@ -993,10 +1322,10 @@ def _migrate_assign_legacy_owner(): # fell through to "first user" every time. auth_path = os.path.join(os.path.dirname(DATABASE_URL.replace("sqlite:///", "")), "auth.json") if not os.path.isabs(auth_path): - auth_path = os.path.join("data", "auth.json") + auth_path = AUTH_FILE admin_user = None try: - with open(auth_path, "r") as f: + with open(auth_path, "r", encoding="utf-8") as f: auth_data = _json.load(f) users = auth_data.get("users", {}) if users: @@ -1017,6 +1346,7 @@ def _migrate_assign_legacy_owner(): return logger = logging.getLogger(__name__) + conn = None try: conn = sqlite3.connect(db_path) # Every table with an `owner` column. New tables added later will be @@ -1041,12 +1371,16 @@ def _migrate_assign_legacy_owner(): except Exception as e: logger.warning(f"Legacy owner assignment for {table} failed: {e}") conn.commit() - conn.close() except Exception as e: logger.warning(f"Legacy owner migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass # Also migrate memory.json - mem_path = os.path.join("data", "memory.json") + mem_path = MEMORY_FILE try: if os.path.exists(mem_path): with open(mem_path, "r", encoding="utf-8") as f: @@ -1064,15 +1398,15 @@ def _migrate_assign_legacy_owner(): logger.warning(f"memory.json legacy migration failed: {e}") # Also migrate user_prefs.json to per-user format - prefs_path = os.path.join("data", "user_prefs.json") + prefs_path = USER_PREFS_FILE try: if os.path.exists(prefs_path): - with open(prefs_path, "r") as f: + with open(prefs_path, "r", encoding="utf-8") as f: prefs = _json.load(f) if "_users" not in prefs and prefs: # Flat format → nest under admin user new_prefs = {"_users": {admin_user: prefs}} - with open(prefs_path, "w") as f: + with open(prefs_path, "w", encoding="utf-8") as f: _json.dump(new_prefs, f, indent=2) logger.info(f"Migrated user_prefs.json to per-user format under '{admin_user}'") except Exception as e: @@ -1216,6 +1550,25 @@ def _migrate_add_task_automation_columns(): except Exception as e: logging.getLogger(__name__).warning(f"task automation migration: {e}") +def _migrate_add_email_oauth_columns(): + """Add Google OAuth and display_name columns to email_accounts if missing.""" + try: + with engine.connect() as conn: + cols = [r[1] for r in conn.execute(text("PRAGMA table_info(email_accounts)"))] + for col, typedef in [ + ("oauth_provider", "TEXT"), + ("oauth_access_token", "TEXT"), + ("oauth_refresh_token", "TEXT"), + ("oauth_token_expiry", "TEXT"), + ("display_name", "TEXT"), + ]: + if col not in cols: + conn.execute(text(f"ALTER TABLE email_accounts ADD COLUMN {col} {typedef}")) + conn.commit() + except Exception as e: + logging.getLogger(__name__).warning(f"email oauth columns migration: {e}") + + def _migrate_add_oauth_config(): """Add oauth_config column to mcp_servers table if missing.""" try: @@ -1240,6 +1593,23 @@ def _migrate_add_disabled_tools(): except Exception as e: logging.getLogger(__name__).warning(f"disabled_tools migration: {e}") +def _migrate_add_mcp_oauth_tokens_column(): + """Add oauth_tokens column to mcp_servers table if missing. + + The model declares this column as EncryptedText, but the SQL type is plain + TEXT on purpose: EncryptedText is a SQLAlchemy TypeDecorator that encrypts at + the Python layer and stores the ciphertext as TEXT, so the DB column type is + TEXT. This matches the existing encrypted columns (see _migrate_encrypt_*).""" + try: + with engine.connect() as conn: + cols = [r[1] for r in conn.execute(text("PRAGMA table_info(mcp_servers)"))] + if "oauth_tokens" not in cols: + conn.execute(text("ALTER TABLE mcp_servers ADD COLUMN oauth_tokens TEXT")) + conn.commit() + logging.getLogger(__name__).info("Added oauth_tokens column to mcp_servers") + except Exception as e: + logging.getLogger(__name__).warning(f"oauth_tokens migration: {e}") + def _migrate_add_task_v2_columns(): """Add cron_expression, then_task_id, webhook_token to scheduled_tasks.""" new_cols = { @@ -1369,7 +1739,12 @@ class CalendarCal(TimestampMixin, Base): owner = Column(String, nullable=True, index=True) name = Column(String, nullable=False) color = Column(String, default="#5b8abf") - source = Column(String, default="local") # "local" or "timetree" + source = Column(String, default="local") # "local" or "caldav" + # UUID of the CalDAV account in user prefs that owns this calendar. + # NULL for local calendars and for CalDAV calendars created before + # multi-account support was added (treated as "use any configured account"). + account_id = Column(String, nullable=True, index=True) + caldav_base_url = Column(String, nullable=True) events = relationship("CalendarEvent", back_populates="calendar", cascade="all, delete-orphan") @@ -1391,15 +1766,37 @@ class CalendarEvent(TimestampMixin, Base): # `Z`-suffix on serialization so the frontend interprets correctly. is_utc = Column(Boolean, default=False, nullable=False) rrule = Column(String, default="") + recurrence_exdates = Column(Text, default="") # JSON list of skipped occurrence starts color = Column(String, nullable=True) # per-event color override status = Column(String, default="confirmed") # confirmed, cancelled importance = Column(String, default="normal") # low | normal | high | critical event_type = Column(String, nullable=True) # work | personal | health | travel | meal | social | admin | other last_pinged = Column(DateTime, nullable=True) # last time the assistant pinged about this event + # "caldav" = pulled from a CalDAV server (so the sync may prune it when it + # vanishes upstream). NULL/local = created locally (agent, email triage, or + # a UI event whose write-back failed) and must NOT be pruned by the sync. + origin = Column(String, nullable=True, index=True) + remote_href = Column(String, nullable=True) # CalDAV object URL for updates/deletes + remote_etag = Column(String, nullable=True) # Last seen CalDAV ETag, when available + caldav_sync_pending = Column(String, nullable=True) # create | update | delete retry marker calendar = relationship("CalendarCal", back_populates="events") +class CalendarDeletedEvent(TimestampMixin, Base): + """Hidden CalDAV delete tombstone retained until remote delete succeeds.""" + __tablename__ = "caldav_deleted_events" + + uid = Column(String, primary_key=True, index=True) + owner = Column(String, nullable=True, index=True) + calendar_id = Column(String, nullable=True, index=True) + remote_href = Column(String, nullable=True) + remote_etag = Column(String, nullable=True) + caldav_base_url = Column(String, nullable=True) + summary = Column(String, nullable=True) + last_error = Column(Text, nullable=True) + + class Integration(TimestampMixin, Base): """An external service connection (email, RSS, webhook, etc.).""" __tablename__ = "integrations" @@ -1433,11 +1830,11 @@ def _migrate_seed_email_account(): import json as _json import uuid as _uuid from pathlib import Path - settings_file = Path("data/settings.json") + settings_file = Path(SETTINGS_FILE) if not settings_file.exists(): return try: - s = _json.loads(settings_file.read_text()) + s = _json.loads(settings_file.read_text(encoding="utf-8")) except Exception: return @@ -1446,7 +1843,7 @@ def _migrate_seed_email_account(): if not imap_host and not smtp_host: return # nothing to migrate - now = datetime.utcnow() + now = utcnow_naive() with engine.begin() as conn: conn.execute(text(""" INSERT INTO email_accounts @@ -1483,6 +1880,10 @@ def _migrate_seed_email_account(): logging.getLogger(__name__).warning(f"seed email account migration: {e}") +# WARNING: Foreign-key enforcement is enabled globally for all SQLite connections. +# Any future migrations or schema changes that temporarily violate foreign-key +# constraints will fail. To perform such operations, foreign_keys must be +# temporarily disabled around the migration workflow. def init_db(): """ Initialize the database by creating all tables. @@ -1490,11 +1891,49 @@ def init_db(): """ _migrate_model_endpoints() Base.metadata.create_all(bind=engine) + # Lock the DB file (and any SQLite sidecars) to 0o600 — it holds bearer-token + # + bcrypt hashes and encrypted provider keys. POSIX only; safe_chmod no-ops + # on Windows (ACL-restricted profile dir) and the path helper returns None for + # Postgres / in-memory. Must stay AFTER create_all: the file is born here at + # the umask default, and nothing below resets the mode. The path comes from + # engine.url (SQLAlchemy's parsed URL), so a driver-qualified or query-tagged + # DATABASE_URL still resolves to the real file instead of slipping through. + db_path = _sqlite_db_path(engine.url) + if db_path is not None: + # Fail closed-loud on the main file: this is the only access control on + # it, so if the chmod genuinely fails (read-only FS, foreign owner) an + # operator should hear about it. safe_chmod also returns False as a + # Windows no-op, so guard on IS_WINDOWS to avoid a spurious warning there. + if not safe_chmod(db_path, 0o600) and not IS_WINDOWS: + logger.warning( + "Could not restrict %s to 0o600; it holds secrets and may be " + "world-readable. Check filesystem permissions and ownership.", + db_path, + ) + # Re-lock any sidecars present at startup. New ones inherit the main + # file's mode (now 0o600, since we set it first), and they're usually + # absent here, but a stale -wal/-shm/-journal left by an older 0o644 + # install could still expose secret pages. Absent sidecars are the + # normal case, not an error — only a failed chmod warrants a warning. + for suffix in _SQLITE_SIDECARS: + sidecar = db_path + suffix + if ( + os.path.exists(sidecar) + and not safe_chmod(sidecar, 0o600) + and not IS_WINDOWS + ): + logger.warning( + "Could not restrict %s to 0o600; it may expose DB pages.", + sidecar, + ) _migrate_add_hidden_models_column() _migrate_add_cached_models_column() + _migrate_add_pinned_models_column() _migrate_add_notes_sort_order() _migrate_add_model_type_column() + _migrate_add_model_endpoint_refresh_columns() _migrate_add_model_endpoint_owner_column() + _migrate_add_provider_auth_id_column() _migrate_add_supports_tools_column() _migrate_add_task_run_model_column() _migrate_add_owner_column() @@ -1504,25 +1943,209 @@ def init_db(): _migrate_add_token_columns() _migrate_add_mode_column() _migrate_add_multiuser_owner_columns() + _migrate_add_gallery_caption_column() _migrate_add_api_token_scopes_column() _migrate_backfill_document_owner_from_session() _migrate_assign_legacy_owner() _migrate_add_tidy_verdict() _migrate_add_doc_source_email_cols() _migrate_add_oauth_config() + _migrate_add_email_oauth_columns() _migrate_add_task_automation_columns() _migrate_add_disabled_tools() + _migrate_add_mcp_oauth_tokens_column() _migrate_add_task_v2_columns() _migrate_add_notifications_enabled() _migrate_drop_ping_notes_tasks() _migrate_add_crew_member_id() _migrate_add_assistant_columns() + _migrate_add_email_smtp_security() _migrate_seed_email_account() _migrate_add_calendar_metadata() _migrate_add_calendar_is_utc() + _migrate_add_calendar_origin() + _migrate_add_calendar_account_id() + _migrate_add_caldav_sync_columns() + _migrate_add_calendar_recurrence_exdates() + _migrate_chat_messages_fts() _migrate_encrypt_email_passwords() _migrate_encrypt_signatures() _migrate_encrypt_endpoint_keys() + _migrate_backfill_task_folders() + + +def _migrate_backfill_task_folders(): + """Backfill folder='Tasks' on pre-existing task/research sessions. + + Sessions created by the task scheduler (LLM tasks, action tasks, research + runs) now set folder='Tasks' at creation time. This migration tags any + older sessions that predate that assignment. Idempotent — only touches + rows where folder is NULL or empty and the title matches known prefixes. + """ + try: + with engine.connect() as conn: + cols = [r[1] for r in conn.execute(text("PRAGMA table_info(sessions)"))] + if "folder" not in cols: + return + res = conn.execute(text( + "UPDATE sessions SET folder = 'Tasks' " + "WHERE (folder IS NULL OR folder = '') " + "AND (name LIKE '[Task] %' OR name LIKE '[Research] %')" + )) + conn.commit() + if res.rowcount: + logging.getLogger(__name__).info( + f"Backfilled folder='Tasks' on {res.rowcount} task/research sessions") + except Exception as e: + logging.getLogger(__name__).warning(f"task folder backfill: {e}") + + +def _migrate_chat_messages_fts(): + """Create and backfill the session transcript FTS index for SQLite.""" + if not DATABASE_URL.startswith("sqlite"): + return + + db_path = DATABASE_URL.replace("sqlite:///", "") + if db_path == ":memory:": + return + conn = None + try: + conn = sqlite3.connect(db_path) + fts_content_expr_new = ( + "CASE WHEN instr(COALESCE(new.content, ''), ';base64,') > 0 " + "OR instr(COALESCE(new.content, ''), 'data:image/') > 0 " + "OR instr(COALESCE(new.content, ''), 'data:audio/') > 0 " + "THEN '[inline media omitted from search index]' " + "ELSE COALESCE(new.content, '') END" + ) + fts_content_expr_cm = ( + "CASE WHEN instr(COALESCE(cm.content, ''), ';base64,') > 0 " + "OR instr(COALESCE(cm.content, ''), 'data:image/') > 0 " + "OR instr(COALESCE(cm.content, ''), 'data:audio/') > 0 " + "THEN '[inline media omitted from search index]' " + "ELSE COALESCE(cm.content, '') END" + ) + try: + conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS temp._odysseus_fts5_probe USING fts5(content)") + conn.execute("DROP TABLE IF EXISTS temp._odysseus_fts5_probe") + except Exception as e: + logging.getLogger(__name__).warning(f"chat_messages FTS migration skipped; FTS5 unavailable: {e}") + return + + conn.executescript( + f""" + CREATE VIRTUAL TABLE IF NOT EXISTS chat_messages_fts USING fts5( + content, + message_id UNINDEXED, + session_id UNINDEXED, + role UNINDEXED + ); + + DROP TRIGGER IF EXISTS chat_messages_fts_ai; + DROP TRIGGER IF EXISTS chat_messages_fts_ad; + DROP TRIGGER IF EXISTS chat_messages_fts_au; + + CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ai + AFTER INSERT ON chat_messages BEGIN + INSERT INTO chat_messages_fts(content, message_id, session_id, role) + VALUES ({fts_content_expr_new}, new.id, new.session_id, new.role); + END; + + CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ad + AFTER DELETE ON chat_messages BEGIN + DELETE FROM chat_messages_fts WHERE message_id = old.id; + END; + + CREATE TRIGGER IF NOT EXISTS chat_messages_fts_au + AFTER UPDATE ON chat_messages BEGIN + DELETE FROM chat_messages_fts WHERE message_id = old.id; + INSERT INTO chat_messages_fts(content, message_id, session_id, role) + VALUES ({fts_content_expr_new}, new.id, new.session_id, new.role); + END; + """ + ) + conn.execute( + f""" + INSERT INTO chat_messages_fts(content, message_id, session_id, role) + SELECT {fts_content_expr_cm}, cm.id, cm.session_id, cm.role + FROM chat_messages cm + WHERE NOT EXISTS ( + SELECT 1 FROM chat_messages_fts fts + WHERE fts.message_id = cm.id + ) + """ + ) + _scrub_legacy_chat_message_fts_media(conn) + conn.commit() + except Exception as e: + logging.getLogger(__name__).warning(f"chat_messages FTS migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass + + +def _scrub_legacy_chat_message_fts_media(conn) -> None: + """Replace already-indexed inline media rows with searchable text only.""" + try: + from src.attachment_refs import search_index_text + except Exception as e: + logging.getLogger(__name__).warning(f"chat_messages FTS media scrub skipped: {e}") + return + + try: + rows = conn.execute( + """ + SELECT id, session_id, role, content + FROM chat_messages + WHERE instr(COALESCE(content, ''), ';base64,') > 0 + OR instr(COALESCE(content, ''), 'data:image/') > 0 + OR instr(COALESCE(content, ''), 'data:audio/') > 0 + """ + ).fetchall() + for message_id, session_id, role, content in rows: + conn.execute("DELETE FROM chat_messages_fts WHERE message_id = ?", (message_id,)) + conn.execute( + """ + INSERT INTO chat_messages_fts(content, message_id, session_id, role) + VALUES (?, ?, ?, ?) + """, + (search_index_text(content), message_id, session_id, role), + ) + except Exception as e: + logging.getLogger(__name__).warning(f"chat_messages FTS media scrub failed: {e}") + + +def _migrate_add_email_smtp_security(): + """Add explicit SMTP security mode for Proton Bridge/custom local SMTP.""" + import sqlite3 + db_path = DATABASE_URL.replace("sqlite:///", "") + if not os.path.exists(db_path): + return + conn = None + try: + conn = sqlite3.connect(db_path) + cursor = conn.execute("PRAGMA table_info(email_accounts)") + columns = [row[1] for row in cursor.fetchall()] + if columns and "smtp_security" not in columns: + conn.execute("ALTER TABLE email_accounts ADD COLUMN smtp_security TEXT DEFAULT 'ssl'") + conn.execute( + "UPDATE email_accounts SET smtp_security = CASE " + "WHEN COALESCE(smtp_port, 465) = 587 THEN 'starttls' " + "WHEN COALESCE(smtp_port, 465) = 465 THEN 'ssl' " + "ELSE 'ssl' END " + "WHERE smtp_security IS NULL OR smtp_security = ''" + ) + conn.commit() + logging.getLogger(__name__).info("Migrated: added smtp_security column to email_accounts") + except Exception as e: + logging.getLogger(__name__).warning(f"smtp_security migration skipped: {e}") + finally: + try: + conn.close() + except Exception: + pass def _migrate_encrypt_endpoint_keys(): @@ -1623,6 +2246,7 @@ def _migrate_add_calendar_is_utc(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(calendar_events)") @@ -1631,9 +2255,91 @@ def _migrate_add_calendar_is_utc(): conn.execute("ALTER TABLE calendar_events ADD COLUMN is_utc BOOLEAN DEFAULT 0 NOT NULL") conn.commit() logging.getLogger(__name__).info("Migrated: added 'is_utc' column to calendar_events") - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"is_utc migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass + + +def _migrate_add_calendar_origin(): + """Add `origin` to calendar_events so the CalDAV sync can tell server-pulled + rows (prunable when they vanish upstream) from locally-created ones (agent / + email triage / failed write-back), which must never be pruned. Idempotent.""" + import sqlite3 + db_path = DATABASE_URL.replace("sqlite:///", "") + if not os.path.exists(db_path): + return + conn = None + try: + conn = sqlite3.connect(db_path) + cursor = conn.execute("PRAGMA table_info(calendar_events)") + columns = [row[1] for row in cursor.fetchall()] + if columns and "origin" not in columns: + conn.execute("ALTER TABLE calendar_events ADD COLUMN origin TEXT") + conn.execute("CREATE INDEX IF NOT EXISTS ix_calendar_events_origin ON calendar_events(origin)") + conn.commit() + logging.getLogger(__name__).info("Migrated: added 'origin' column to calendar_events") + except Exception as e: + logging.getLogger(__name__).warning(f"calendar_events.origin migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass + + +def _migrate_add_calendar_account_id(): + """Add `account_id` to calendars so each CalDAV-backed calendar knows which + credential set (from caldav_accounts in user prefs) owns it. Idempotent.""" + import sqlite3 + db_path = DATABASE_URL.replace("sqlite:///", "") + if not os.path.exists(db_path): + return + conn = None + try: + conn = sqlite3.connect(db_path) + cursor = conn.execute("PRAGMA table_info(calendars)") + columns = [row[1] for row in cursor.fetchall()] + if columns and "account_id" not in columns: + conn.execute("ALTER TABLE calendars ADD COLUMN account_id TEXT") + conn.execute("CREATE INDEX IF NOT EXISTS ix_calendars_account_id ON calendars(account_id)") + conn.commit() + logging.getLogger(__name__).info("Migrated: added 'account_id' column to calendars") + except Exception as e: + logging.getLogger(__name__).warning(f"calendars.account_id migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass + + +def _migrate_add_caldav_sync_columns(): + """Add remote CalDAV metadata used for bidirectional sync.""" + import sqlite3 + db_path = DATABASE_URL.replace("sqlite:///", "") + if not os.path.exists(db_path): + return + try: + conn = sqlite3.connect(db_path) + ev_columns = [row[1] for row in conn.execute("PRAGMA table_info(calendar_events)").fetchall()] + if ev_columns and "remote_href" not in ev_columns: + conn.execute("ALTER TABLE calendar_events ADD COLUMN remote_href TEXT") + if ev_columns and "remote_etag" not in ev_columns: + conn.execute("ALTER TABLE calendar_events ADD COLUMN remote_etag TEXT") + if ev_columns and "caldav_sync_pending" not in ev_columns: + conn.execute("ALTER TABLE calendar_events ADD COLUMN caldav_sync_pending TEXT") + + cal_columns = [row[1] for row in conn.execute("PRAGMA table_info(calendars)").fetchall()] + if cal_columns and "caldav_base_url" not in cal_columns: + conn.execute("ALTER TABLE calendars ADD COLUMN caldav_base_url TEXT") + conn.commit() + conn.close() + except Exception as e: + logging.getLogger(__name__).warning(f"CalDAV sync metadata migration failed: {e}") def _migrate_add_calendar_metadata(): @@ -1642,6 +2348,7 @@ def _migrate_add_calendar_metadata(): db_path = DATABASE_URL.replace("sqlite:///", "") if not os.path.exists(db_path): return + conn = None try: conn = sqlite3.connect(db_path) cursor = conn.execute("PRAGMA table_info(calendar_events)") @@ -1653,9 +2360,35 @@ def _migrate_add_calendar_metadata(): if columns and "last_pinged" not in columns: conn.execute("ALTER TABLE calendar_events ADD COLUMN last_pinged DATETIME") conn.commit() - conn.close() except Exception as e: logging.getLogger(__name__).warning(f"calendar_events migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass + + +def _migrate_add_calendar_recurrence_exdates(): + """Add skipped recurrence occurrences for deleting one instance of a series.""" + import sqlite3 + db_path = DATABASE_URL.replace("sqlite:///", "") + if not os.path.exists(db_path): + return + conn = None + try: + conn = sqlite3.connect(db_path) + columns = [row[1] for row in conn.execute("PRAGMA table_info(calendar_events)").fetchall()] + if columns and "recurrence_exdates" not in columns: + conn.execute("ALTER TABLE calendar_events ADD COLUMN recurrence_exdates TEXT DEFAULT ''") + conn.commit() + except Exception as e: + logging.getLogger(__name__).warning(f"calendar_events recurrence_exdates migration failed: {e}") + finally: + try: + conn.close() + except Exception: + pass def get_db(): """ @@ -1694,7 +2427,7 @@ def bulk_insert_messages(session_id: str, messages: list): 'session_id': session_id, 'role': msg['role'], 'content': msg['content'], - 'timestamp': datetime.utcnow() + 'timestamp': utcnow_naive() } for msg in messages ] @@ -1705,7 +2438,7 @@ def cleanup_old_sessions(days: int = 30): from datetime import timedelta with get_db_session() as db: - cutoff_date = datetime.utcnow() - timedelta(days=days) + cutoff_date = utcnow_naive() - timedelta(days=days) deleted_count = db.query(Session).filter( Session.archived == True, @@ -1750,16 +2483,69 @@ def update_session_last_accessed(session_id: str): with get_db_session() as db: db_session = db.query(Session).filter(Session.id == session_id).first() if db_session: - db_session.last_accessed = datetime.utcnow() + db_session.last_accessed = utcnow_naive() db.commit() return True return False +def get_session_mode(session_id: str): + """Return a session's persisted `mode`, or None if unset/unknown. + + Best-effort: never raises (returns None on any DB error) so callers on hot + request paths needn't guard it. Routed through get_db_session() so the + connection is always returned to the pool.""" + try: + with get_db_session() as db: + return db.query(Session.mode).filter(Session.id == session_id).scalar() + except Exception: + logger.warning("Failed to read mode for session %s", session_id) + return None + +def set_session_mode(session_id: str, mode: str) -> bool: + """Persist a session's `mode`. Best-effort: never raises, returns success. + + Routed through get_db_session() so a failure mid-write (e.g. a SQLite + 'database is locked' under concurrent streams) still returns the connection + to the pool instead of leaking it — repeated leaks would exhaust it.""" + try: + with get_db_session() as db: + db.query(Session).filter(Session.id == session_id).update({"mode": mode}) + return True + except Exception: + logger.warning("Failed to persist mode %r for session %s", mode, session_id) + return False + def get_session_by_id(session_id: str): """Get a session by ID""" with get_db_session() as db: return db.query(Session).filter(Session.id == session_id).first() +def get_upcoming_events(owner, horizon_days: int = 60, limit: int = 40): + """Upcoming, non-cancelled events as {uid, title, start} dicts, soonest first. + + owner=None means NO owner scoping (single-user / legacy). Multi-user callers + MUST pass the owning username — otherwise they read every tenant's events. + The autonomous email->calendar pass relies on this to avoid disclosing (and + acting on) other users' calendars.""" + from datetime import timedelta + now = utcnow_naive() + with get_db_session() as db: + q = db.query(CalendarEvent).join(CalendarCal).filter( + CalendarEvent.dtstart >= now, + CalendarEvent.dtstart <= now + timedelta(days=horizon_days), + CalendarEvent.status != "cancelled", + ) + if owner is not None: + q = q.filter(CalendarCal.owner == owner) + return [ + { + "uid": e.uid, + "title": e.summary or "", + "start": e.dtstart.isoformat() if e.dtstart else "", + } + for e in q.order_by(CalendarEvent.dtstart).limit(limit).all() + ] + def archive_session(session_id: str): """Archive a session""" with get_db_session() as db: diff --git a/core/exceptions.py b/core/exceptions.py index 26a411e6d..1840b049e 100644 --- a/core/exceptions.py +++ b/core/exceptions.py @@ -1,4 +1,4 @@ -# src/exceptions.py +# core/exceptions.py """Custom exceptions for the application.""" class SessionNotFoundError(Exception): diff --git a/core/log_safety.py b/core/log_safety.py new file mode 100644 index 000000000..2339a73b6 --- /dev/null +++ b/core/log_safety.py @@ -0,0 +1,27 @@ +"""Helpers for keeping sensitive data out of logs. + +Endpoint URLs configured by admins can embed credentials in the userinfo +(``https://user:pass@host``) or query string (``?api_key=...``). Logging them +raw leaks those secrets, so route/diagnostic logs run URLs through +``redact_url`` first. Reconstructing the URL without userinfo/query/fragment +also doubles as a sanitizer barrier for CodeQL's clear-text-logging query. +""" + +from urllib.parse import urlparse, urlunparse + + +def redact_url(url: str) -> str: + """Return a URL safe for logs by removing userinfo and query/fragment. + + Keeps scheme, host, port and path so logs stay useful for debugging. + """ + try: + parsed = urlparse(url or "") + host = parsed.hostname or "" + if ":" in host: # IPv6 literal — re-bracket so host:port stays unambiguous + host = f"[{host}]" + if parsed.port: + host = f"{host}:{parsed.port}" + return urlunparse((parsed.scheme, host, parsed.path, "", "", "")) + except Exception: + return "" diff --git a/core/middleware.py b/core/middleware.py index a3e9e9ae9..0e164e35a 100644 --- a/core/middleware.py +++ b/core/middleware.py @@ -15,6 +15,17 @@ # same value from this module. Never persisted or exposed externally. INTERNAL_TOOL_TOKEN = os.environ.get("ODYSSEUS_INTERNAL_TOKEN") or secrets.token_hex(32) INTERNAL_TOOL_HEADER = "X-Odysseus-Internal-Token" +# Pseudo-username on in-process tool-loopback requests; require_admin trusts it and it is reserved. +INTERNAL_TOOL_USER = "internal-tool" + + +def is_cors_preflight(method: str, headers) -> bool: + """True for a genuine CORS preflight: an OPTIONS request carrying the + Access-Control-Request-Method header. Such requests are credential-less by + design and must reach CORSMiddleware to be answered -- gating them on auth + 401s the preflight and breaks every cross-origin browser/WebView client. + Pure so it can be unit-tested without standing up the app.""" + return method == "OPTIONS" and "access-control-request-method" in headers def require_admin(request: Request): @@ -27,9 +38,10 @@ def require_admin(request: Request): # (b) the auth middleware already validated the token and stamped # request.state.current_user = "internal-tool". try: - if request.headers.get(INTERNAL_TOOL_HEADER) == INTERNAL_TOOL_TOKEN: + hdr = request.headers.get(INTERNAL_TOOL_HEADER) + if hdr and secrets.compare_digest(hdr, INTERNAL_TOOL_TOKEN): return - if getattr(request.state, "current_user", None) == "internal-tool": + if getattr(request.state, "current_user", None) == INTERNAL_TOOL_USER: return except Exception: pass @@ -55,13 +67,23 @@ async def dispatch(self, request: Request, call_next) -> Response: response = await call_next(request) path = request.url.path - # Tool render endpoints are served inside iframes — allow framing by self + # Tool render endpoints is_tool_render = path.startswith("/api/tools/") and path.endswith("/render") + # Document library PDF preview endpoint + is_document_pdf_preview = path.startswith("/api/document/") and path.endswith("/render-pdf") # Visual report pages are self-contained HTML — need inline scripts + external images is_report = path.startswith("/api/research/report/") response.headers["X-Content-Type-Options"] = "nosniff" response.headers["Referrer-Policy"] = "no-referrer" + response.headers["Permissions-Policy"] = "camera=(), microphone=(self), geolocation=()" + + is_https = ( + request.url.scheme == "https" + or request.headers.get("X-Forwarded-Proto") == "https" + ) + if is_https: + response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains" if is_report: response.headers["Content-Security-Policy"] = ( @@ -74,10 +96,14 @@ async def dispatch(self, request: Request, call_next) -> Response: "frame-ancestors 'none'" ) elif is_tool_render: - # Tool iframe content: skip all framing headers — the iframe's - # sandbox="allow-scripts" attribute provides isolation. - # Don't overwrite the route's own restrictive CSP either. + # Skip framing headers for tools. pass + elif is_document_pdf_preview: + response.headers["X-Frame-Options"] = "SAMEORIGIN" + response.headers["Content-Security-Policy"] = ( + "default-src 'none'; " + "frame-ancestors 'self'" + ) else: response.headers["X-Frame-Options"] = "DENY" # NOTE: `style-src 'unsafe-inline'` is intentionally retained. @@ -91,7 +117,7 @@ async def dispatch(self, request: Request, call_next) -> Response: f"script-src 'self' 'nonce-{nonce}' https://cdn.jsdelivr.net; " "style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; " "font-src 'self' https://cdn.jsdelivr.net; " - "img-src 'self' data: blob:; " + "img-src 'self' data: blob: https:; " "media-src 'self' blob:; " "connect-src 'self'; " "frame-src 'self'; " diff --git a/core/models.py b/core/models.py index 6914b20a4..56f05dc4e 100644 --- a/core/models.py +++ b/core/models.py @@ -11,14 +11,24 @@ if TYPE_CHECKING: from .session_manager import SessionManager -# Module-level session manager reference (set at app startup) -_session_manager: Optional["SessionManager"] = None +# Module-level session manager singleton (single source of truth) +_SESSION_MANAGER_INSTANCE: Optional["SessionManager"] = None -def set_session_manager(manager: "SessionManager"): - """Set the global session manager reference.""" - global _session_manager - _session_manager = manager +def set_session_manager_instance(manager: "SessionManager"): + """Set the global SessionManager singleton.""" + global _SESSION_MANAGER_INSTANCE + _SESSION_MANAGER_INSTANCE = manager + + +def get_session_manager_instance() -> Optional["SessionManager"]: + """Get the global SessionManager singleton.""" + return _SESSION_MANAGER_INSTANCE + + +# Keep legacy name for backward compatibility +set_session_manager = set_session_manager_instance +get_session_manager = get_session_manager_instance @dataclass @@ -42,7 +52,17 @@ def get(self, key: str, default=None): @dataclass class Session: - """A chat session — pure data container.""" + """A chat session — pure data container. + + ``.history`` is the authoritative mutable message list. Callers may + read, append, pop, or reassign it directly — these changes take + effect immediately. ``_history`` remains a compatibility alias that + always resolves to the authoritative ``history`` list. + + Each session gets its own unique history list at construction time + (the dataclass default is never shared between instances). + """ + id: str name: str endpoint_url: str @@ -56,29 +76,56 @@ class Session: message_count: int = 0 def __post_init__(self): - if self.history is None: - self.history = [] if self.headers is None: self.headers = {} + # Ensure each session gets its OWN list (not the shared dataclass default) + if self.history is None: + self.history = [] + + @property + def _history(self) -> List[ChatMessage]: + """Compatibility alias for callers that still reference ``_history``.""" + return self.history + + @_history.setter + def _history(self, messages: List[ChatMessage]): + self.history = messages def add_message(self, message: ChatMessage): """ Add a message to this session. - Delegates to SessionManager for persistence if available, - otherwise just appends to history. + Appends to the authoritative history list and increments + message_count. Delegates to SessionManager for persistence + if available. """ self.history.append(message) self.message_count = len(self.history) # Delegate to session manager for persistence - if _session_manager: - _session_manager._persist_message(self.id, message) + if _SESSION_MANAGER_INSTANCE: + _SESSION_MANAGER_INSTANCE._persist_message(self.id, message) def get_context_messages(self) -> List[Dict[str, Any]]: - """Get messages in format for LLM API.""" - return [msg.to_dict() for msg in self.history] + """Get messages in format for LLM API. + + Slash-command / setup replies are persisted to history so they render + in the transcript, but they are UI chatter (e.g. ``/setup ...`` and its + status lines) the user never meant as conversation. They carry + ``metadata.source == "slash"``; exclude them here so they never reach + the model. Display/history-load paths use the raw ``history`` and are + unaffected. + """ + return [ + msg.to_dict() + for msg in self.history + if (msg.metadata or {}).get("source") != "slash" + ] def get(self, key: str, default=None): """Dict-like access for compatibility.""" return getattr(self, key, default) + + def __getitem__(self, key: str): + """Allow session['field'] syntax.""" + return getattr(self, key) diff --git a/core/platform_compat.py b/core/platform_compat.py new file mode 100644 index 000000000..efa496ac6 --- /dev/null +++ b/core/platform_compat.py @@ -0,0 +1,452 @@ +"""Cross-platform OS compatibility helpers. + +Odysseus began as a Linux/macOS/Docker-only app. This module centralizes the +small set of OS differences needed to run it *natively* on Windows so the rest +of the codebase can stay platform-agnostic. Import from here instead of +sprinkling ``os.name == "nt"`` checks (and POSIX-only calls) across modules. + +Design rules: + * Stdlib + ctypes only — no new third-party deps (no psutil/pywinpty). + * POSIX behaviour is unchanged; Windows gets a faithful equivalent or a + safe, documented no-op. +""" + +from __future__ import annotations + +import os +import ntpath +import shutil +import subprocess +from pathlib import Path +import sys +from typing import List, Optional +import platform + +IS_WINDOWS = os.name == "nt" +IS_POSIX = not IS_WINDOWS +# Allows APFEL support and ARM-native binary recommendations on Apple Silicon Macs. +IS_APPLE_SILICON = ( + IS_POSIX + and platform.system() == "Darwin" + and platform.machine().lower() + in { + "arm64", + "aarch64", + } +) + + +# ── File permissions ──────────────────────────────────────────────────────── +def safe_chmod(path, mode: int) -> bool: + """``os.chmod`` that is a harmless no-op on Windows. + + On POSIX we apply the mode — used to lock secret/key files down to 0o600. + Windows has no POSIX permission bits; files under the user profile are + already ACL-restricted to that user, so we skip rather than raise. Returns + True when the mode was actually applied. + """ + if IS_WINDOWS: + return False + try: + os.chmod(path, mode) + return True + except OSError: + return False + + +# ── Process detach / liveness / teardown ──────────────────────────────────── +def detached_popen_kwargs() -> dict: + """Keyword args for :class:`subprocess.Popen` that fully detach a child so + it outlives the request/stream that launched it. + + POSIX: ``start_new_session=True`` (setsid) — new session + process group. + Windows: ``CREATE_NEW_PROCESS_GROUP | DETACHED_PROCESS`` — the child gets + its own process group (so it isn't killed when the parent's console closes) + and is detached from any console. + """ + if IS_WINDOWS: + flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200) | getattr( + subprocess, "DETACHED_PROCESS", 0x00000008 + ) + return {"creationflags": flags} + return {"start_new_session": True} + + +def pid_alive(pid: Optional[int]) -> bool: + """True if a process with ``pid`` is currently running. + + POSIX uses the classic ``os.kill(pid, 0)`` probe. That is **unsafe on + Windows**: CPython's ``os.kill`` calls ``TerminateProcess(handle, sig)`` for + any signal other than CTRL_C/CTRL_BREAK, so ``os.kill(pid, 0)`` would *kill* + the process it is checking. We instead open the process and read its exit + code via the Win32 API. + """ + if not pid: + return False + if IS_WINDOWS: + import ctypes + from ctypes import wintypes + + PROCESS_QUERY_LIMITED_INFORMATION = 0x1000 + STILL_ACTIVE = 259 + kernel32 = ctypes.windll.kernel32 + handle = kernel32.OpenProcess( + PROCESS_QUERY_LIMITED_INFORMATION, False, int(pid) + ) + if not handle: + return False + try: + code = wintypes.DWORD() + if kernel32.GetExitCodeProcess(handle, ctypes.byref(code)): + return code.value == STILL_ACTIVE + return False + finally: + kernel32.CloseHandle(handle) + try: + os.kill(pid, 0) + return True + except (OSError, ProcessLookupError): + return False + + +def kill_process_tree(pid: Optional[int]) -> None: + """Terminate ``pid`` and all of its descendants. + + POSIX: signal the whole process group (``killpg``), falling back to a plain + ``kill`` if the pid isn't a group leader. + Windows: ``taskkill /T /F`` walks and kills the child tree (there is no + process-group signalling). + """ + if not pid: + return + if IS_WINDOWS: + try: + subprocess.run( + ["taskkill", "/F", "/T", "/PID", str(pid)], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + except Exception: + pass + return + import signal + + try: + os.killpg(os.getpgid(pid), signal.SIGTERM) + except Exception: + try: + os.kill(pid, signal.SIGTERM) + except Exception: + pass + + +# ── Shell / executable resolution ─────────────────────────────────────────── +_BASH_CACHE: Optional[str] = None +_BASH_PROBED = False + +# Common Git-for-Windows install locations to probe when bash isn't on PATH. +_WINDOWS_BASH_ROOT_ENV_VARS = ( + "ProgramFiles", + "ProgramW6432", + "ProgramFiles(x86)", + "LocalAppData", +) +_WINDOWS_BASH_DEFAULT_ROOTS = ( + r"C:\Program Files\Git", + r"C:\Program Files (x86)\Git", +) +_WINDOWS_BASH_RELATIVE_PATHS = ( + ("bin", "bash.exe"), + ("usr", "bin", "bash.exe"), +) + +# Paths to add to the remote SSH probe command to find tools like nvidia-smi that may not be on PATH. +_SSH_PATH_MEMBERS = ( + "/usr/bin", + "/usr/local/bin", + "/usr/local/cuda/bin", + "/usr/lib/wsl/lib" +) +# Fallback locations for nvidia-smi on WSL and other Linux distros where it may not be on PATH. +NVIDIA_PATH_CANDIDATES = ( + "/usr/bin/nvidia-smi", + "/usr/local/bin/nvidia-smi", + "/usr/local/cuda/bin/nvidia-smi", + "/usr/lib/wsl/lib/nvidia-smi", +) + + +def _ssh_path_override() -> str: + """Build the PATH export snippet used for remote SSH shell probes.""" + return f"export PATH=\"$PATH:{':'.join(_SSH_PATH_MEMBERS)}\"; " + + +SSH_PATH_OVERRIDE = _ssh_path_override() + + +def _windows_bash_fallbacks() -> List[str]: + roots: List[str] = [] + for env_name in _WINDOWS_BASH_ROOT_ENV_VARS: + base = os.environ.get(env_name) + if base: + roots.append(ntpath.join(base, "Git")) + if env_name == "LocalAppData": + roots.append(ntpath.join(base, "Programs", "Git")) + roots.extend(_WINDOWS_BASH_DEFAULT_ROOTS) + + paths: List[str] = [] + seen = set() + for root in roots: + for rel in _WINDOWS_BASH_RELATIVE_PATHS: + path = ntpath.join(root, *rel) + key = path.lower() + if key not in seen: + seen.add(key) + paths.append(path) + return paths + + +def _is_windows_bash_stub(path: str) -> bool: + lowered = path.lower() + return ( + "system32\\bash.exe" in lowered + or "sysnative\\bash.exe" in lowered + or "windowsapps\\bash.exe" in lowered + ) + + +def git_bash_path(path: str | Path) -> str: + """Convert a path to POSIX style suitable for Git Bash on Windows. + + Transforms drive letters (e.g., 'C:\\path') to POSIX '/c/path', + and uses forward slashes. + """ + p = Path(path) + p_str = p.as_posix() + if IS_WINDOWS and len(p_str) >= 2 and p_str[1] == ":": + drive = p_str[0].lower() + return f"/{drive}{p_str[2:]}" + return p_str + + + +def find_bash() -> Optional[str]: + """Locate a real ``bash`` interpreter, or None. + + On Windows this is typically Git Bash / WSL. Many Odysseus features (the + agent ``bash`` tool, background jobs, Cookbook scripts) emit bash syntax, so + when a bash is present we use it and keep full parity with POSIX. Result is + cached. + """ + global _BASH_CACHE, _BASH_PROBED + if _BASH_PROBED: + return _BASH_CACHE + _BASH_PROBED = True + found = which_tool("bash") + if found and IS_WINDOWS and _is_windows_bash_stub(found): + found = None + if not found and IS_WINDOWS: + for cand in _windows_bash_fallbacks(): + if os.path.exists(cand): + found = cand + break + _BASH_CACHE = found + return found + + +def has_bash() -> bool: + return find_bash() is not None + + +def which_tool(name: str) -> Optional[str]: + """``shutil.which`` that also tries Windows executable suffixes. + + On Windows, Node/npm shims are ``npx.cmd``/``npm.cmd`` and binaries end in + ``.exe``; a bare ``which("npx")`` can miss them depending on PATHEXT. We try + the bare name first, then the common suffixes. + """ + found = shutil.which(name) + if found: + return found + if IS_WINDOWS: + for ext in (".cmd", ".exe", ".bat"): + found = shutil.which(name + ext) + if found: + return found + return None + + +def run_script_argv(script_path) -> List[str]: + """argv to execute a shell *script file*. + + Prefers bash (so existing ``.sh`` wrappers work verbatim, including on + Windows via Git Bash). On Windows with no bash available, falls back to + ``cmd.exe /c`` — simple commands still run, but bash-specific syntax won't. + Callers that need guaranteed bash should check :func:`has_bash` first and + surface a clear "install Git Bash" message. + """ + bash = find_bash() + if bash: + return [bash, str(script_path)] + if IS_WINDOWS: + comspec = os.environ.get("ComSpec", "cmd.exe") + return [comspec, "/c", str(script_path)] + return ["sh", str(script_path)] + + +def is_wsl() -> bool: + """True if running inside Windows Subsystem for Linux (WSL).""" + import sys + if sys.platform.startswith("linux") or os.name == "posix": + try: + with open("/proc/version", "r", encoding="utf-8", errors="ignore") as f: + if "microsoft" in f.read().lower(): + return True + except Exception: + pass + return False + + +def translate_path(path_str: str) -> str: + """Translate a path (possibly a Windows path) to the current OS format. + + Particularly handles Windows paths (e.g. C:\\foo or C:/foo) when running + under WSL, translating them to /mnt/c/foo. + Also handles standard path normalization to avoid string breakages. + """ + if not path_str: + return path_str + + if is_wsl(): + path_str = path_str.replace("\\", "/") + import re + m = re.match(r"^([a-zA-Z]):(.*)", path_str) + if m: + drive = m.group(1).lower() + rest = m.group(2) + if not rest.startswith("/"): + rest = "/" + rest + return f"/mnt/{drive}{rest}" + + try: + return str(Path(path_str).resolve()) + except Exception: + return path_str + + +def get_wsl_windows_user_profile() -> Optional[str]: + """Retrieve the Windows host User Profile path from inside WSL.""" + if not is_wsl(): + return None + try: + r = run_wsl_windows_powershell("Write-Output $env:USERPROFILE", timeout=5) + if r.returncode == 0 and r.stdout.strip(): + return translate_path(r.stdout.strip()) + except Exception: + pass + + try: + users_dir = "/mnt/c/Users" + if os.path.isdir(users_dir): + for entry in os.listdir(users_dir): + if entry not in ("All Users", "Default", "Default User", "desktop.ini", "Public"): + path = os.path.join(users_dir, entry) + if os.path.isdir(path): + return path + except Exception: + pass + return None + + +def _ssh_exec_argv( + remote: str, + ssh_port: str | None, + *, + remote_cmd: str | None = None, + connect_timeout: int | None = None, + strict_host_key_checking: bool | None = None, +) -> list[str]: + """Build a consistent ssh argv for remote command execution.""" + remote_value = str(remote or "").strip() + remote_host = remote_value.rsplit("@", 1)[-1] + if not remote_value or remote_value.startswith("-") or not remote_host or remote_host.startswith("-"): + raise ValueError("Invalid SSH remote host") + argv = ["ssh"] + if connect_timeout is not None: + argv.extend(["-o", f"ConnectTimeout={int(connect_timeout)}"]) + if strict_host_key_checking is not None: + argv.extend( + [ + "-o", + "StrictHostKeyChecking=yes" + if strict_host_key_checking + else "StrictHostKeyChecking=no", + ] + ) + if ssh_port and ssh_port != "22": + argv.extend(["-p", str(ssh_port)]) + argv.append(remote) + if remote_cmd is not None: + argv.append(remote_cmd) + return argv + + +def run_ssh_command( + remote: str, + ssh_port: str | None, + remote_cmd: str, + *, + timeout: float, + connect_timeout: int | None = None, + strict_host_key_checking: bool | None = None, + text: bool = True, +) -> subprocess.CompletedProcess: + """Run an ssh command with centralized timeout and stderr/stdout capture.""" + return subprocess.run( + _ssh_exec_argv( + remote, + ssh_port, + remote_cmd=remote_cmd, + connect_timeout=connect_timeout, + strict_host_key_checking=strict_host_key_checking, + ), + timeout=timeout, + capture_output=True, + text=text, + ) + + +def _windows_powershell_argv( + command: str, + *, + no_profile: bool = True, + non_interactive: bool = True, +) -> List[str]: + argv: List[str] = ["powershell.exe"] + if no_profile: + argv.append("-NoProfile") + if non_interactive: + argv.append("-NonInteractive") + argv.extend(["-Command", command]) + return argv + + +def run_wsl_windows_powershell( + command: str, + *, + timeout: float = 5, +) -> subprocess.CompletedProcess[str]: + """Run a PowerShell command on the Windows host from WSL. + + Raises ``RuntimeError`` when called outside WSL. + """ + + if not is_wsl(): + raise RuntimeError("run_wsl_windows_powershell is only supported in WSL") + return subprocess.run( + _windows_powershell_argv(command), + capture_output=True, + text=True, + timeout=timeout, + ) diff --git a/core/session_manager.py b/core/session_manager.py index 699a59b8d..f5024d212 100644 --- a/core/session_manager.py +++ b/core/session_manager.py @@ -14,12 +14,52 @@ from datetime import datetime, timezone, timedelta from typing import Dict, Optional -from .database import Session as DbSession, ChatMessage as DbChatMessage, Document as DbDocument, SessionLocal +from .database import Session as DbSession, ChatMessage as DbChatMessage, Document as DbDocument, SessionLocal, utcnow_naive from .models import Session, ChatMessage +from src.attachment_refs import persistable_message_content +from src.upload_handler import reserve_message_upload_references + +# Re-export singleton accessors from models for convenience +from .models import set_session_manager_instance, get_session_manager_instance logger = logging.getLogger(__name__) +def _message_timestamp_iso(value: Optional[datetime]) -> Optional[str]: + """Return a stable ISO timestamp for chat message metadata.""" + if not value: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=timezone.utc) + return value.isoformat().replace("+00:00", "Z") + + +def _parse_msg_content(raw): + """Parse message content from DB — deserialises JSON arrays back to lists + (multimodal content with image/audio attachments).""" + if isinstance(raw, list): + return raw + if isinstance(raw, str) and raw.startswith('[{') and '"type"' in raw: + try: + parsed = json.loads(raw) + # Only treat as serialized multimodal content when EVERY element is + # a dict whose "type" is a recognized content-block kind. Otherwise a + # plain text message that merely *looks* like a JSON array of objects + # (e.g. a user pasting an API schema/sample with a "type" field) was + # silently parsed back into a list, destroying the original string. + _BLOCK_TYPES = { + "text", "image", "image_url", "audio", "input_audio", + "input_image", "document", "file", + } + if (isinstance(parsed, list) and parsed + and all(isinstance(p, dict) and p.get("type") in _BLOCK_TYPES + for p in parsed)): + return parsed + except (json.JSONDecodeError, ValueError): + pass + return raw + + class SessionManager: """ Manages chat sessions with database persistence. @@ -34,6 +74,7 @@ class SessionManager: def __init__(self, sessions_file: str = None): # sessions_file kept for backward compat, not used self.sessions: Dict[str, Session] = {} + self.upload_handler = None self.load_sessions() # ------------------------------------------------------------------ @@ -107,9 +148,10 @@ def _db_to_session(self, db_session: DbSession, db) -> Optional[Session]: meta = json.loads(db_msg.meta_data) if db_msg.meta_data else {} if meta is None: meta = {} meta['_db_id'] = db_msg.id + meta.setdefault('timestamp', _message_timestamp_iso(db_msg.timestamp)) history.append(ChatMessage( role=db_msg.role, - content=db_msg.content, + content=_parse_msg_content(db_msg.content), metadata=meta, )) else: @@ -121,9 +163,10 @@ def _db_to_session(self, db_session: DbSession, db) -> Optional[Session]: meta = json.loads(db_msg.meta_data) if db_msg.meta_data else {} if meta is None: meta = {} meta['_db_id'] = db_msg.id + meta.setdefault('timestamp', _message_timestamp_iso(db_msg.timestamp)) history.append(ChatMessage( role=db_msg.role, - content=db_msg.content, + content=_parse_msg_content(db_msg.content), metadata=meta, )) @@ -162,12 +205,17 @@ def add_message(self, session_id: str, message: ChatMessage): """ Add a message to a session and persist to database. + Updates the authoritative history list and persists through this + manager directly so tests and temporary managers do not depend on the + process-wide session-manager singleton. + Args: session_id: Session ID message: ChatMessage to add """ session = self.get_session(session_id) session.history.append(message) + session._history = session.history session.message_count = len(session.history) self._persist_message(session_id, message) @@ -176,31 +224,59 @@ def _persist_message(self, session_id: str, message: ChatMessage): """Persist a single message to the database.""" db = SessionLocal() try: + db_session = db.query(DbSession).filter(DbSession.id == session_id).first() + if db_session is None: + # A stream/tool callback can outlive a session delete. Do not + # create a chat_messages row with no parent session; also drop + # any stale cached session so later writes fail closed too. + self.sessions.pop(session_id, None) + logger.warning("Dropping message for deleted session %s", session_id) + return + + missing_upload_id = reserve_message_upload_references( + getattr(self, "upload_handler", None), + getattr(db_session, "owner", None), + message.content, + message.metadata, + ) + if missing_upload_id: + raise ValueError( + f"Referenced upload is no longer available: {missing_upload_id}" + ) + msg_id = str(uuid.uuid4()) + msg_time = datetime.utcnow() + if message.metadata is None: + message.metadata = {} + message.metadata.setdefault('timestamp', _message_timestamp_iso(msg_time)) + # Multimodal content may contain provider data URLs for the live + # model call. Persist only readable text plus attachment references + # so chat_messages/FTS do not duplicate upload bytes. + _content = persistable_message_content(message.content, message.metadata) db_message = DbChatMessage( id=msg_id, session_id=session_id, role=message.role, - content=message.content, - meta_data=json.dumps(message.metadata) if message.metadata else None + content=_content, + meta_data=json.dumps(message.metadata) if message.metadata else None, + timestamp=msg_time, ) db.add(db_message) - db_session = db.query(DbSession).filter(DbSession.id == session_id).first() - if db_session: - db_session.message_count = len(self.sessions.get(session_id, {}).history) if session_id in self.sessions else 0 - _now = datetime.now(timezone.utc) - db_session.last_accessed = _now - # Clean "last conversation" timestamp — only bumped here on a - # real message persist, so it powers an accurate "Last active" - # sort that ignores renames / model swaps / mere opens. - db_session.last_message_at = _now + if session_id in self.sessions: + db_session.message_count = len(self.sessions[session_id].history) + else: + db_session.message_count = 0 + _now = datetime.now(timezone.utc) + db_session.last_accessed = _now + # Clean "last conversation" timestamp — only bumped here on a + # real message persist, so it powers an accurate "Last active" + # sort that ignores renames / model swaps / mere opens. + db_session.last_message_at = _now db.commit() # Store DB ID on the in-memory message for edit/delete by ID - if message.metadata is None: - message.metadata = {} message.metadata['_db_id'] = msg_id logger.debug(f"Persisted message to session {session_id}") @@ -231,13 +307,17 @@ def truncate_messages(self, session_id: str, keep_count: int) -> bool: db_session = db.query(DbSession).filter(DbSession.id == session_id).first() if db_session: - db_session.message_count = keep_count + # keep_count can exceed the real message total (e.g. the AI tool + # defaults to keep_count=10 on a short session); message_count must + # track the rows that actually remain, not the requested cap. + db_session.message_count = min(keep_count, len(db_messages)) db_session.updated_at = datetime.now(timezone.utc) db.commit() # Update in-memory session.history = session.history[:keep_count] + session._history = session.history logger.info(f"Truncated session {session_id} to {keep_count} messages") return True @@ -254,6 +334,28 @@ def replace_messages(self, session_id: str, messages: list) -> bool: session = self.get_session(session_id) db = SessionLocal() try: + db_session = db.query(DbSession).filter(DbSession.id == session_id).first() + if db_session is None: + logger.warning("Cannot replace history for missing session %s", session_id) + return False + + # Reserve every incoming attachment before removing any durable + # message row. reserve_upload() shares the upload lifecycle lock + # with cleanup, so an upload cannot be deleted between this + # ownership check/access touch and the replacement transaction. + # A failed reservation must leave the existing transcript intact. + for message in messages: + missing_upload_id = reserve_message_upload_references( + getattr(self, "upload_handler", None), + getattr(db_session, "owner", None), + message.content, + message.metadata, + ) + if missing_upload_id: + raise ValueError( + f"Referenced upload is no longer available: {missing_upload_id}" + ) + db.query(DbChatMessage).filter(DbChatMessage.session_id == session_id).delete() now = datetime.now(timezone.utc) for i, message in enumerate(messages): @@ -262,7 +364,9 @@ def replace_messages(self, session_id: str, messages: list) -> bool: id=msg_id, session_id=session_id, role=message.role, - content=message.content, + # Mirrors _persist_message: keep raw media bytes out of the + # persisted transcript and search index. + content=persistable_message_content(message.content, message.metadata), meta_data=json.dumps(message.metadata) if message.metadata else None, timestamp=now + timedelta(microseconds=i), ) @@ -271,15 +375,14 @@ def replace_messages(self, session_id: str, messages: list) -> bool: message.metadata = {} message.metadata["_db_id"] = msg_id - db_session = db.query(DbSession).filter(DbSession.id == session_id).first() - if db_session: - db_session.message_count = len(messages) - db_session.updated_at = now - db_session.last_accessed = now - db_session.last_message_at = now + db_session.message_count = len(messages) + db_session.updated_at = now + db_session.last_accessed = now + db_session.last_message_at = now db.commit() session.history = list(messages) + session._history = session.history session.message_count = len(messages) logger.info("Replaced session %s history with %d messages", session_id, len(messages)) return True @@ -452,11 +555,17 @@ def delete_session(self, session_id: str) -> bool: db_session = db.query(DbSession).filter(DbSession.id == session_id).first() if db_session: db.delete(db_session) - db.commit() - if session_id in self.sessions: - del self.sessions[session_id] + # Drop the in-memory copy even when there is no DB row. A "ghost" + # session lives only here (never persisted, or its row was removed + # out-of-band); without this it can never be cleared and keeps + # 404ing on every operation (issue #1044). + removed_in_memory = self.sessions.pop(session_id, None) is not None + if db_session or removed_in_memory: + # Commit the document-detach / message-delete above (a no-op when + # the ghost had no rows) together with the session delete. + db.commit() logger.info(f"Deleted session {session_id}") return True return False @@ -549,24 +658,52 @@ def get_sessions_for_user(self, username: Optional[str] = None) -> Dict[str, Ses def save_sessions(self): """No-op for DB compatibility.""" + def ensure_task_session(self, session_id: str, name: str, endpoint_url: str, model: str, owner: str = None, task: object = None) -> Session: + """Create a task session if it doesn't exist, or return the existing one. + + Unlike create_session, this checks the cache first and does NOT + overwrite an existing in-memory session. The task scheduler must + use this instead of direct dict assignment. + """ + if session_id in self.sessions: + return self.sessions[session_id] + + session = self.create_session(session_id, name, endpoint_url, model, owner=owner) + if task is not None: + task.session_id = session_id + return session + # ------------------------------------------------------------------ # Cleanup # ------------------------------------------------------------------ - def cleanup_empty_sessions(self, auto_archive_days: int = 30) -> dict: - """Clean up empty and old sessions.""" + def cleanup_empty_sessions(self, auto_archive_days: int = 30, min_age_hours: int = 1) -> dict: + """Clean up empty and old sessions. + + Args: + auto_archive_days: Age in days before non-important sessions are archived. + min_age_hours: Minimum age in hours before an empty session can be deleted. + Prevents deleting sessions that were just created. + """ db = SessionLocal() stats = {'deleted_empty': 0, 'archived_old': 0, 'total_checked': 0} try: all_sessions = db.query(DbSession).all() - cutoff_date = datetime.now(timezone.utc) - timedelta(days=auto_archive_days) + cutoff_date = utcnow_naive() - timedelta(days=auto_archive_days) + min_age = utcnow_naive() - timedelta(hours=min_age_hours) for db_session in all_sessions: stats['total_checked'] += 1 - # Delete empty sessions + # Delete empty sessions only if older than min_age_hours if db_session.message_count == 0: + if db_session.created_at is not None: + created = db_session.created_at + if created.tzinfo is None: + created = created.replace(tzinfo=timezone.utc) + if created > min_age: + continue # Too young to delete if db_session.id in self.sessions: del self.sessions[db_session.id] db.delete(db_session) diff --git a/docker-compose.gpu-amd.yml b/docker-compose.gpu-amd.yml new file mode 100644 index 000000000..82e22e440 --- /dev/null +++ b/docker-compose.gpu-amd.yml @@ -0,0 +1,173 @@ +# Standalone AMD ROCm GPU Compose file for stack-management UIs (Portainer, +# Coolify, Dockhand, etc.) that accept only a single Compose file and do not +# reliably honor COMPOSE_FILE or multiple `-f` overlays. +# +# This is equivalent to: docker-compose.yml + docker/gpu.amd.yml. +# The base docker-compose.yml plus the docker/gpu.amd.yml overlay remain the +# source of truth — CLI users should keep using the COMPOSE_FILE overlay +# workflow. Keep this file in sync with both when either changes. +# +# Requires ROCm drivers on the host (kfd + DRI devices) and the host user +# running Docker in the `video` and `render` groups. Set RENDER_GID to your +# host's numeric render group id when needed. See docker/gpu.amd.yml for details. +services: + odysseus: + build: . + ports: + - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000" + volumes: + - ${APP_DATA_DIR:-./data}:/app/data:z + - ${APP_LOGS_DIR:-./logs}:/app/logs:z + # Cookbook remote-server SSH identity. Odysseus can generate a key here; + # add the shown public key to each remote server's authorized_keys. + - ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z + # Cookbook local model cache. Inside Docker, "Local" means the Odysseus + # container, so persist its HuggingFace cache under ./data/huggingface. + - ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z + # Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.) + # land under /app/.local for the odysseus user. Persist them so a + # container recreate does not silently remove installed serve engines. + - ${APP_DATA_DIR:-./data}/local:/app/.local:z + extra_hosts: + # Lets the container reach local services on the Docker host, including + # Ollama at http://host.docker.internal:11434. + - "host.docker.internal:host-gateway" + environment: + - LLM_HOST=${LLM_HOST:-localhost} + - LLM_HOSTS=${LLM_HOSTS:-} + - OPENAI_API_KEY=${OPENAI_API_KEY:-} + - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-} + - RESEARCH_LLM_ENDPOINT=${RESEARCH_LLM_ENDPOINT:-} + - HF_TOKEN=${HF_TOKEN:-} + - HUGGING_FACE_HUB_TOKEN=${HUGGING_FACE_HUB_TOKEN:-} + - SEARXNG_INSTANCE=http://searxng:8080 + - CHROMADB_HOST=chromadb + - CHROMADB_PORT=8000 + - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db} + - AUTH_ENABLED=${AUTH_ENABLED:-true} + - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false} + - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin} + - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-} + - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1} + - SECURE_COOKIES=${SECURE_COOKIES:-false} + - EMBEDDING_URL=${EMBEDDING_URL:-} + - EMBEDDING_MODEL=${EMBEDDING_MODEL:-} + - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-} + - FASTEMBED_MODEL=${FASTEMBED_MODEL:-sentence-transformers/all-MiniLM-L6-v2} + - FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-} + - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24} + - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1} + - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1} + - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost} + - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760} + - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600} + - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760} + - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} + - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} + - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} + - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} + - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} + - TAVILY_API_KEY=${TAVILY_API_KEY:-} + - SERPER_API_KEY=${SERPER_API_KEY:-} + # PUID / PGID — the user/group the container drops to before + # running uvicorn (entrypoint also chowns /app/data + /app/logs + # to match, so bind-mounted files stay editable from the host). + # 1000 is the default first user on most Linux installs. If your + # host user has a different id, override here or via .env, e.g.: + # PUID=1001 + # PGID=1001 + # Find yours with: id -u / id -g + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + depends_on: + searxng: + condition: service_healthy + chromadb: + condition: service_started + restart: unless-stopped + # AMD ROCm overlay (from docker/gpu.amd.yml). + devices: + - /dev/kfd + - /dev/dri + group_add: + - video + - ${RENDER_GID:-render} + + chromadb: + image: docker.io/chromadb/chroma:latest + ports: + - "${CHROMADB_BIND:-127.0.0.1}:8100:8000" + volumes: + - chromadb-data:/chroma/chroma + environment: + - ANONYMIZED_TELEMETRY=FALSE + restart: unless-stopped + + searxng: + # Pinned, not :latest — odysseus waits on searxng's healthcheck + # (depends_on: condition: service_healthy), so a broken upstream `latest` + # tag blocks the whole app from starting. 2026.6.2 crashes on boot with + # `KeyError: 'default_doi_resolver'`, failing the healthcheck (issue #1414). + # Bump this deliberately after verifying a newer tag boots clean. + image: docker.io/searxng/searxng:2026.5.31-7159b8aed + entrypoint: + - /bin/sh + - -c + - | + set -eu + if [ ! -s /etc/searxng/settings.yml ] || grep -q 'odysseus-local-searxng-json-2026-05-30\|__SEARXNG_SECRET__' /etc/searxng/settings.yml; then + secret="$${SEARXNG_SECRET:-}" + if [ -z "$$secret" ]; then + secret="$$(python -c 'import secrets; print(secrets.token_urlsafe(48))')" + fi + sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml + fi + exec /usr/local/searxng/entrypoint.sh + ports: + - "127.0.0.1:8080:8080" + volumes: + - searxng-data:/etc/searxng + - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z + environment: + - SEARXNG_BASE_URL=http://localhost:8080/ + - SEARXNG_SECRET=${SEARXNG_SECRET:-} + # The official searxng image runs as the non-root `searxng` user, but its + # entrypoint still needs to chown /etc/searxng on first boot, drop privs via + # su-exec, and (with our wrapper above) write settings.yml into the named + # volume. Without these capabilities the wrapper aborts at the redirection + # with EACCES and the container fails its healthcheck with permission + # errors during setup. Mirrors the cap set recommended by the upstream + # searxng-docker compose file. See issue #721. + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + - DAC_OVERRIDE + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8080/', timeout=5).read(1)\""] + interval: 5s + timeout: 6s + retries: 20 + start_period: 10s + restart: unless-stopped + + ntfy: + image: docker.io/binwiederhier/ntfy + command: serve + ports: + - "${NTFY_BIND:-127.0.0.1}:8091:80" + volumes: + - ntfy-cache:/var/cache/ntfy + environment: + - NTFY_BASE_URL=${NTFY_BASE_URL:-http://localhost:8091} + restart: unless-stopped + +volumes: + searxng-data: + chromadb-data: + ntfy-cache: diff --git a/docker-compose.gpu-nvidia.yml b/docker-compose.gpu-nvidia.yml new file mode 100644 index 000000000..1b551c669 --- /dev/null +++ b/docker-compose.gpu-nvidia.yml @@ -0,0 +1,176 @@ +# Standalone NVIDIA GPU Compose file for stack-management UIs (Portainer, +# Coolify, Dockhand, etc.) that accept only a single Compose file and do not +# reliably honor COMPOSE_FILE or multiple `-f` overlays. +# +# This is equivalent to: docker-compose.yml + docker/gpu.nvidia.yml. +# The base docker-compose.yml plus the docker/gpu.nvidia.yml overlay remain +# the source of truth — CLI users should keep using the COMPOSE_FILE overlay +# workflow. Keep this file in sync with both when either changes. +# +# Requires the NVIDIA Container Toolkit on the host. See docker/gpu.nvidia.yml +# for setup details. +services: + odysseus: + build: . + ports: + - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000" + volumes: + - ${APP_DATA_DIR:-./data}:/app/data:z + - ${APP_LOGS_DIR:-./logs}:/app/logs:z + # Cookbook remote-server SSH identity. Odysseus can generate a key here; + # add the shown public key to each remote server's authorized_keys. + - ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z + # Cookbook local model cache. Inside Docker, "Local" means the Odysseus + # container, so persist its HuggingFace cache under ./data/huggingface. + - ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z + # Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.) + # land under /app/.local for the odysseus user. Persist them so a + # container recreate does not silently remove installed serve engines. + - ${APP_DATA_DIR:-./data}/local:/app/.local:z + extra_hosts: + # Lets the container reach local services on the Docker host, including + # Ollama at http://host.docker.internal:11434. + - "host.docker.internal:host-gateway" + environment: + - LLM_HOST=${LLM_HOST:-localhost} + - LLM_HOSTS=${LLM_HOSTS:-} + - OPENAI_API_KEY=${OPENAI_API_KEY:-} + - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-} + - RESEARCH_LLM_ENDPOINT=${RESEARCH_LLM_ENDPOINT:-} + - HF_TOKEN=${HF_TOKEN:-} + - HUGGING_FACE_HUB_TOKEN=${HUGGING_FACE_HUB_TOKEN:-} + - SEARXNG_INSTANCE=http://searxng:8080 + - CHROMADB_HOST=chromadb + - CHROMADB_PORT=8000 + - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db} + - AUTH_ENABLED=${AUTH_ENABLED:-true} + - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false} + - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin} + - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-} + - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1} + - SECURE_COOKIES=${SECURE_COOKIES:-false} + - EMBEDDING_URL=${EMBEDDING_URL:-} + - EMBEDDING_MODEL=${EMBEDDING_MODEL:-} + - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-} + - FASTEMBED_MODEL=${FASTEMBED_MODEL:-sentence-transformers/all-MiniLM-L6-v2} + - FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-} + - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24} + - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1} + - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1} + - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost} + - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760} + - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600} + - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760} + - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} + - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} + - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} + - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} + - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} + - TAVILY_API_KEY=${TAVILY_API_KEY:-} + - SERPER_API_KEY=${SERPER_API_KEY:-} + # PUID / PGID — the user/group the container drops to before + # running uvicorn (entrypoint also chowns /app/data + /app/logs + # to match, so bind-mounted files stay editable from the host). + # 1000 is the default first user on most Linux installs. If your + # host user has a different id, override here or via .env, e.g.: + # PUID=1001 + # PGID=1001 + # Find yours with: id -u / id -g + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + # NVIDIA overlay (from docker/gpu.nvidia.yml). + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + depends_on: + searxng: + condition: service_healthy + chromadb: + condition: service_started + restart: unless-stopped + # NVIDIA overlay (from docker/gpu.nvidia.yml). + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] + + chromadb: + image: docker.io/chromadb/chroma:latest + ports: + - "${CHROMADB_BIND:-127.0.0.1}:8100:8000" + volumes: + - chromadb-data:/chroma/chroma + environment: + - ANONYMIZED_TELEMETRY=FALSE + restart: unless-stopped + + searxng: + # Pinned, not :latest — odysseus waits on searxng's healthcheck + # (depends_on: condition: service_healthy), so a broken upstream `latest` + # tag blocks the whole app from starting. 2026.6.2 crashes on boot with + # `KeyError: 'default_doi_resolver'`, failing the healthcheck (issue #1414). + # Bump this deliberately after verifying a newer tag boots clean. + image: docker.io/searxng/searxng:2026.5.31-7159b8aed + entrypoint: + - /bin/sh + - -c + - | + set -eu + if [ ! -s /etc/searxng/settings.yml ] || grep -q 'odysseus-local-searxng-json-2026-05-30\|__SEARXNG_SECRET__' /etc/searxng/settings.yml; then + secret="$${SEARXNG_SECRET:-}" + if [ -z "$$secret" ]; then + secret="$$(python -c 'import secrets; print(secrets.token_urlsafe(48))')" + fi + sed "s|__SEARXNG_SECRET__|$$secret|g" /tmp/searxng-settings.yml.template > /etc/searxng/settings.yml + fi + exec /usr/local/searxng/entrypoint.sh + ports: + - "127.0.0.1:8080:8080" + volumes: + - searxng-data:/etc/searxng + - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z + environment: + - SEARXNG_BASE_URL=http://localhost:8080/ + - SEARXNG_SECRET=${SEARXNG_SECRET:-} + # The official searxng image runs as the non-root `searxng` user, but its + # entrypoint still needs to chown /etc/searxng on first boot, drop privs via + # su-exec, and (with our wrapper above) write settings.yml into the named + # volume. Without these capabilities the wrapper aborts at the redirection + # with EACCES and the container fails its healthcheck with permission + # errors during setup. Mirrors the cap set recommended by the upstream + # searxng-docker compose file. See issue #721. + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + - DAC_OVERRIDE + healthcheck: + test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8080/', timeout=5).read(1)\""] + interval: 5s + timeout: 6s + retries: 20 + start_period: 10s + restart: unless-stopped + + ntfy: + image: docker.io/binwiederhier/ntfy + command: serve + ports: + - "${NTFY_BIND:-127.0.0.1}:8091:80" + volumes: + - ntfy-cache:/var/cache/ntfy + environment: + - NTFY_BASE_URL=${NTFY_BASE_URL:-http://localhost:8091} + restart: unless-stopped + +volumes: + searxng-data: + chromadb-data: + ntfy-cache: diff --git a/docker-compose.yml b/docker-compose.yml index 2d8e3083d..cbeec1e37 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,26 +2,64 @@ services: odysseus: build: . ports: - - "7000:7000" + - "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000" volumes: - - ./data:/app/data - - ./logs:/app/logs + - ${APP_DATA_DIR:-./data}:/app/data:z + - ${APP_LOGS_DIR:-./logs}:/app/logs:z # Cookbook remote-server SSH identity. Odysseus can generate a key here; # add the shown public key to each remote server's authorized_keys. - - ./data/ssh:/app/.ssh + - ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z # Cookbook local model cache. Inside Docker, "Local" means the Odysseus # container, so persist its HuggingFace cache under ./data/huggingface. - - ./data/huggingface:/app/.cache/huggingface + - ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z + # Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.) + # land under /app/.local for the odysseus user. Persist them so a + # container recreate does not silently remove installed serve engines. + - ${APP_DATA_DIR:-./data}/local:/app/.local:z extra_hosts: # Lets the container reach local services on the Docker host, including # Ollama at http://host.docker.internal:11434. - "host.docker.internal:host-gateway" - env_file: - - .env environment: + - LLM_HOST=${LLM_HOST:-localhost} + - LLM_HOSTS=${LLM_HOSTS:-} + - OPENAI_API_KEY=${OPENAI_API_KEY:-} + - OLLAMA_BASE_URL=${OLLAMA_BASE_URL:-} + - RESEARCH_LLM_ENDPOINT=${RESEARCH_LLM_ENDPOINT:-} + - HF_TOKEN=${HF_TOKEN:-} + - HUGGING_FACE_HUB_TOKEN=${HUGGING_FACE_HUB_TOKEN:-} - SEARXNG_INSTANCE=http://searxng:8080 - CHROMADB_HOST=chromadb - CHROMADB_PORT=8000 + - DATABASE_URL=${DATABASE_URL:-sqlite:///./data/app.db} + - AUTH_ENABLED=${AUTH_ENABLED:-true} + - LOCALHOST_BYPASS=${LOCALHOST_BYPASS:-false} + - ODYSSEUS_ADMIN_USER=${ODYSSEUS_ADMIN_USER:-admin} + - ODYSSEUS_ADMIN_PASSWORD=${ODYSSEUS_ADMIN_PASSWORD:-} + - ALLOWED_ORIGINS=${ALLOWED_ORIGINS:-http://localhost,http://127.0.0.1} + - SECURE_COOKIES=${SECURE_COOKIES:-false} + - EMBEDDING_URL=${EMBEDDING_URL:-} + - EMBEDDING_MODEL=${EMBEDDING_MODEL:-} + - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-} + - FASTEMBED_MODEL=${FASTEMBED_MODEL:-sentence-transformers/all-MiniLM-L6-v2} + - FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-} + - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24} + - ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1} + - ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1} + - ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost} + - ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760} + - ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600} + - ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760} + - ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400} + - ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400} + - ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760} + - DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-} + - GOOGLE_API_KEY=${GOOGLE_API_KEY:-} + - GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-} + - TAVILY_API_KEY=${TAVILY_API_KEY:-} + - SERPER_API_KEY=${SERPER_API_KEY:-} # PUID / PGID — the user/group the container drops to before # running uvicorn (entrypoint also chowns /app/data + /app/logs # to match, so bind-mounted files stay editable from the host). @@ -40,7 +78,7 @@ services: restart: unless-stopped chromadb: - image: chromadb/chroma:latest + image: docker.io/chromadb/chroma:latest ports: - "${CHROMADB_BIND:-127.0.0.1}:8100:8000" volumes: @@ -50,7 +88,12 @@ services: restart: unless-stopped searxng: - image: searxng/searxng:latest + # Pinned, not :latest — odysseus waits on searxng's healthcheck + # (depends_on: condition: service_healthy), so a broken upstream `latest` + # tag blocks the whole app from starting. 2026.6.2 crashes on boot with + # `KeyError: 'default_doi_resolver'`, failing the healthcheck (issue #1414). + # Bump this deliberately after verifying a newer tag boots clean. + image: docker.io/searxng/searxng:2026.5.31-7159b8aed entrypoint: - /bin/sh - -c @@ -68,10 +111,24 @@ services: - "127.0.0.1:8080:8080" volumes: - searxng-data:/etc/searxng - - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro + - ./config/searxng/settings.yml:/tmp/searxng-settings.yml.template:ro,z environment: - SEARXNG_BASE_URL=http://localhost:8080/ - SEARXNG_SECRET=${SEARXNG_SECRET:-} + # The official searxng image runs as the non-root `searxng` user, but its + # entrypoint still needs to chown /etc/searxng on first boot, drop privs via + # su-exec, and (with our wrapper above) write settings.yml into the named + # volume. Without these capabilities the wrapper aborts at the redirection + # with EACCES and the container fails its healthcheck with permission + # errors during setup. Mirrors the cap set recommended by the upstream + # searxng-docker compose file. See issue #721. + cap_drop: + - ALL + cap_add: + - CHOWN + - SETGID + - SETUID + - DAC_OVERRIDE healthcheck: test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8080/', timeout=5).read(1)\""] interval: 5s @@ -81,7 +138,7 @@ services: restart: unless-stopped ntfy: - image: binwiederhier/ntfy + image: docker.io/binwiederhier/ntfy command: serve ports: - "${NTFY_BIND:-127.0.0.1}:8091:80" diff --git a/docker/build-realesrgan-wheels.sh b/docker/build-realesrgan-wheels.sh new file mode 100755 index 000000000..311b412cf --- /dev/null +++ b/docker/build-realesrgan-wheels.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Build patched wheels for Real-ESRGAN's unmaintained dependencies. +# +# basicsr / gfpgan / facexlib (xinntao, last released 2022) read their version +# in setup.py with: +# +# exec(compile(f.read(), version_file, 'exec')) +# return locals()['__version__'] +# +# Python 3.13+ implements PEP 667: locals() inside a function returns an +# independent snapshot that exec() can no longer mutate, so the read raises +# `KeyError: '__version__'` and the sdist build fails. That is why the Cookbook +# "install realesrgan" button dies on the python:3.14 image. The packages have +# no fixed release, so we patch get_version() to exec into an explicit namespace +# dict (works on every Python) and build wheels from the patched source. +# +# Usage: build-realesrgan-wheels.sh [OUTPUT_DIR] (default: /wheels) +set -euo pipefail + +OUT="${1:-/wheels}" +mkdir -p "$OUT" + +work="$(mktemp -d)" +trap 'rm -rf "$work"' EXIT +cd "$work" + +# Pinned to the versions Real-ESRGAN 0.3.0 resolves to. +SPECS="basicsr==1.4.2 gfpgan==1.3.8 facexlib==0.3.0" + +for spec in $SPECS; do + name="${spec%%==*}" + ver="${spec##*==}" + # pip download builds metadata (and trips the same bug), so fetch the raw + # sdist URL from the PyPI JSON API instead. + url="$(python - "$name" "$ver" <<'PY' +import json, sys, urllib.request +name, ver = sys.argv[1], sys.argv[2] +data = json.load(urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{ver}/json")) +for f in data["urls"]: + if f["packagetype"] == "sdist": + print(f["url"]); break +else: + sys.exit(f"no sdist found for {name}=={ver}") +PY +)" + echo ">> fetching ${name} ${ver}: ${url}" + curl -fsSL "$url" -o "${name}.tar.gz" + tar xzf "${name}.tar.gz" +done + +echo ">> patching get_version()" +python - <<'PY' +import pathlib +old_exec = "exec(compile(f.read(), version_file, 'exec'))" +new_exec = "_ver_ns = {}\n exec(compile(f.read(), version_file, 'exec'), _ver_ns)" +old_ret = "return locals()['__version__']" +new_ret = "return _ver_ns['__version__']" +patched = 0 +for setup in pathlib.Path(".").glob("*/setup.py"): + s = setup.read_text() + if old_exec in s and old_ret in s: + setup.write_text(s.replace(old_exec, new_exec).replace(old_ret, new_ret)) + print(" patched", setup) + patched += 1 +assert patched == 3, f"expected to patch 3 setup.py files, patched {patched}" +PY + +echo ">> building wheels into ${OUT}" +pip wheel --no-deps -w "$OUT" ./basicsr-* ./gfpgan-* ./facexlib-* +ls -l "$OUT" diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh index dd4cb2aeb..aec3b8eec 100644 --- a/docker/entrypoint.sh +++ b/docker/entrypoint.sh @@ -13,6 +13,8 @@ set -e PUID="${PUID:-1000}" PGID="${PGID:-1000}" +GOSU_BIN="$(command -v gosu)" +PYTHON_BIN="$(command -v python)" # Reuse an existing matching group/user if the host's UID/GID already # corresponds to one in /etc/passwd (e.g. when the image is rebuilt @@ -24,29 +26,121 @@ if ! getent passwd "$PUID" >/dev/null 2>&1; then useradd -u "$PUID" -g "$PGID" -M -s /bin/sh -d /app odysseus fi -# Repair ownership on every writable path the app touches at runtime. -# -# Bind-mounted dirs (/app/data, /app/logs) are the obvious ones, but -# the app ALSO writes inside the image's own source tree at runtime: -# - services/cache/{search,content}/* (search cache LRU) -# - services/search_analytics.json -# - services/search_engine_error.log -# - services/tts cache, etc. -# These dirs were created as root during `docker build`, so dropping -# to PUID:PGID would otherwise crash on the first import that tries -# to mkdir them. Chown the whole /app tree — fast (<1s on this size) -# and idempotent via the `-not -uid` filter so we only touch files -# that need fixing. -for dir in /app /app/data /app/logs; do +ODY_USER="$(getent passwd "$PUID" | cut -d: -f1)" +[ -z "$ODY_USER" ] && ODY_USER=odysseus + +# Docker-socket group plumbing for the explicit host-Docker overlay. When +# opted in, the socket is owned by root:. Add the app user +# to that group and later call gosu by username so supplementary groups are +# retained. +DOCKER_SOCK="${DOCKER_SOCK:-/var/run/docker.sock}" +if [ "${ODYSSEUS_ENABLE_HOST_DOCKER:-}" = "true" ] && [ -S "$DOCKER_SOCK" ]; then + SOCK_GID="$(stat -c '%g' "$DOCKER_SOCK" 2>/dev/null || echo '')" + if [ -n "$SOCK_GID" ] && [ "$SOCK_GID" != "0" ]; then + if ! getent group "$SOCK_GID" >/dev/null 2>&1; then + groupadd -g "$SOCK_GID" docker_host || true + fi + SOCK_GROUP="$(getent group "$SOCK_GID" | cut -d: -f1)" + if [ -n "$SOCK_GROUP" ]; then + usermod -aG "$SOCK_GROUP" "$ODY_USER" 2>/dev/null || true + fi + fi +fi + +mount_root_for() { + awk -v target="$1" '$5 == target { print $4; exit }' /proc/self/mountinfo 2>/dev/null || true +} + +is_broad_mount_root() { + case "$1" in + /|/home|/srv|/var|/usr|/opt|/tmp|/mnt|/media) + return 0 + ;; + esac + return 1 +} + +repair_tree_ownership() { + dir="$1" if [ -d "$dir" ]; then - # `find ... -not -uid` keeps this O(touched-files), not - # O(everything), so terabyte-sized maildirs don't slow startup. - find "$dir" -not -uid "$PUID" -print0 2>/dev/null \ + find "$dir" -xdev -not -uid "$PUID" -print0 2>/dev/null \ + | xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true + fi +} + +repair_app_tree_ownership() { + if [ -d /app ]; then + find /app -xdev \ + \( -path /app/data -o -path /app/logs -o -path /app/.ssh -o -path /app/.cache -o -path /app/.local \) -prune \ + -o -not -uid "$PUID" -print0 2>/dev/null \ | xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true fi +} + +repair_bind_mount_ownership() { + dir="$1" + if [ ! -d "$dir" ]; then + return + fi + + mount_root="$(mount_root_for "$dir")" + if is_broad_mount_root "$mount_root"; then + echo "Skipping recursive ownership repair for $dir because it maps to broad host path $mount_root" >&2 + chown "$PUID:$PGID" "$dir" 2>/dev/null || true + return + fi + + repair_tree_ownership "$dir" +} + +# Repair image-owned writable paths without walking into bind-mounted host +# trees, then repair the app-owned mount roots separately. +repair_app_tree_ownership +for dir in /app/data /app/logs /app/.ssh /app/.cache/huggingface /app/.local; do + repair_bind_mount_ownership "$dir" +done + +# Cookbook installs vllm/etc. via `pip install --user`, which pulls +# nvidia-cuda-* wheels into /app/.local but does not set CUDA_HOME or +# symlink /usr/local/cuda. vllm 0.22+ then crashes during engine init +# when FlashInfer tries to JIT a sampler kernel ("Could not find nvcc", +# then "CUDA compiler and toolkit headers are incompatible" on the +# mixed cuda-nvcc 13.3 / cuda-runtime 13.0 wheel combo). +# +# Auto-set CUDA_HOME if a pip-installed nvcc is present, and disable the +# FlashInfer JIT sampler — sampler only, no impact on attention path. +# No-op when vllm isn't installed. +# +# Checked layouts (all are real pip-wheel install paths): +# nvidia/cu13 — nvidia-nvcc-cu13 (CUDA 13.x wheel style) +# nvidia/cu12 — nvidia-nvcc-cu12 (CUDA 12.x wheel style) +# nvidia/cuda_nvcc — nvidia-cuda-nvcc-cu12 (older cu12 sub-package style) +for cu in \ + /app/.local/lib/python*/site-packages/nvidia/cu13 \ + /app/.local/lib/python*/site-packages/nvidia/cu12 \ + /app/.local/lib/python*/site-packages/nvidia/cuda_nvcc; do + if [ -x "$cu/bin/nvcc" ]; then + export CUDA_HOME="$cu" + break + fi done +# Disable the FlashInfer JIT sampler unconditionally — it is sampler-only +# and has no impact on the attention path, but requires nvcc + matching +# CUDA headers at startup. Without this, vLLM crashes with "Could not find +# nvcc" even when the GPU itself is fully visible to the container. +export VLLM_USE_FLASHINFER_SAMPLER="${VLLM_USE_FLASHINFER_SAMPLER:-0}" + +# Make Cookbook-installed Python CLIs visible after `pip install --user`. +# vLLM and helper scripts land here because /app is the non-root user's HOME. +export PATH="/app/.local/bin:$PATH" + +# Run first-time setup as the app user so data/ files get the right ownership. +# setup.py is idempotent — skips auth.json / .env if they already exist. +# || true so a setup failure never prevents the container from starting. +"$GOSU_BIN" "$ODY_USER" "$PYTHON_BIN" /app/setup.py || true + # Drop root and run the actual app. `gosu` is preferred over `su` / # `sudo` because it cleans up the process tree (no extra shell layer) # so signals (SIGTERM from `docker stop`) reach uvicorn directly. -exec gosu "$PUID:$PGID" "$@" +exec "$GOSU_BIN" "$ODY_USER" "$@" diff --git a/docker/gpu.amd.yml b/docker/gpu.amd.yml new file mode 100644 index 000000000..1bda9cfdd --- /dev/null +++ b/docker/gpu.amd.yml @@ -0,0 +1,19 @@ +# AMD ROCm GPU overlay. Enable by setting COMPOSE_FILE in .env: +# COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml +# RENDER_GID= +# +# Requires ROCm drivers on the host (kfd + DRI devices). The host user +# running Docker must be in the `video` and `render` groups. +# +# This overlay only passes the host GPU through to the container. +# The slim Odysseus image does not bundle ROCm userspace or inference +# engines — install ROCm-compatible builds of vLLM / llama-cpp-python +# via Cookbook -> Dependencies (or pip) before serving GPU models. +services: + odysseus: + devices: + - /dev/kfd + - /dev/dri + group_add: + - video + - ${RENDER_GID:-render} diff --git a/docker/gpu.nvidia.yml b/docker/gpu.nvidia.yml new file mode 100644 index 000000000..5590ba439 --- /dev/null +++ b/docker/gpu.nvidia.yml @@ -0,0 +1,34 @@ +# NVIDIA GPU overlay. Enable by setting COMPOSE_FILE in .env: +# COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml +# +# Use scripts/check-docker-gpu.sh to diagnose GPU passthrough, optionally +# install the NVIDIA Container Toolkit (Ubuntu/Debian), and write COMPOSE_FILE +# to .env. The script is read-only by default — it installs nothing and never +# edits .env unless explicitly asked. +# +# Requires the NVIDIA Container Toolkit on the host. +# Arch: sudo pacman -S nvidia-container-toolkit +# Debian: sudo apt install nvidia-container-toolkit +# Fedora: sudo dnf install nvidia-container-toolkit +# Then: +# sudo nvidia-ctk runtime configure --runtime=docker +# sudo systemctl restart docker +# Verify with: +# docker info | grep -i nvidia +# +# This overlay only passes the host GPU through to the container. +# The slim Odysseus image does not bundle CUDA userspace or inference +# engines — install vLLM / llama-cpp-python / SGLang via +# Cookbook -> Dependencies (or pip) before serving GPU models. +services: + odysseus: + environment: + - NVIDIA_VISIBLE_DEVICES=all + - NVIDIA_DRIVER_CAPABILITIES=compute,utility + deploy: + resources: + reservations: + devices: + - driver: nvidia + count: all + capabilities: [gpu] diff --git a/docker/host-docker.yml b/docker/host-docker.yml new file mode 100644 index 000000000..b5b4f4968 --- /dev/null +++ b/docker/host-docker.yml @@ -0,0 +1,12 @@ +# High-trust host Docker access. Enable only when local Docker-daemon +# management from Cookbook is required and you accept that raw socket access +# grants broad control over the host Docker daemon. +# COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml +# DOCKER_GID= +services: + odysseus: + volumes: + - /var/run/docker.sock:/var/run/docker.sock + group_add: ["${DOCKER_GID:-963}"] + environment: + - ODYSSEUS_ENABLE_HOST_DOCKER=true diff --git a/docs/agent-migration.md b/docs/agent-migration.md new file mode 100644 index 000000000..ff082159e --- /dev/null +++ b/docs/agent-migration.md @@ -0,0 +1,194 @@ +# Agent migration manifests + +Odysseus should be able to learn from another agent without blindly trusting +that agent's whole state. The safe migration path is: + +```text +source agent export -> source adapter -> agent-migration.v1 manifest -> preview -> apply +``` + +The manifest is intentionally source-neutral. OpenClaw, Hermes, a folder of +Markdown notes, or any other agent can have its own adapter, but Odysseus only +needs to understand the normalized manifest. + +## Why not import everything as memory? + +Durable memory should stay compact and useful. Long notes, logs, session +transcripts, and project archives are useful context, but they are not all +memories. A good migration keeps two layers separate: + +- **Archive documents** preserve source material for search, reading, and later + extraction. +- **Memory candidates** are short facts or preferences that can be reviewed + before being saved into Odysseus memory. + +This keeps Odysseus' existing memory-review flow intact while giving it better +source material to review. + +## Manifest shape + +`agent-migration.v1` is a JSON object: + +```json +{ + "schema_version": "agent-migration.v1", + "generated_at": "2026-06-06T00:00:00Z", + "source": { + "name": "example-agent", + "kind": "generic" + }, + "summary": { + "item_count": 3, + "counts_by_kind": { + "memory": 1, + "skill": 1, + "conversation_thread": 1, + "archive_document": 1 + }, + "warning_count": 0 + }, + "items": [], + "warnings": [] +} +``` + +Each item has a stable `id`, a `kind`, source metadata, and enough content for a +future importer to preview it before applying. + +Supported item kinds in the first pass: + +- `memory` — a candidate memory with `text`, `category`, `source`, and + provenance metadata. +- `skill` — a `SKILL.md` file with content and parsed frontmatter metadata. +- `conversation_thread` — a normalized transcript thread from an exported chat + history. Message content is optional; adapters can preserve only thread + metadata, message counts, timestamps, and hashes when a manifest should stay + small or avoid embedding private transcript text. +- `archive_document` — long-form source material. Content is optional; adapters + can preserve only path/hash/size metadata when a manifest should stay small. + +## Build a manifest + +Use the read-only helper: + +```bash +python3 scripts/agent_migration_manifest.py \ + --source-name old-agent \ + --source-kind generic \ + --memory-json /path/to/memories.json \ + --skills-dir /path/to/skills \ + --conversation-json /path/to/conversations.json \ + --archive /path/to/notes \ + --output /tmp/agent-migration.json +``` + +The helper does not write to `data/`, call an LLM, import Odysseus modules, or +modify the source. It only writes JSON. + +Memory JSON may be: + +```json +[ + "A plain memory string", + { + "text": "A categorized memory", + "category": "preference", + "source": "old-agent" + } +] +``` + +or an object containing a list under `memories`, `memory`, `items`, or `data`. + +Skills are scanned recursively for `SKILL.md`: + +```bash +python3 scripts/agent_migration_manifest.py \ + --source-name hermes \ + --source-kind hermes \ + --skills-dir ~/.hermes/skills \ + --output /tmp/hermes-skills-manifest.json +``` + +Archive documents are metadata-only by default. To embed text content: + +```bash +python3 scripts/agent_migration_manifest.py \ + --source-name notes-export \ + --archive /path/to/markdown-notes \ + --include-archive-content \ + --output /tmp/notes-manifest.json +``` + +Conversation exports are also metadata-only by default: + +```bash +python3 scripts/agent_migration_manifest.py \ + --source-name chatgpt-export \ + --source-kind chatgpt \ + --conversation-json /path/to/conversations.json \ + --output /tmp/chatgpt-conversations-manifest.json +``` + +The first pass supports generic conversation JSON such as: + +```json +[ + { + "id": "thread-1", + "title": "Project plan", + "messages": [ + {"role": "user", "content": "Can we design this?"}, + {"role": "assistant", "content": "Yes, start with a narrow slice."} + ] + } +] +``` + +It also recognizes ChatGPT-style `mapping` exports from `conversations.json`. +To embed normalized messages: + +```bash +python3 scripts/agent_migration_manifest.py \ + --source-name chatgpt-export \ + --source-kind chatgpt \ + --conversation-json /path/to/conversations.json \ + --include-conversation-content \ + --max-conversation-messages 2000 \ + --output /tmp/chatgpt-conversations-with-content.json +``` + +Content embedding is explicit because exported chat histories can be huge and +private. A future source-specific adapter can add ZIP traversal, attachment +metadata, and provider-specific project/workspace fields while still emitting +the same `conversation_thread` manifest item. + +## Recommended apply behavior + +A future Odysseus importer should treat the manifest as untrusted user-provided +data and apply it in stages: + +1. Show a dry-run summary with counts, warnings, duplicates, and sample items. +2. Back up current `data/` state before writing anything. +3. Import archive documents as documents or another searchable source, not as + memory. +4. Import conversation threads as searchable archived context first, with + citations back to the source thread. Do not turn whole transcripts into + memory. +5. Show memory candidates for review before saving through the normal memory + path. +6. Import skills only after name/category conflict checks. +7. Skip secrets by default. Credentials need explicit, provider-specific flows. + +## What belongs in source adapters? + +Adapters can be source-specific. The core manifest should not be. + +For example, an OpenClaw adapter may know about OpenClaw's workspace files. A +Hermes adapter may know about `~/.hermes/config.yaml` and `~/.hermes/skills`. +A ChatGPT adapter may know about `conversations.json`, uploaded-file metadata, +and image attachment directories. A Claude adapter may know about Claude's +export shape and project boundaries. A generic adapter may only know about +memory JSON, conversation JSON, `SKILL.md`, and Markdown folders. + +Nonstandard folders should be adapter details, not required Odysseus concepts. diff --git a/docs/attachments.md b/docs/attachments.md new file mode 100644 index 000000000..93f9e0ffe --- /dev/null +++ b/docs/attachments.md @@ -0,0 +1,85 @@ +# Attachment References and Upload Storage + +Odysseus stores uploaded bytes once under the configured upload directory and +passes stable references through chat history, tools, and future artifact work. +The goal is to avoid duplicating large inline media payloads in +`chat_messages.content` or the SQLite FTS index. + +## Reference Shape + +Attachment references use this minimum shape: + +```json +{ + "type": "attachment_ref", + "attachment_id": "32hex-or-32hex.ext", + "name": "original-filename.png", + "mime": "image/png", + "size": 12345, + "checksum_sha256": "hex-digest", + "created_at": "2026-07-09T12:00:00" +} +``` + +Optional fields such as `width`, `height`, `vision`, `vision_model`, and +`gallery_id` may be present when the uploader or preprocessing path knows them. + +## Persistence + +The live model call may still receive provider-specific multimodal blocks for +the current turn. Persistence is different: + +- `chat_messages.content` stores readable text plus compact attachment reference + lines, never raw `data:*;base64,...` upload bytes. +- `chat_messages.metadata.attachments` stores structured attachment reference + metadata for UI reloads and future processing. +- The SQLite FTS migration recreates chat-message FTS triggers so new rows do + not index inline media payloads, and it scrubs legacy rows that were already + indexed with data URLs. + +## Tool Access + +Agent/tool context receives upload entries as `attachment_ref` manifests with an +`odysseus://attachment/` URI and `read_policy: "owner_checked_upload"`. + +For compatibility with existing built-in tools, a local `path` may be included +only after all of these checks pass: + +- the upload ID resolves through `UploadHandler.resolve_upload`; +- the requested owner is allowed to read the upload; +- the file remains inside the configured upload directory; +- the file path is inside the tool-readable roots. + +External MCP/custom tools should treat the URI and attachment ID as the stable +contract and request bytes through an owner-checked server path, not by assuming +host filesystem layout. + +## Retention and Deletion + +Current retention behavior is conservative: + +- uploads are indexed in `uploads.json` with owner, checksum, MIME type, size, + and creation time; +- admin cleanup first scans persisted chat metadata/content, document versions, + PDF source markers, gallery hashes, notes, and calendar records for live + references; +- cleanup fails closed if that reference scan cannot complete, and the lower-level + cleanup API removes nothing unless it receives a complete reference snapshot; +- expired, unreferenced uploads are removed during the completed scan, while + attachment-bearing writers must first take an owner-checked reservation that + serializes with deletion and refreshes the upload's access timestamp; +- deliberate removal atomically drops matching `uploads.json` rows before deleting + the bytes and restores those rows if filesystem removal fails; +- deleting a chat removes the chat rows but does not immediately delete shared + upload bytes, because the same upload may also be referenced by gallery items, + documents, duplicate-upload rows, or future artifact records. + +There is no distinct artifact table in the current schema. Artifact-like upload +references persisted in chat or document text are covered by the canonical +attachment-ID scan; any future artifact store must be added to reference discovery +before cleanup is allowed to consider its uploads unreferenced. + +Cleanup and write reservations share the upload-index lock. This closes the +scan/write/delete race in the documented single-worker deployment; a future +multi-process deployment must add an inter-process lock or move lifecycle state +into the database before enabling destructive cleanup in more than one worker. diff --git a/docs/backup-restore.md b/docs/backup-restore.md new file mode 100644 index 000000000..902c9e683 --- /dev/null +++ b/docs/backup-restore.md @@ -0,0 +1,129 @@ +# Backup & Restore + +Odysseus keeps all of your state in the `data/` directory — the SQLite database +(`app.db`), the Fernet encryption key (`data/.app_key`), the vault, memory, RAG +indexes, personal documents, and uploads. The `scripts/odysseus-backup` tool +snapshots that directory into a single gzip tarball and restores it later. + +Snapshots are safe to take while the app is running: SQLite databases are copied +through SQLite's own `.backup` API rather than a raw file copy, so an in-flight +write can't corrupt the snapshot. + +> **A snapshot contains your secrets.** The tarball includes the Fernet +> encryption key (`data/.app_key`), the vault, sessions, and any stored +> provider/API tokens — so treat it like a password. Store backups somewhere +> private, never commit them to Git, and prefer an encrypted destination when +> copying them offsite. + +## Quick start + +Run the tool from the repository root: + +```bash +# Create a snapshot → backups/odysseus-backup-.tar.gz +./scripts/odysseus-backup snapshot + +# List existing snapshots (most recent first) +./scripts/odysseus-backup list + +# Check a tarball's integrity without extracting it +./scripts/odysseus-backup verify backups/odysseus-backup-20260101-120000.tar.gz + +# Restore (destructive — see the warning below) +./scripts/odysseus-backup restore backups/odysseus-backup-20260101-120000.tar.gz --yes +``` + +The script depends only on the Python standard library, so any `python3` on your +`PATH` will run it — you don't need the app's virtualenv active. + +Every command prints a JSON result. Add `--pretty` for indented output. + +## Commands + +### `snapshot` + +Writes a `tar.gz` of `data/` to `backups/.tar.gz`. + +| Flag | Effect | +| --- | --- | +| `--out PATH` | Write to a specific path instead of the default `backups/` location. Must be **outside** `data/`. | +| `--include-research` | Include `data/deep_research/` (skipped by default — research runs are large). | +| `--include-attachments` | Include `data/mail-attachments/` (skipped by default — cached IMAP extractions, re-derivable). | + +By default the snapshot includes everything under `data/` **except** +`deep_research/` and `mail-attachments/`. Personal uploads and documents are +included. + +```bash +# Snapshot straight to a mounted NAS path +./scripts/odysseus-backup snapshot --out /mnt/nas/odysseus-$(date +%F).tar.gz + +# Full snapshot including research runs and mail attachments +./scripts/odysseus-backup snapshot --include-research --include-attachments +``` + +### `list` + +Lists the tarballs in `backups/`, most recent first, with size and modification +time. + +### `verify PATH` + +Opens the tarball read-only and walks every member to confirm it is intact and +safe to restore. Nothing is extracted. Use this before relying on an old backup +or after copying one across machines. + +### `restore PATH --yes` + +Overwrites `data/` from a tarball. + +> **Restore is destructive.** It replaces the current `data/` directory. `--yes` +> is required so a mistyped command can't wipe your live state. + +Restore is not a blind delete: before extracting, the tool **renames your current +`data/` to `data.before-restore-`** in the repository root. If a +restore turns out to be wrong, your previous state is still there — delete the +restored `data/` and rename the stashed directory back. The restore path is also +validated entry-by-entry: archives containing absolute paths, `..` segments, +symlinks, or anything outside `data/` are rejected. + +## Scheduling offsite backups + +The tarball output composes cleanly with cron and any copy tool. For example, a +nightly snapshot copied offsite: + +```cron +0 3 * * * cd /path/to/odysseus && ./scripts/odysseus-backup snapshot --out "/mnt/nas/odysseus-$(date +\%F).tar.gz" +``` + +Swap the `--out` target for `scp`, `rclone`, `s3cmd`, or similar to push the +snapshot to remote storage. + +## Docker vs native installs + +The tool reads `data/` and writes `backups/` relative to the repository root, so +where you run it matters: + +- **Native installs** — run it from the repo root as shown above. `data/` and + `backups/` are both in the repo directory. +- **Docker** — `docker-compose.yml` bind-mounts the host's `./data` to + `/app/data`, so the live data is also present on the host. **Run the tool on + the host** from the repo root; the snapshot reads the bind-mounted `./data` and + writes to `./backups` on the host. Running it *inside* the container is not + recommended, because `backups/` is not a mounted volume and the tarball would + be lost when the container is recreated. + +> **ChromaDB caveat (Docker only).** In the Docker setup, ChromaDB stores its +> vectors in a separate Compose-managed volume (declared as `chromadb-data`), +> **not** under `./data`. `odysseus-backup` therefore does not capture the Docker +> ChromaDB store. Back it up separately if you need it. Compose prefixes the +> volume with the project name, so find the real name first +> (`docker volume ls | grep chromadb`), then archive it — for example: +> +> ```bash +> docker run --rm -v _chromadb-data:/data -v "$PWD":/backup \ +> alpine tar czf /backup/chromadb.tar.gz -C /data . +> ``` +> +> On native installs ChromaDB lives at `data/chroma/` and is included in the +> snapshot normally. diff --git a/docs/chat.gif b/docs/chat.gif deleted file mode 100644 index 90ca0eaac..000000000 Binary files a/docs/chat.gif and /dev/null differ diff --git a/docs/compare.gif b/docs/compare.gif deleted file mode 100644 index 7b939aa01..000000000 Binary files a/docs/compare.gif and /dev/null differ diff --git a/docs/document.gif b/docs/document.gif deleted file mode 100644 index b2a89e435..000000000 Binary files a/docs/document.gif and /dev/null differ diff --git a/docs/email-outlook.md b/docs/email-outlook.md new file mode 100644 index 000000000..1f8b97d5d --- /dev/null +++ b/docs/email-outlook.md @@ -0,0 +1,17 @@ +# Outlook / Office 365 email accounts + +Odysseus email accounts currently use IMAP and SMTP with username/password +authentication. That works for providers that still allow app passwords or +mailbox passwords for IMAP/SMTP. + +Microsoft disables basic authentication for Outlook and Microsoft 365 in most +modern accounts and tenants. If you try to add an Outlook account with a normal +password, Microsoft may return errors such as: + +- `IMAP: AUTHENTICATE failed` +- `SMTP: 535 5.7.139 Authentication unsuccessful, basic authentication is disabled` + +This is expected. Odysseus does not support Microsoft OAuth or Graph Mail yet, +so Outlook / Office 365 accounts cannot currently be added through the password +form. Use another email provider with app-password support, or track the future +Microsoft Graph OAuth integration. diff --git a/docs/index.html b/docs/index.html index 001dffd5c..c87ecc211 100644 --- a/docs/index.html +++ b/docs/index.html @@ -25,9 +25,16 @@ --radius: 8px; } * { box-sizing: border-box; } - html { scroll-behavior: smooth; scroll-snap-type: y mandatory; scroll-padding-top: 60px; } - /* Each section is a full-viewport "page" with its content centered, so only - one shows at a time and the snap is obvious. */ + html { scroll-behavior: smooth; scroll-padding-top: 60px; } + /* REMOVED: "scroll-snap-type: y proximity" + The idea was: >>Each section is a full-viewport "page" with its content centered, + so only one shows at a time and the snap is obvious.<< + + PROBLEM: sections easily grow taller than 100vh IRL + This cause forced jumps mid-read. It's intrusive UX. + The landing-page is not a PowerPoint presentation! + + Preserved: CSS snap-points to avoid destroying code meta-data*/ .hero, section { scroll-snap-align: start; min-height: 100vh; display: flex; flex-direction: column; justify-content: center; @@ -357,8 +364,16 @@ display: inline-flex; align-items: center; gap: 14px; margin: 18px auto 8px; background: var(--bg2); border: 1px solid var(--border); border-radius: 10px; padding: 12px 16px; font-family: ui-monospace, monospace; font-size: 14px; color: var(--fg); + text-align: left; } .codeblock .prompt { color: var(--accent); } + .codeblock .copy-btn { + background: none; border: 1px solid var(--border); border-radius: 6px; + color: var(--muted); cursor: pointer; font-size: 12px; padding: 4px 10px; + font-family: inherit; transition: border-color .12s ease, color .12s ease; + } + .codeblock .copy-btn:hover { border-color: var(--accent); color: var(--fg); } + .codeblock .copy-btn.copied { border-color: var(--green); color: var(--green); } .pill-row { display: flex; gap: 8px; justify-content: center; flex-wrap: wrap; margin-top: 44px; } .pill { font-size: 12.5px; color: var(--muted); border: 1px solid var(--border); border-radius: 999px; padding: 5px 12px; background: var(--panel); } @@ -369,6 +384,9 @@ .grid { grid-template-columns: repeat(2, 1fr); } .shotrow { grid-template-columns: 1fr; } .nav-links a:not(.btn) { display: none; } + .codeblock { display: flex; flex-wrap: wrap; gap: 8px; align-items: flex-start; overflow-wrap: anywhere; } + .codeblock > span { flex: 1 1 auto; min-width: 0; } + .codeblock .copy-btn { margin-left: auto; } } @media (max-width: 520px) { .grid { grid-template-columns: 1fr; } @@ -391,7 +409,7 @@ Testimonials How it started Get started - + GitHub @@ -419,7 +437,7 @@

Your own AI workspace,
running on your hardware

@@ -660,9 +678,9 @@

Uncompromised local LLM experience.

Get started

Odysseus is yours.

It's open source and free. No sales team, no demo request, no Trojan horse.

-
$ git clone https://github.com/pewdiepie-archdaemon/odysseus.git && cd odysseus
+
$ git clone https://github.com/odysseus-dev/odysseus.git && cd odysseus
Self-hosted @@ -925,6 +943,18 @@

Odysseus is yours.

show(0); })(); + + // Copy button for the codeblock command. + (function () { + var btn = document.querySelector('.codeblock .copy-btn'); + if (!btn) return; + btn.addEventListener('click', function () { + navigator.clipboard.writeText(btn.getAttribute('data-copy')).then(function () { + btn.textContent = 'Copied!'; btn.classList.add('copied'); + setTimeout(function () { btn.textContent = 'Copy'; btn.classList.remove('copied'); }, 2000); + }); + }); + })(); diff --git a/docs/notes.gif b/docs/notes.gif deleted file mode 100644 index 891ec2e1b..000000000 Binary files a/docs/notes.gif and /dev/null differ diff --git a/docs/odysseus-browser.jpg b/docs/odysseus-browser.jpg new file mode 100644 index 000000000..5c1e9a764 Binary files /dev/null and b/docs/odysseus-browser.jpg differ diff --git a/docs/odysseus-wordmark.png b/docs/odysseus-wordmark.png new file mode 100644 index 000000000..dce21eb66 Binary files /dev/null and b/docs/odysseus-wordmark.png differ diff --git a/docs/odysseus.jpg b/docs/odysseus.jpg index 982a00f77..9637929db 100644 Binary files a/docs/odysseus.jpg and b/docs/odysseus.jpg differ diff --git a/docs/pr-blocker-audit.md b/docs/pr-blocker-audit.md new file mode 100644 index 000000000..b56f28cb3 --- /dev/null +++ b/docs/pr-blocker-audit.md @@ -0,0 +1,188 @@ +# PR Blocker Audit + +`scripts/pr_blocker_audit.py` is a small, read-only triage helper for maintainers who need to inspect open pull request overlap before reviewing or starting related work. + +It is a triage helper, not a replacement for maintainer judgment. + +## What it does + +- Reads open PR metadata from a local JSON file or from `gh`. +- Reports files touched by more than one open PR. +- Groups active work into broad code areas. +- Ranks PRs with a deterministic heuristic score. +- Flags possible duplicate candidates based on title keyword overlap and changed-file similarity. +- Suggests quieter areas for conservative new work. +- Prints Markdown by default, compact terminal output when requested, or machine-readable JSON. + +## What it does not do + +- It does not post comments. +- It does not review, approve, label, close, merge, or otherwise mutate PRs. +- It does not add or run GitHub Actions. +- It does not import the Odysseus application package. +- It does not claim that a PR is definitely blocked or duplicated. + +## Read-only safety guarantee + +Offline mode only reads a local JSON file. Live mode runs read-only GitHub CLI commands: + +```bash +gh pr list --repo OWNER/REPO --state open --limit 1000 --json number,title,author,files,mergeStateStatus,reviewDecision,updatedAt,url +``` + +If a PR from that list has missing or empty changed-file metadata, live mode fills it with read-only per-PR REST calls: + +```bash +gh api --paginate "repos/OWNER/REPO/pulls/NUMBER/files?per_page=100" +``` + +If that GraphQL-backed command fails, it falls back to: + +```bash +gh api --paginate "repos/OWNER/REPO/pulls?state=open&per_page=100" +``` + +Per-PR file fetching makes live overlap results useful, but it can be slower on repositories with hundreds of open PRs. + +## Generate input JSON + +For repeatable offline audits, capture PR metadata first: + +```bash +gh pr list --repo OWNER/REPO --state open --limit 1000 --json number,title,author,files,mergeStateStatus,reviewDecision,updatedAt,url > open-prs.json +``` + +## Run offline mode + +```bash +python3 scripts/pr_blocker_audit.py --input open-prs.json +``` + +## Run live mode + +```bash +python3 scripts/pr_blocker_audit.py --repo OWNER/REPO +``` + +Live mode fetches up to 1000 open PRs by default. Use `--limit` to cap how many open PRs are fetched and analyzed, and `--top` to cap how many rows are displayed in ranked sections: + +```bash +python3 scripts/pr_blocker_audit.py --repo OWNER/REPO --limit 50 --top 10 +``` + +Live mode may take time on large PR queues because it fetches changed-file metadata for each PR that did not include it in the initial list response. Progress is shown on `stderr` by default only when `stderr` is a TTY: + +```bash +python3 scripts/pr_blocker_audit.py --repo OWNER/REPO --progress auto +python3 scripts/pr_blocker_audit.py --repo OWNER/REPO --progress always +python3 scripts/pr_blocker_audit.py --repo OWNER/REPO --progress never +``` + +Use `--quiet` to suppress progress and non-fatal warning output. Progress and warnings never go to `stdout`, so redirected reports and `--output` files remain clean. + +For a faster metadata-only scan, skip changed-file metadata entirely: + +```bash +python3 scripts/pr_blocker_audit.py --repo OWNER/REPO --no-fetch-files +``` + +## JSON output + +Use `--format json` for machine-readable output suitable for scripting or downstream tooling: + +```bash +python3 scripts/pr_blocker_audit.py --input open-prs.json --format json +python3 scripts/pr_blocker_audit.py --input open-prs.json --format json --output report.json +``` + +JSON output is stable and deterministic for the same input. It uses `sort_keys=True` so field order does not vary between runs. It never includes ANSI escape codes, even with `--color always`. Progress text is always `stderr`-only and never appears in JSON output. + +The top-level object contains these keys: + +- `summary` — scalar overview: `total_prs_analyzed`, `unique_files_touched`, `prs_missing_changed_file_metadata`, `main_overlap_drivers`, `highest_risk_areas`, `recommended_first_review_target` +- `locked_areas` — list of objects with `area`, `files` (top paths as a string), `prs` (list of PR numbers), `why`, `priority` +- `hot_files` — list of objects with `file`, `pr_count`, `pr_numbers` (list of PR numbers); capped at `--top` +- `review_priorities` — ranked list with `rank`, `number`, `score`, `title`, `url`, `merge_state`, `review_decision`, `reasons` (list); capped at `--top` +- `duplicate_candidates` — list of objects with `pr_numbers` (list) and `titles` (list, one entry per PR in the group) +- `safer_areas` — list of strings + +## Write output to a file + +```bash +python3 scripts/pr_blocker_audit.py --input open-prs.json --output pr-blocker-report.md +python3 scripts/pr_blocker_audit.py --input open-prs.json --format json --output report.json +``` + +Markdown and JSON output never include ANSI color codes. ANSI codes are stripped defensively when writing any output file. + +## Terminal output and color + +Use terminal output for quick interactive scans: + +```bash +python3 scripts/pr_blocker_audit.py --input open-prs.json --format terminal +``` + +Terminal output includes locked areas, hot files, review / blocker priorities, possible duplicate candidates, and safer areas. + +Color is readability-only. It is never included in Markdown reports and is stripped defensively when writing output files. Color modes are: + +```bash +python3 scripts/pr_blocker_audit.py --input open-prs.json --format terminal --color auto +python3 scripts/pr_blocker_audit.py --input open-prs.json --format terminal --color always +python3 scripts/pr_blocker_audit.py --input open-prs.json --format terminal --color never +``` + +`--no-color` is kept as an alias for `--color never`. With `--color auto`, color is used only for terminal output on a TTY when `NO_COLOR` is not set and output is not being written to a file. + +## Interpret locked areas + +Locked areas are broad categories with one or more open PRs. An area is higher priority when several PRs touch it, when PRs share files, or when the highest scoring PR in that area has risk signals. Treat this as a prompt to inspect the PRs together. + +`PRs missing changed-file metadata` counts PRs that still had no changed-file paths after live file fetching, or PRs from offline input that did not include files. Those PRs can still appear in area summaries from title matching, but file overlap analysis is weaker for them. + +`Docs / tooling / tests` is conservative: runtime PRs are not classified there just because they include tests or README changes. Docs-only, README-only, scripts-only, tests-only, or strongly titled docs/tooling/test work still maps there. + +`Other / unclassified` is kept visible for PRs that do not match the area rules. When most of it comes from missing file metadata, the report summarizes that instead of letting long PR lists dominate the locked-area section. + +## Interpret duplicate candidates + +Duplicate candidates are labeled as possible duplicate / needs human review. The script groups PRs only when their file sets are highly similar and their titles share meaningful keywords. Similar PRs can still be complementary. + +## Interpret heuristic scores + +The review priority score is deterministic for the same input. Recency is measured against the newest parseable PR update timestamp in the input, and the score uses simple weights for: + +- direct auth, bearer-token, API-token, privilege, or permission lifecycle signals +- security, secret, or data exposure keywords +- persistence, migration, database, SQLite, or Postgres keywords +- memory, vector, RAG, embedding, or retrieval keywords +- overlapping changed files +- clean merge state as a small actionability signal +- review state +- recently updated PRs when timestamp data exists + +Higher scores mean "inspect earlier", not "correct" or "merge-ready". Broad PRs can score high because they overlap many files and may block other work, but they still need normal review and validation. + +Dirty, blocked, conflicting, and unknown merge states are shown as risk/caution reasons. They do not add importance points by themselves. + +## Design note: intentional single-script layout + +`pr_blocker_audit.py` is intentionally kept as one standalone script. The goal is to keep this maintainer/contributor workflow helper low-friction while broader repo tooling and test-suite conventions are still evolving. Splitting it into packages or modules is not ruled out, but is deferred until there is a clearer settled pattern to follow. + +## Limitations + +- Some PRs may still lack changed files if GitHub file metadata calls fail or metadata-only mode is used. +- Area classification is intentionally small and editable. +- Title keyword matching misses semantic duplicates. +- Heuristic scoring cannot know project strategy, reviewer availability, or hidden dependency chains. +- Empty or missing file metadata produces a valid report but weak overlap analysis. + +## Validation + +```bash +python3 -m py_compile scripts/pr_blocker_audit.py tests/test_pr_blocker_audit.py +python3 -m pytest tests/test_pr_blocker_audit.py -q +python3 scripts/pr_blocker_audit.py --help +git diff --check +``` diff --git a/docs/research.gif b/docs/research.gif deleted file mode 100644 index b817eeb1a..000000000 Binary files a/docs/research.gif and /dev/null differ diff --git a/docs/security-ci.md b/docs/security-ci.md new file mode 100644 index 000000000..8cceea258 --- /dev/null +++ b/docs/security-ci.md @@ -0,0 +1,101 @@ +# Security CI guide + +This project runs a set of automated security checks on pull requests and +selected branch pushes. This page explains what each one does, whether it can +block a merge, and the few one-time settings you should turn on to get the full +benefit. + +## What runs, and why + +Most checks live in files under `.github/workflows/`. CodeQL uses the +checked-in advanced configuration in `.github/workflows/codeql.yml`. They run +automatically; you do not start them. + +| Check | What it protects against | Blocks a merge? | +|---|---|---| +| **Secret scan** (gitleaks) | An API key, token, or password being committed by mistake or on purpose | Yes | +| **Workflow security** (actionlint + zizmor) | A broken or insecure automation file that could leak the repo's access token | Yes | +| **Dependency review** | A pull request that adds a software library with a known security hole | Yes | +| **pip-audit** | Known security holes in the Python libraries already used | No (advisory) | +| **Container scan: hadolint** | Mistakes and insecure patterns in the `Dockerfile` | Yes | +| **Container scan: Trivy** | Known security holes in the Docker image | No (advisory) | +| **CodeQL** | Real bugs in the app's own code: injection, auth mistakes, path traversal | No (advisory) | + +"Blocks a merge" means a red X appears on the pull request and, once you enable +the setting below, the **Merge** button is disabled until it is fixed. + +"Advisory" means it reports problems into the repository's **Security** tab so +you can review them on your own schedule, but it never stops a merge. These are +advisory on purpose: they often flag long-standing issues in other people's +libraries, not something a given pull request introduced. + +## Where results appear + +- **Checks tab of a pull request**: the pass/fail of each check. A green tick is + good; a red X needs attention. +- **Security tab of the repository**: detailed findings from the advisory + scanners (Trivy and CodeQL). This is your dashboard. + +## If a check fails + +- **Secret scan failed**: a real credential may have been committed. Treat it as + leaked: rotate (regenerate) that key or token immediately, then remove it from + the file. Do not just delete the commit; assume it was seen. +- **Dependency review failed**: the pull request adds a library with a known + vulnerability. Ask the contributor to use a patched version, or decline the + change. +- **hadolint / workflow security failed**: the contributor changed the + `Dockerfile` or an automation file in a way the linter rejects. Ask them to + address the message shown in the failed check. + +## One-time settings to turn on + +These two settings unlock the full value. You only do them once. + +### 1. Require the blocking checks before merging + +This makes the **Merge** button refuse to work until the gating checks pass. + +1. Go to the repository on GitHub. +2. Click **Settings** (top right of the repo). +3. In the left sidebar, click **Branches**. +4. Under **Branch protection rules**, click **Add branch ruleset** (or **Add + rule**), and set the branch name pattern to `dev` (this is the branch all + pull requests target; `main` is fast-forwarded at releases). +5. Enable **Require status checks to pass before merging**. +6. In the search box that appears, add these checks by name: + - `Python syntax (compileall)` + - `JS syntax (node --check)` + - `gitleaks` + - `actionlint` + - `zizmor (Actions SAST)` + - `hadolint (Dockerfile lint)` + - `dependency-review (PR gate)` + + The first two come from the correctness CI (`ci.yml`); the rest are this + security suite. Leave pytest, pip-audit, Trivy, and CodeQL unchecked so they + stay advisory. +7. Also enable **Require a pull request before merging** and **Require review + from Code Owners** (this uses the `.github/CODEOWNERS` file so every change + needs your sign-off). +8. Click **Create** / **Save changes**. + +Note: a check name only appears in the list after it has run at least once, so +let the workflows run on one pull request first, then add them here. + +### 2. Turn on the Security tab features + +1. **Settings -> Code security** (or **Code security and analysis**). +2. Turn on **Dependency graph** (usually on by default for public repos) -- this + powers Dependency review and Dependabot. +3. Turn on **Dependabot alerts** and **Dependabot security updates**. +4. Under **Code scanning**, keep **Default setup** disabled. CodeQL is + configured by `.github/workflows/codeql.yml`; enabling default setup at the + same time causes GitHub to reject uploads from the checked-in workflow. + +## Keeping it current + +`.github/dependabot.yml` opens small weekly pull requests to update Python and +npm packages, the Docker base image, and the pinned automation actions +themselves. Review and merge those like any other pull request; they keep the +project patched without manual tracking. diff --git a/docs/setup.md b/docs/setup.md new file mode 100644 index 000000000..53a6fb28c --- /dev/null +++ b/docs/setup.md @@ -0,0 +1,548 @@ +# Odysseus Setup Guide + +This page keeps the detailed install, deployment, troubleshooting, and configuration notes out of the front README. + +## Quick Start + +> **Branch note:** `dev` is the default branch and contains the latest development changes, but it may be unstable. For the more stable curated branch, use [`main`](https://github.com/odysseus-dev/odysseus/tree/main). + +Defaults work out of the box: clone, run, then configure models/search/email +inside **Settings**. Only edit `.env` for deployment-level overrides like +`APP_BIND`, `APP_PORT`, `AUTH_ENABLED`, `DATABASE_URL`, or a pre-seeded admin password. + +On first setup, Odysseus creates an admin account (`admin` unless +`ODYSSEUS_ADMIN_USER` is set) and prints a temporary password in the terminal. +For Docker installs, the same line is in `docker compose logs odysseus`. +Use that for the first login, then change it in **Settings**. + +Contributing? See [CONTRIBUTING.md](../CONTRIBUTING.md) for setup, testing, and +pull request guidelines. + +### Docker (recommended) +```bash +git clone https://github.com/odysseus-dev/odysseus.git +cd odysseus +cp .env.example .env # optional, but recommended for explicit defaults +docker compose up -d --build +``` +To include optional extras in the image (PDF viewer, Office extraction; includes AGPL PyMuPDF), build with `docker compose build --build-arg INSTALL_OPTIONAL=true` before `up`. + +Open `http://localhost:7000` when the containers are healthy. Docker Compose +binds the web UI to `127.0.0.1` by default. If the port is taken, set +`APP_PORT=7001` in `.env` and recreate the container. Set `APP_BIND=0.0.0.0` +only when you intentionally want LAN/reverse-proxy access. + +> **On Apple Silicon (M-series) Macs:** Docker can't reach the Metal GPU, so +> Cookbook serves local models on CPU only. For GPU-accelerated model serving, +> run natively instead — see [Apple Silicon](#apple-silicon) below. + +### Native Linux / macOS +```bash +git clone https://github.com/odysseus-dev/odysseus.git +cd odysseus +python3 -m venv venv +source venv/bin/activate +pip install -r requirements.txt +python setup.py +python -m uvicorn app:app --host 127.0.0.1 --port 7000 +``` +Requirements: Python 3.11+. Cookbook also needs `tmux` for background model +downloads and serves. The app itself is lightweight; local model serving is the +heavy part and depends on the model, runtime, GPU, and VRAM, so small hosts can +connect to API or remote model servers instead. Use `--host 0.0.0.0` only when you intentionally want LAN/reverse-proxy access. + +### Apple Silicon +Docker on macOS cannot use the Metal GPU. For GPU-accelerated Cookbook on an +M-series Mac, run Odysseus natively: + +```bash +git clone https://github.com/odysseus-dev/odysseus.git +cd odysseus +./start-macos.sh +``` + +It launches at `http://127.0.0.1:7860`. To expose it to your phone over a trusted LAN/VPN such as Tailscale, bind all interfaces: + +```bash +ODYSSEUS_HOST=0.0.0.0 ./start-macos.sh +# then open http://:7860 +``` + +The script also reads `.env` at startup, so `APP_BIND=0.0.0.0` and `APP_PORT` +set there are picked up automatically without a command-line override each run. + +Keep `AUTH_ENABLED=true` (the default) before binding outside loopback. Do not +expose this port directly to the public internet. To build a clickable app wrapper: + +```bash +./build-macos-app.sh +``` + +
+Cookbook, GPU, Ollama, and troubleshooting notes + +**Docker bundled services.** Compose starts Odysseus, ChromaDB, SearXNG, and +ntfy. Odysseus and the bundled service ports bind to `127.0.0.1` by default, so +they are reachable from the host but not exposed to your LAN/public internet +unless you opt in. + +**Cookbook storage in Docker.** Downloads live in `./data/huggingface` +(`~/.cache/huggingface` in the container). Cookbook-installed Python CLIs and +serve engines live in `./data/local` (`~/.local` in the container), so they +survive container recreation. + +**Remote servers.** In **Cookbook -> Settings -> Servers**, generate the +Odysseus SSH key and add the public key to the remote server's +`~/.ssh/authorized_keys`. From the host you can also run: + +```bash +ssh-copy-id -i data/ssh/id_ed25519.pub user@server +``` + +**Host Docker access (explicit opt-in).** Default Docker Compose intentionally +does not mount `/var/run/docker.sock`. You can still connect Odysseus to +existing Ollama, vLLM, and other OpenAI-compatible endpoints without Docker +socket access. + +Cookbook/local Docker-daemon management requires the opt-in overlay below. Raw +Docker socket access is high-trust because it can effectively grant broad +control over the host Docker daemon. Remote server Docker workflows over SSH +remain preferred. + +Place these values in `.env`, or export them in the shell before running +`docker compose`: + +```bash +COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml +DOCKER_GID= +``` + +Combine host Docker access with a GPU overlay when both are intentionally +required: + +```bash +COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml +# or +COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml +``` + +**Docker GPU overlays.** CPU-only users can skip this section. Cookbook can +only detect GPUs that Docker exposes to the container — if the host runtime or +device passthrough is not configured, Cookbook sees the iGPU, another card, or +CPU instead of your intended GPU. + +For NVIDIA, `scripts/check-docker-gpu.sh` diagnoses GPU passthrough and can +optionally install the host runtime or update `.env`. + +```bash +# Read-only diagnostic (default — installs nothing, never edits .env): +scripts/check-docker-gpu.sh + +# Print OS-specific install commands without running them: +scripts/check-docker-gpu.sh --print-install-commands + +# Install NVIDIA Container Toolkit on Ubuntu/Debian (requires sudo): +scripts/check-docker-gpu.sh --install-nvidia-toolkit + +# Write COMPOSE_FILE to .env (only when GPU passthrough is confirmed working): +scripts/check-docker-gpu.sh --enable-nvidia-overlay + +# Full assisted setup — install toolkit, then enable overlay if passthrough works: +scripts/check-docker-gpu.sh --install-nvidia-toolkit --enable-nvidia-overlay +``` +#### Arch Linux NVIDIA Docker notes + +On Arch Linux, verify the host NVIDIA driver and Docker GPU passthrough before enabling the Odysseus NVIDIA overlay. + +Install the required packages: + +```bash +sudo pacman -Syu +sudo pacman -S docker docker-compose nvidia-container-toolkit nvidia-utils +sudo systemctl enable --now docker +``` + +Configure Docker to use the NVIDIA container runtime: + +```bash +sudo nvidia-ctk runtime configure --runtime=docker +sudo systemctl restart docker +``` + +Verify the host GPU: + +```bash +nvidia-smi +``` + +Verify Docker GPU passthrough: + +```bash +docker run --rm --gpus all nvidia/cuda:12.9.0-base-ubuntu22.04 nvidia-smi +``` + +Then enable the Odysseus NVIDIA compose overlay: + +```env +COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml +``` + +Rebuild and verify the GPU inside the Odysseus container: + +```bash +docker compose up -d --build +docker compose exec odysseus nvidia-smi -L +``` + +For first-time local model testing on 8 GB laptop GPUs, start with GGUF/Q4 models on llama.cpp before trying GPTQ/AWQ models on vLLM or SGLang. This keeps the first run simpler while confirming GPU passthrough works. + +**WSL2 + snap Docker.** If the NVIDIA check fails with this error, Docker may be +installed via snap: + +```text +failed to fulfil mount request: open /usr/lib/wsl/lib/libdxcore.so: no such file or directory +``` + +Check with `snap list docker` or: + +```bash +docker info --format '{{.DockerRootDir}}' +``` + +A Docker root under `/var/snap/docker/` means snap confinement can prevent +Docker from seeing WSL2's `/usr/lib/wsl/lib` GPU libraries even when the files +exist on the host. Reinstalling or reconfiguring `nvidia-container-toolkit` will +not fix that. Remove snap Docker, install the official apt-based Docker Engine +([Docker docs](https://docs.docker.com/engine/install/ubuntu/)), then configure +the NVIDIA runtime again: + +```bash +sudo snap remove docker +sudo nvidia-ctk runtime configure --runtime=docker +sudo systemctl restart docker +``` + +Then re-run `scripts/check-docker-gpu.sh`. + +Safety notes: +- The app never installs host GPU runtime automatically. +- The app never edits `.env` automatically. +- `.env` is only modified when `--enable-nvidia-overlay` is explicitly passed, + and only after GPU passthrough succeeds. `--yes` skips prompts but does not + bypass the passthrough gate. +- `.env.bak.*` backups created by `--enable-nvidia-overlay` are ignored by + Git and the Docker build context. + +To enable manually without the script, add this to `.env`: + +```bash +COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml +``` + +**AMD / ROCm.** AMD setup is read-only diagnostic plus manual `.env` edit. Run: + +```bash +scripts/check-docker-amd-gpu.sh +``` + +Then add the reported values to `.env`, replacing `RENDER_GID` with your host's +numeric render group id: + +```bash +COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml +RENDER_GID=989 +``` + +For NVIDIA/AMD GPU support, also read the comments in the selected overlay file: docker/gpu.nvidia.yml or docker/gpu.amd.yml. + +**Stack-management UIs (Portainer, Coolify, Dockhand, etc.).** These tools +often accept only a single Compose file and do not reliably honor `COMPOSE_FILE` +or multiple `-f` overlays. CLI users should keep using the `COMPOSE_FILE` +overlay workflow above. For stack UIs, point the stack at one of the standalone +files instead, which bundle the base stack plus the GPU settings: + +- `docker-compose.gpu-nvidia.yml` — still requires the NVIDIA Container Toolkit + on the host. +- `docker-compose.gpu-amd.yml` — still requires host ROCm/kfd/DRI setup, the + `video`/`render` group membership, and `RENDER_GID` when needed. + +The base `docker-compose.yml` plus the `docker/gpu.*.yml` overlays remain the +source of truth; the standalone files mirror them for single-file deployments. + +Verify after enabling either overlay: + +```bash +docker compose exec odysseus nvidia-smi -L # NVIDIA +docker compose exec odysseus sh -lc 'test -e /dev/kfd && test -d /dev/dri && ls -l /dev/kfd /dev/dri/renderD*' # AMD +``` + +> **GPU passthrough ≠ llama.cpp CUDA.** `nvidia-smi` passing inside the +> container confirms Docker GPU access, but llama.cpp also needs `cudart` and +> the CUDA Toolkit at runtime. If Cookbook logs show `Unable to find cudart +> library`, `Could NOT find CUDAToolkit`, `CUDA Toolkit not found`, or +> tensors/layers assigned to CPU, that is a Cookbook/llama.cpp build issue — +> not a Docker passthrough failure. Reinstall the serve engine via +> **Cookbook → Dependencies** to get a CUDA-enabled build. +> +> The same split applies to AMD/ROCm: seeing `/dev/kfd` and `/dev/dri` inside +> the container confirms device passthrough, not ROCm userspace or a +> ROCm-enabled vLLM/llama.cpp build. `rocm-smi` and `rocminfo` are not expected +> inside the slim Odysseus image. + +**Ollama with Docker.** If Ollama runs on the host, add this endpoint in +Settings: + +```text +http://host.docker.internal:11434/v1 +``` + +Ollama must listen outside its own loopback interface: + +```bash +OLLAMA_HOST=0.0.0.0:11434 ollama serve +``` + +This connects Odysseus in Docker to an Ollama server that is already running on +your host machine; it does not start Ollama inside the container. +`host.docker.internal` is Docker's hostname for the host machine from inside the +container. Cookbook **Serve** is a separate workflow for serving downloaded +models through Odysseus/llama.cpp, so Windows users with an existing Ollama +install usually only need to add the endpoint in Settings. + +**Useful checks.** + +```bash +docker compose ps +docker compose logs --tail=120 odysseus +docker compose logs odysseus | grep -E 'ChromaDB|MemoryVectorStore|DEGRADED' +``` + +**macOS details.** `start-macos.sh` installs Homebrew deps, creates the venv, +runs setup, and starts uvicorn on port `7860` because AirPlay often holds +`7000`. It uses llama.cpp/Ollama for Metal. vLLM/SGLang are CUDA/ROCm-only and +do not run on macOS. MLX-only models are not served by Odysseus. + +
+ +### Native Windows + +**One-command launcher** (creates the venv, installs deps, runs setup, starts the +server; safe to re-run): + +```powershell +git clone https://github.com/odysseus-dev/odysseus.git +cd odysseus +powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1 +``` + +Or do it by hand: + +```powershell +git clone https://github.com/odysseus-dev/odysseus.git +cd odysseus +py -3.11 -m venv venv +venv\Scripts\Activate.ps1 +pip install -r requirements.txt +python setup.py +python -m uvicorn app:app --host 127.0.0.1 --port 7000 +``` + +If `python` points at an older interpreter, use `py -3.12` (or another installed +3.11+ version) for the venv step. + +**Exposing on a LAN/Tailscale (Windows):** the launcher binds to `127.0.0.1` and +does **not** read `APP_BIND` / `ODYSSEUS_HOST` from `.env`, so editing `.env` +alone leaves the native Windows server on loopback. Pass the launcher's +`-BindHost` flag instead: + +```powershell +powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1 -BindHost 0.0.0.0 +``` + +The manual `uvicorn` command takes the same address as `--host 0.0.0.0`. Bind +outside loopback only for a trusted LAN/VPN such as Tailscale: keep +`AUTH_ENABLED=true` and do not expose the port directly to the public internet. + +**Requirements:** Python 3.11+. The core app (chat, agent, memory, documents, +email, calendar, deep research) runs fully native. For full **Cookbook** background +model downloads and the agent shell tool, also install +[Git for Windows](https://git-scm.com/download/win) (provides `bash.exe`). +Local GPU *serving* of vLLM/SGLang needs Linux/WSL2; for a local model on Windows, +[Ollama](https://ollama.com/download) is the easiest path — point Odysseus at +`http://localhost:11434/v1` in Settings. + +Open `http://localhost:7000`, log in with the generated admin password, +and configure everything else inside **Settings**. + +## Troubleshooting & Advanced Setup + +### `chromadb-client` conflicts with embedded ChromaDB +If `chromadb-client` (the lightweight HTTP-only package) is installed alongside the full `chromadb` package, Odysseus starts but ChromaDB silently falls back to HTTP-only mode and fails. + +**Fix:** uninstall `chromadb-client` and force-reinstall the full package: +```bash +./venv/bin/pip uninstall chromadb-client -y +./venv/bin/pip install --force-reinstall chromadb +``` + +### HTTPS + LAN/Tailscale exposure +To expose Odysseus on a local network or Tailscale with HTTPS: +1. Change the bind address to `0.0.0.0` in `.env` (`APP_BIND=0.0.0.0` or `ODYSSEUS_HOST=0.0.0.0`). +2. Generate a locally-trusted cert for your LAN/Tailscale IPs using [mkcert](https://github.com/FiloSottile/mkcert): + ```bash + mkcert -install + mkcert -cert-file cert.pem -key-file key.pem 192.168.1.100 tailscale-ip + ``` +3. Run `uvicorn` with the generated certs: + ```bash + python -m uvicorn app:app --host 0.0.0.0 --port 7000 --ssl-certfile=cert.pem --ssl-keyfile=key.pem + ``` +4. Install the `mkcert` CA on any other device you want to access Odysseus from (e.g., for iOS, email the `rootCA.pem` to yourself, install the profile, and trust it in Certificate Trust Settings). + +### Common self-host traps (30-second fixes) +A grab-bag of small gotchas that otherwise turn into long debugging sessions. + +- **`AUTH_ENABLED=false` is ignored / you're still forced to log in (Windows).** If you edited `.env` in Notepad it may have saved a UTF-8 **BOM**, turning the first key into `AUTH_ENABLED` so it is never matched. Odysseus loads `.env` with `encoding="utf-8-sig"` to tolerate a leading BOM, but the safe fix is to re-save `.env` as **UTF-8 without BOM** (VS Code: *Save with Encoding → UTF-8*). +- **macOS: the app isn't at `http://localhost:7000`.** macOS AirPlay Receiver usually holds port `7000`, so the macOS start script serves on **`7860`** instead — open `http://localhost:7860`. To use `7000`, free it (System Settings → General → AirDrop & Handoff → turn off *AirPlay Receiver*) and set `APP_PORT=7000`. +- **Copy buttons do nothing over a plain-HTTP Tailscale/LAN URL.** Browsers only expose the clipboard API (`navigator.clipboard`) on **secure origins** — HTTPS, or `localhost`. Over `http://100.x.y.z:7860` it is blocked. Serve over HTTPS (see *HTTPS + LAN/Tailscale exposure* above); `localhost` is exempt, so copy still works on the host itself. +- **Self-hosted ntfy reminders don't reach your phone.** Two things: (1) the bundled ntfy binds to loopback by default — to reach it from your phone set `NTFY_BIND` to your host/Tailscale IP and `NTFY_BASE_URL` to the same server URL in `.env`, then recreate the ntfy container (see the `NTFY_*` block in `.env.example`); (2) in the ntfy **Android** app, subscribe to the topic with **Instant delivery** enabled — non-`ntfy.sh` servers don't get instant push otherwise. +- **Local mail (Dovecot) login fails: "Plaintext authentication disallowed on non-encrypted connections."** Your IMAP/SMTP server is refusing cleartext auth over an unencrypted link. Prefer enabling TLS on the mail server; on a trusted LAN only, you can allow cleartext (Dovecot: `disable_plaintext_auth = no`). +- **Calendar/contacts (Radicale) won't sync.** Point Odysseus at the **full collection URL** with its trailing slash — e.g. `http://host:5232///` — not just the server root. Radicale shows this address for each calendar/address book in its web UI. + +### Optional Dependencies +`requirements-optional.txt` contains packages that unlock extra features. It is not installed by default. + +| Package | Feature unlocked | +|---------|-----------------| +| `faster-whisper` | Local speech-to-text (microphone -> text) via the "local" STT provider. | +| `ddgs` | DuckDuckGo as a search provider option. | +| `PyMuPDF` | PDF page rendering in the side viewer panel and form-filling. (Note: AGPL-3.0) | +| `markitdown` | Office/EPUB document text extraction (converts .docx/.xlsx/.pptx/.xls/.epub to Markdown). | + +### Faster, reproducible installs with uv (optional) +[uv](https://docs.astral.sh/uv/) works as a drop-in replacement for the +venv + pip steps in the native install guides, no project changes are needed but this change results in faster installs along with a lockfile for reproducible environments. After [installing `uv`](https://docs.astral.sh/uv/getting-started/installation/), use: + +```bash +uv venv venv --python 3.13 +uv pip install -r requirements.txt +# then continue as usual: python setup.py, uvicorn, ... +``` + +`requirements.txt` is intentionally unpinned, so two installs at different times can produce different package versions. If you want a reproducible environment (e.g. across your own machines, or to roll back after a bad upgrade), snapshot and restore exact versions with: + +```bash +uv pip compile requirements.txt -o requirements.lock # snapshot current resolution +uv pip sync requirements.lock # reproduce it exactly later +``` + +`requirements.lock` is gitignored and platform-specific (compile it on the OS you deploy to). Regenerate it deliberately when you want to take upgrades. The plain `uv pip install -r requirements.txt` keeps following the unpinned requirements like pip does. + +### Outlook / Office 365 email +Odysseus email accounts currently use IMAP/SMTP username-password auth. Outlook +and Microsoft 365 generally require OAuth instead, so normal Microsoft mailbox +passwords will fail. See [docs/email-outlook.md](docs/email-outlook.md) for the +current limitation and the planned integration direction. + +## Security Notes +Odysseus is a self-hosted workspace with powerful local tools: shell access, file uploads, model downloads, web research, email/calendar integrations, and API tokens. Treat it like an admin console. + +- Keep `AUTH_ENABLED=true` for any network-accessible deployment. +- Keep `LOCALHOST_BYPASS=false` outside local development. +- Use `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway. +- Do not expose it directly to the public internet without HTTPS and a trusted reverse proxy or private access layer. +- Keep `.env`, `data/`, `logs/`, databases, uploads, generated media, backups, auth/session files, API keys, and model/provider tokens out of Git and private shares. They are ignored by default. +- Review `data/auth.json` after first boot: disable open signup unless you intentionally want it, make only your own account admin, and keep demo/test accounts non-admin. +- Non-admin users do not get shell/Python/file read/write by default, and admin-only routes/tools such as MCP management, API tokens, webhooks, model/cookbook serving, backup/vault, and app settings are admin-gated. Other features are controlled by per-user privileges, so review each user's privileges before exposing a deployment. +- Rotate any API keys or tokens that were ever pasted into a shared chat, demo, screenshot, or log. +- If you enable API tokens or webhooks, create separate tokens per integration and delete unused ones. +- Prefer binding manual development runs to `127.0.0.1`; bind to `0.0.0.0` only when you intentionally want LAN/reverse-proxy access. +- Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only. Expose only the authenticated Odysseus web/API entrypoint through your trusted proxy or private access layer. +- Before publishing a fork, run `git status --short` and confirm no private files from `.env`, `data/`, `logs/`, uploads, backups, or local databases are staged. + +### Private or proxied deployments +Odysseus serves plain HTTP on its app port. Docker Compose binds Odysseus and the bundled services to `127.0.0.1` by default, so a typical production/private setup is: + +1. Keep Odysseus on localhost, for example `127.0.0.1:7000`. +2. Terminate HTTPS at a trusted reverse proxy or private access gateway. +3. Put the authenticated Odysseus web/API entrypoint behind that layer. +4. Keep raw service and model ports internal-only. + +Cloudflare Access, Tailscale, Caddy, nginx, and Traefik can all fit this pattern; none are required by Odysseus. If your access layer reaches Odysseus on the same host, proxy to `http://127.0.0.1:7000` and keep `AUTH_ENABLED=true`, `LOCALHOST_BYPASS=false`, and `SECURE_COOKIES=true`. +`ALLOWED_ORIGINS` lists exact permitted origins for cross-origin browser/API clients; ordinary same-origin reverse-proxy access usually does not need a special CORS entry. + +Common internal-only ports from the default docs/compose setup: + +| Port | Service | +|---|---| +| `7000` | Odysseus raw app port | +| `8080` | SearXNG | +| `8091` | ntfy | +| `8100` | ChromaDB host port for manual/compose access | +| `11434` | Ollama | +| `8000-8020` | Common local model/provider APIs | + +## Configuration +Most setup is done inside the app with `/setup` or **Settings**. Use `.env` +for deployment-level defaults and secrets you want present before first boot. +Key settings: + +| Variable | Default | Description | +|---|---|---| +| `LLM_HOST` | `localhost` | Your LLM server (e.g. `llm-host.local:8000`) | +| `LLM_HOSTS` | -- | Comma-separated list for model discovery | +| `OPENAI_API_KEY` | -- | Optional OpenAI key. Prefer adding providers in the app unless pre-seeding. | +| `SEARXNG_INSTANCE` | `http://localhost:8080` | SearXNG URL. Docker overrides this to `http://searxng:8080`. | +| `SEARXNG_SECRET` | generated on first Docker boot | Optional SearXNG cookie/CSRF secret. Leave blank unless you need to pin it. | +| `APP_BIND` | `127.0.0.1` | Docker Compose host bind address for the web UI. Use `0.0.0.0` only for intentional LAN/reverse-proxy access. | +| `APP_PORT` | `7000` | Docker Compose host port for the web UI. | +| `APP_DATA_DIR` | `./data` | Docker Compose host directory for application data volumes. | +| `APP_LOGS_DIR` | `./logs` | Docker Compose host directory for application logs. | +| `AUTH_ENABLED` | `true` | Enable/disable login | +| `LOCALHOST_BYPASS` | `false` | Development-only auth bypass for loopback requests. Keep false for shared/network deployments. | +| `ALLOWED_ORIGINS` | `http://localhost,http://127.0.0.1` | Comma-separated exact permitted origins for cross-origin browser/API clients. | +| `SECURE_COOKIES` | `false` | Set true when serving Odysseus through HTTPS at a trusted proxy or private access gateway. | +| `DATABASE_URL` | `sqlite:///./data/app.db` | Database connection string | +| `CHROMADB_HOST` | `localhost` | ChromaDB host for vector memory. Docker overrides this to `chromadb`. | +| `CHROMADB_PORT` | `8100` | ChromaDB port for manual host runs. Docker overrides this to `8000`. | +| `EMBEDDING_URL` | -- | OpenAI-compatible embeddings endpoint | +| `ODYSSEUS_CHAT_UPLOAD_MAX_BYTES` | `10485760` | Chat/agent attachment cap in bytes. Raise for larger local PDFs or text documents. | +| `ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES` | `104857600` | Gallery image upload cap in bytes (100 MB). | +| `ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES` | `26214400` | Gallery transform input cap in bytes (25 MB). | +| `ODYSSEUS_MEMORY_IMPORT_MAX_BYTES` | `10485760` | Memory import file cap in bytes (10 MB). | +| `ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES` | `26214400` | Personal document upload cap in bytes (25 MB). | +| `ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES` | `26214400` | Email compose attachment cap in bytes (25 MB). | +| `ODYSSEUS_STT_MAX_AUDIO_BYTES` | `26214400` | Speech-to-text audio cap in bytes (25 MB). | +| `ODYSSEUS_ICS_MAX_BYTES` | `10485760` | Calendar `.ics` import cap in bytes (10 MB). | + +All upload-limit vars are validated (must be a positive integer) and optional; an invalid value fails fast at startup. + +### Built-in MCP servers (optional setup) + +Odysseus auto-registers a few built-in MCP servers at startup. The npx-based ones (currently the browser server, `@playwright/mcp`) only start when their npm package is already in the local npx cache. If a package isn't cached, that server is skipped with a startup log message explaining what to do, so a fresh install does not block on a multi-minute npm download or hang if Playwright system deps are missing. + +To enable the browser MCP (page navigation, screenshots, vision), run once: + +```bash +npx -y @playwright/mcp@latest --version +``` + +That installs `@playwright/mcp` plus Playwright (~300MB total). Restart Odysseus and the server will register at startup. + +## Architecture +``` +app.py # FastAPI entry point +core/ auth, database, middleware, constants +src/ llm_core, agent_loop, agent_tools, chat_processor, search/ +routes/ chat, session, document, memory, model … endpoints +services/ docs, memory, search, hwfit (Cookbook) … +static/ index.html + app.js + style.css + js/ (modular front-end) +docs/ landing page (index.html) + preview clips +``` + +## Data +All user data lives in `data/` (gitignored): `app.db` (sessions, messages, documents), +`memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`. + +To back up or restore everything in `data/`, see the +[Backup & Restore guide](backup-restore.md). diff --git a/integrations/claude/README.md b/integrations/claude/README.md new file mode 100644 index 000000000..e2671f8c3 --- /dev/null +++ b/integrations/claude/README.md @@ -0,0 +1,36 @@ +# Odysseus Claude Code Integration + +This directory contains the Claude Code skill bundle for Odysseus. + +## User Flow + +1. Open Odysseus Settings > Integrations. +2. Add a Claude Agent. +3. Copy the full setup commands shown after the generated token. +4. Toggle the tools Claude is allowed to use. +5. Configure the terminal Claude Code session: + +```bash +export ODYSSEUS_URL=http://your-odysseus-host:7000 +export ODYSSEUS_API_TOKEN=ody_generated_token +mkdir -p ~/.claude +curl -fsSL -H "Authorization: Bearer $ODYSSEUS_API_TOKEN" "$ODYSSEUS_URL/api/claude/plugin.zip" -o /tmp/odysseus-claude-skill.zip +python3 -m zipfile -e /tmp/odysseus-claude-skill.zip ~/.claude/ +``` + +Claude Code auto-loads anything under `~/.claude/skills/`, so the `odysseus` skill is +available in any session that has `ODYSSEUS_URL` and `ODYSSEUS_API_TOKEN` in its +environment. + +## What's in the bundle + +- `skills/odysseus/SKILL.md` — the skill definition Claude Code reads. +- `skills/odysseus/scripts/odysseus_api.py` — small helper that calls the scoped + `/api/codex/*` endpoints (these are the canonical scope-gated agent API; the + `codex` path is historic and shared by all agent integrations). + +## Scope enforcement + +The token is scope-gated. Every tool surface is checked server-side in Odysseus, +so even if Claude tries to call a forbidden endpoint, it gets `403` until the +user enables the matching toggle in Settings > Integrations > Claude Agent. diff --git a/integrations/claude/skills/odysseus/SKILL.md b/integrations/claude/skills/odysseus/SKILL.md new file mode 100644 index 000000000..31b40ee01 --- /dev/null +++ b/integrations/claude/skills/odysseus/SKILL.md @@ -0,0 +1,154 @@ +--- +name: odysseus +description: Use when the user asks Claude Code to read or write Odysseus data (todos, email, calendar, memory, documents) or to launch/monitor/stop a Cookbook model-serve task through the scoped Claude Agent API. Requires ODYSSEUS_URL and ODYSSEUS_API_TOKEN. +--- + +# Odysseus + +Use this skill when a user asks to interact with Odysseus from Claude Code. + +## Configuration + +Expect these environment variables: + +- `ODYSSEUS_URL`: Base URL for the user's Odysseus instance, for example `http://127.0.0.1:7000`. +- `ODYSSEUS_API_TOKEN`: Scoped API token created in Odysseus Settings > Integrations > Add Integration > Claude Agent. + +If either value is missing, do not guess credentials. Tell the user to create a Claude Agent token in Odysseus Settings and expose both values to the terminal session. + +## When to use what + +- **Reminder ("remind me at 5pm to do X")** → TODO with `due_date`. The due_date IS the reminder — it fires a notification automatically via the user's configured channel (browser/email/ntfy). **Do NOT create a calendar event for a reminder.** Creating a calendar event named "Reminder" does NOT trigger a notification — it's just a time block on the calendar. +- **Calendar event ("meeting at 3pm", "dentist Tuesday 10am")** → calendar event. Use for scheduled time blocks, meetings, appointments, recurring schedules. These show up on the calendar grid; reminders for them are configured separately in Odysseus settings. +- **Note / freeform info ("note that the wifi password is ...")** → memory or todo without a due_date (depending on whether it's a fact about the user or an action item). +- **Persistent fact / preference about the user** → memory. + +If the user says "reminder" + a time, default to TODO with due_date. Only switch to calendar if the user explicitly says "calendar", "event", "meeting", "appointment", or describes a time *range*. + +## Safety + +- All Odysseus data access MUST go through the scoped HTTP API under `/api/codex/*` (the canonical scope-gated agent API, shared by all agent integrations). +- Check `/api/codex/capabilities` before using a tool surface. +- Treat `403` as an intentional Settings restriction. Do not work around it. +- Do not use SSH, Docker, direct Python imports, SQLite queries, MCP internals, browser cookies, or local files to read/write Odysseus user data. +- Do not call helpers like `do_manage_notes`, email MCP internals, or database sessions directly for user data, even if shell access exists. +- Never send email directly unless the user explicitly asks to send and the token has a send-capable scope. +- Keep actions scoped to the token owner. + +## Todos + +The scoped agent API supports todos/checklists: + +- `GET /api/codex/todos` +- `POST /api/codex/todos` + +Use the bundled helper script when available: + +```bash +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py capabilities +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py todos list +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py todos add "Follow up" +``` + +Supported todo actions are `list`, `add`, `update`, `delete`, and `toggle_item`. + +**Reminders (todos with a due date)** — the backend parses natural language. Send `due_date` in the body via the generic POST so the time becomes a structured reminder, NOT a literal substring inside the title. The `todos add TITLE` shortcut only sets the title, so use the POST form for anything with a time: + +```bash +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py POST /api/codex/todos '{"action":"add","title":"Call dentist","due_date":"tomorrow at 5pm"}' +``` + +The backend accepts both ISO timestamps and natural language like `"tomorrow 5pm"`, `"next Monday 9am"`, `"in 2 hours"`. It anchors to the user's timezone. + +## Email + +The scoped agent API supports email reads: + +- `GET /api/codex/emails?folder=INBOX&limit=10&offset=0&filter=all` +- `GET /api/codex/emails/{uid}?folder=INBOX` + +Use the bundled helper script when available: + +```bash +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py emails list 5 +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py emails read UID +``` + +If `/api/codex/capabilities` does not show `email.read: true`, do not inspect email. Ask the user to enable Email read in the Claude Agent settings. + +## Memory + +- `GET /api/codex/memory` — list memories for the token owner. +- `POST /api/codex/memory` — body `{"text": "...", "category": "fact", "source": "user", "session_id": null}`. Requires `memory:write`. +- `DELETE /api/codex/memory/{memory_id}` — remove a memory entry. Requires `memory:write`. + +```bash +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py GET /api/codex/memory +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py POST /api/codex/memory '{"text":"User prefers SI units","category":"preference"}' +``` + +## Calendar + +- `GET /api/codex/calendar/events?start=ISO&end=ISO` — list events in window. +- `POST /api/codex/calendar/events` — body matches `EventCreate` (`summary`, `dtstart`, `dtend`, `all_day`, `description`, `location`, `calendar_href`, `rrule`, `color`). Requires `calendar:write`. +- `DELETE /api/codex/calendar/events/{uid}` — delete event by uid (the value returned in the POST response). Requires `calendar:write`. + +## Documents + +- `GET /api/codex/documents?search=...&limit=50` — paginated library. +- `GET /api/codex/documents/{doc_id}` — fetch one document. +- `POST /api/codex/documents` — body `{"session_id": "...", "title": "...", "content": "...", "language": "markdown"}`. Requires `documents:write`. +- `DELETE /api/codex/documents/{doc_id}` — delete a document. Requires `documents:write`. + +## Email draft + send + +- Prefer `POST /api/codex/emails/draft-document` for agent-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send. +- `POST /api/codex/emails/draft` — body matches `SendEmailRequest` (`to`, `cc`, `bcc`, `subject`, `body`, `body_html`, `attachments`, `account_id`, `in_reply_to`, `references`). Requires `email:draft` (or `email:send`). +- `POST /api/codex/emails/send` — same body. Requires `email:send`. Never send without explicit user instruction. + +## Cookbook serve (debug a failing model launch) + +The Cookbook surface lets you reproduce what a human would do in Odysseus → Cookbook: read which serves are running, tail their tmux output to see why they crashed, edit the launch command, relaunch, kill a stuck one. Use this when the user is debugging a model server that won't come up (compute-capability errors, OOM, missing kernels, wrong attention backend, etc.). + +- `GET /api/codex/cookbook/tasks` — list active serve/download/install tasks (sessionId, type, status, repo_id, remoteHost, payload._cmd). Requires `cookbook:read`. +- `GET /api/codex/cookbook/servers` — list configured servers (name, host, port, env type + path, model dirs). Requires `cookbook:read`. +- `GET /api/codex/cookbook/cached?host=` — list models already cached on the named server (HF cache + Ollama + extra modelDirs). Call BEFORE `serve` to see what's already on disk. Requires `cookbook:read`. +- `GET /api/codex/cookbook/presets` — list saved serve presets (model + host + port + cmd). The user's saved preset usually has a working cmd — try `preset NAME` before composing your own. Requires `cookbook:read`. +- `GET /api/codex/cookbook/output/{session_id}?tail=400` — read the last N lines of the task's persistent log file (preferred) or tmux pane (fallback). The log file persists across vllm crashes, so this returns the actual Python traceback even after the bash prompt + neofetch banner overwrites the pane. Default tail=400. Requires `cookbook:read`. +- `POST /api/codex/cookbook/serve` — launch a serve task. Body matches `ServeRequest`: `{ repo_id, cmd, remote_host?, ssh_port?, env_prefix?, gpus?, platform? }`. The `cmd` is validated: leading binary must be `vllm`/`python3`/`sglang`/`llama-server`/`ollama`/`node`/`npx`. NEVER prefix with `cd …`, `source …`, or chain with `&&`/`||`/`;`/`$(...)` — the validator rejects shell metacharacters. The venv activation (`env_prefix`) is added automatically from the host's saved settings, so pass the bare binary + args. Requires `cookbook:launch`. +- `POST /api/codex/cookbook/preset/{name}` — launch a saved preset by name. Reuses the working cmd + host the user already saved. Requires `cookbook:launch`. +- `POST /api/codex/cookbook/adopt` — register an externally-launched tmux session into cookbook tracking. Body: `{ tmux_session, model, host?, port? }`. Use this when serve_model rejected a cmd and you fell back to direct ssh+tmux — without adoption, the session is invisible to the UI. Requires `cookbook:launch`. +- `POST /api/codex/cookbook/stop/{session_id}` — kill the tmux session for that task. Requires `cookbook:launch`. + +```bash +# Survey what's running +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py cookbook tasks + +# Tail the failing one (sessionId from `cookbook tasks`) +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py cookbook output serve-abc12345 400 + +# Stop the previous attempt before you try a new flag set +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py cookbook stop serve-abc12345 + +# Relaunch with new flags. cmd MUST begin with one of the allowlisted binaries. +python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py cookbook serve \ + /mnt/HADES/models/Qwen3.5-397B-A17B-AWQ \ + "vllm serve /mnt/HADES/models/Qwen3.5-397B-A17B-AWQ --host 0.0.0.0 --port 8001 --tensor-parallel-size 8 --max-model-len 262144 --gpu-memory-utilization 0.90 --dtype auto --max-num-seqs 8 --trust-remote-code --enable-expert-parallel --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser qwen3" \ + pewds@192.168.1.12 +``` + +**Debug loop pattern:** when a serve is failing, the productive sequence is + +1. `cookbook tasks` → find the failing sessionId. +2. `cookbook output SID 600` → read the last 600 lines, find the actual root-cause line (often above the visible tail because tmux scrollback rolled — request a larger `tail` if the error references "above"). +3. `cookbook stop SID` — kill the previous attempt before relaunching; two serves on the same `--port` collide. +4. `cookbook serve repo "new cmd"` — try the next variation. Wait ~20s, then `cookbook output` on the new sessionId. + +**Hard limits this surface enforces:** +- `cookbook serve` cmd allowlist + shell-metacharacter rejection — you cannot run arbitrary shell, only model-server binaries. +- `cookbook stop` only targets task sessionIds matching `[a-zA-Z0-9_-]+`. +- The agent CAN spawn GPU-pinning long-lived processes — always `cookbook stop` your previous attempt before relaunching, and check `cookbook tasks` for collisions on the same `--port` before launching. + +## Forbidden Bypass Pattern + +If you are about to reach the Odysseus host/container, import app internals, query the database, or call MCP helper modules directly, stop. Those paths bypass Odysseus Settings and token scopes. Ask the user to enable the relevant Claude Agent tool toggle instead. diff --git a/integrations/claude/skills/odysseus/scripts/odysseus_api.py b/integrations/claude/skills/odysseus/scripts/odysseus_api.py new file mode 100755 index 000000000..8a22eb494 --- /dev/null +++ b/integrations/claude/skills/odysseus/scripts/odysseus_api.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Small Odysseus scoped API helper for Codex terminal sessions.""" + +from __future__ import annotations + +import json +import os +import sys +import urllib.error +import urllib.request + + +def _usage() -> int: + print("usage:", file=sys.stderr) + print(" odysseus_api.py capabilities", file=sys.stderr) + print(" odysseus_api.py todos list", file=sys.stderr) + print(" odysseus_api.py todos add TITLE", file=sys.stderr) + print(" odysseus_api.py emails list [limit]", file=sys.stderr) + print(" odysseus_api.py emails read UID", file=sys.stderr) + print(" odysseus_api.py emails draft-doc JSON_PAYLOAD", file=sys.stderr) + print(" odysseus_api.py documents list [limit]", file=sys.stderr) + print(" odysseus_api.py documents read DOC_ID", file=sys.stderr) + print(" odysseus_api.py documents create JSON_PAYLOAD", file=sys.stderr) + print(" odysseus_api.py documents delete DOC_ID", file=sys.stderr) + print(" odysseus_api.py cookbook tasks", file=sys.stderr) + print(" odysseus_api.py cookbook servers", file=sys.stderr) + print(" odysseus_api.py cookbook cached [HOST]", file=sys.stderr) + print(" odysseus_api.py cookbook presets", file=sys.stderr) + print(" odysseus_api.py cookbook output SESSION_ID [tail]", file=sys.stderr) + print(" odysseus_api.py cookbook serve REPO_ID 'CMD' [REMOTE_HOST]", file=sys.stderr) + print(" odysseus_api.py cookbook preset NAME", file=sys.stderr) + print(" odysseus_api.py cookbook adopt SESSION_ID MODEL [HOST] [PORT]", file=sys.stderr) + print(" odysseus_api.py cookbook stop SESSION_ID", file=sys.stderr) + print(" odysseus_api.py METHOD /api/codex/path [json-body]", file=sys.stderr) + return 2 + + +def _config() -> tuple[str, str] | None: + base_url = os.environ.get("ODYSSEUS_URL", "").strip().rstrip("/") + token = os.environ.get("ODYSSEUS_API_TOKEN", "").strip() + missing = [] + if not base_url: + missing.append("ODYSSEUS_URL") + if not token: + missing.append("ODYSSEUS_API_TOKEN") + if missing: + print(f"missing {', '.join(missing)}; create a Codex Agent token in Odysseus Settings", file=sys.stderr) + return None + return base_url, token + + +def main() -> int: + if len(sys.argv) < 2: + return _usage() + + command = sys.argv[1].lower() + if command == "capabilities": + method = "GET" + path = "/api/codex/capabilities" + body = None + elif command == "todos": + if len(sys.argv) < 3: + return _usage() + action = sys.argv[2].lower() + path = "/api/codex/todos" + if action == "list": + method = "GET" + body = None + elif action == "add" and len(sys.argv) >= 4: + method = "POST" + body = json.dumps({"action": "add", "title": " ".join(sys.argv[3:])}) + else: + return _usage() + elif command == "emails": + if len(sys.argv) < 3: + return _usage() + action = sys.argv[2].lower() + if action == "list": + method = "GET" + limit = sys.argv[3] if len(sys.argv) >= 4 else "10" + path = f"/api/codex/emails?folder=INBOX&limit={limit}&offset=0&filter=all" + body = None + elif action == "read" and len(sys.argv) >= 4: + method = "GET" + path = f"/api/codex/emails/{sys.argv[3]}" + body = None + elif action in ("draft-doc", "draft_document") and len(sys.argv) >= 4: + method = "POST" + path = "/api/codex/emails/draft-document" + body = " ".join(sys.argv[3:]) + else: + return _usage() + elif command in ("documents", "docs"): + if len(sys.argv) < 3: + return _usage() + action = sys.argv[2].lower() + if action == "list": + method = "GET" + limit = sys.argv[3] if len(sys.argv) >= 4 else "50" + path = f"/api/codex/documents?limit={limit}" + body = None + elif action == "read" and len(sys.argv) >= 4: + method = "GET" + path = f"/api/codex/documents/{sys.argv[3]}" + body = None + elif action == "create" and len(sys.argv) >= 4: + method = "POST" + path = "/api/codex/documents" + body = " ".join(sys.argv[3:]) + elif action == "delete" and len(sys.argv) >= 4: + method = "DELETE" + path = f"/api/codex/documents/{sys.argv[3]}" + body = None + else: + return _usage() + elif command == "cookbook": + if len(sys.argv) < 3: + return _usage() + action = sys.argv[2].lower() + if action == "tasks": + method = "GET" + path = "/api/codex/cookbook/tasks" + body = None + elif action == "servers": + method = "GET" + path = "/api/codex/cookbook/servers" + body = None + elif action == "output" and len(sys.argv) >= 4: + method = "GET" + sid = sys.argv[3] + tail = sys.argv[4] if len(sys.argv) >= 5 else "400" + path = f"/api/codex/cookbook/output/{sid}?tail={tail}" + body = None + elif action == "cached": + method = "GET" + if len(sys.argv) >= 4: + from urllib.parse import quote + path = f"/api/codex/cookbook/cached?host={quote(sys.argv[3])}" + else: + path = "/api/codex/cookbook/cached" + body = None + elif action == "presets": + method = "GET" + path = "/api/codex/cookbook/presets" + body = None + elif action == "preset" and len(sys.argv) >= 4: + from urllib.parse import quote + method = "POST" + path = f"/api/codex/cookbook/preset/{quote(sys.argv[3])}" + body = None + elif action == "adopt" and len(sys.argv) >= 5: + method = "POST" + path = "/api/codex/cookbook/adopt" + payload = {"tmux_session": sys.argv[3], "model": sys.argv[4]} + if len(sys.argv) >= 6: payload["host"] = sys.argv[5] + if len(sys.argv) >= 7: payload["port"] = int(sys.argv[6]) + body = json.dumps(payload) + elif action == "serve" and len(sys.argv) >= 5: + method = "POST" + path = "/api/codex/cookbook/serve" + payload = {"repo_id": sys.argv[3], "cmd": sys.argv[4]} + if len(sys.argv) >= 6: + payload["remote_host"] = sys.argv[5] + body = json.dumps(payload) + elif action == "stop" and len(sys.argv) >= 4: + method = "POST" + path = f"/api/codex/cookbook/stop/{sys.argv[3]}" + body = None + else: + return _usage() + else: + if len(sys.argv) < 3: + return _usage() + method = sys.argv[1].upper() + path = sys.argv[2] + body = sys.argv[3] if len(sys.argv) > 3 else None + + if not path.startswith("/"): + path = "/" + path + if not path.startswith("/api/codex/"): + print("refusing non-/api/codex path; use scoped Odysseus integration endpoints only", file=sys.stderr) + return 2 + + config = _config() + if config is None: + return 2 + base_url, token = config + + data = None + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {token}", + } + if body is not None: + try: + parsed = json.loads(body) + except json.JSONDecodeError as exc: + print(f"invalid json body: {exc}", file=sys.stderr) + return 2 + data = json.dumps(parsed).encode("utf-8") + headers["Content-Type"] = "application/json" + + req = urllib.request.Request(base_url + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=20) as resp: + print(resp.read().decode("utf-8")) + return 0 + except urllib.error.HTTPError as exc: + text = exc.read().decode("utf-8", errors="replace") + print(text or f"HTTP {exc.code}", file=sys.stderr) + return 1 + except OSError as exc: + print(f"request failed: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/integrations/codex/.codex-plugin/plugin.json b/integrations/codex/.codex-plugin/plugin.json new file mode 100644 index 000000000..239451f7b --- /dev/null +++ b/integrations/codex/.codex-plugin/plugin.json @@ -0,0 +1,22 @@ +{ + "name": "odysseus", + "version": "0.1.1", + "description": "Connect Codex to a scoped Odysseus instance.", + "author": { + "name": "Odysseus" + }, + "skills": "./skills/", + "interface": { + "displayName": "Odysseus", + "shortDescription": "Use scoped Odysseus tools from Codex.", + "longDescription": "Connects Codex terminal sessions to Odysseus through user-controlled scoped API tokens. Codex must use /api/codex/* endpoints so Odysseus Settings can enforce tool access.", + "developerName": "Odysseus", + "category": "Productivity", + "capabilities": [ + "todos", + "email", + "scoped-api" + ], + "defaultPrompt": "Use Odysseus only through configured scoped access. Check capabilities before reading or writing data." + } +} diff --git a/integrations/codex/README.md b/integrations/codex/README.md new file mode 100644 index 000000000..fff4e84e5 --- /dev/null +++ b/integrations/codex/README.md @@ -0,0 +1,51 @@ +# Odysseus Codex Integration + +This directory contains the Codex plugin/skill bundle for Odysseus. + +## User Flow + +1. Open Odysseus Settings > Integrations. +2. Add a Codex Agent. +3. Copy the full setup commands shown after the generated token. +4. Toggle the tools Codex is allowed to use. +5. Configure the terminal Codex session: + +```bash +export ODYSSEUS_URL=http://your-odysseus-host:7000 +export ODYSSEUS_API_TOKEN=ody_generated_token +mkdir -p ~/plugins +curl -fsSL -H "Authorization: Bearer $ODYSSEUS_API_TOKEN" "$ODYSSEUS_URL/api/codex/plugin.zip" -o /tmp/odysseus-codex-plugin.zip +python3 -m zipfile -e /tmp/odysseus-codex-plugin.zip ~/plugins +python3 - <<'PY' +import json +from pathlib import Path + +p = Path.home() / ".agents" / "plugins" / "marketplace.json" +p.parent.mkdir(parents=True, exist_ok=True) +if p.exists(): + data = json.loads(p.read_text()) +else: + data = {"name": "personal", "interface": {"displayName": "Personal"}, "plugins": []} + +data.setdefault("name", "personal") +data.setdefault("interface", {}).setdefault("displayName", "Personal") +plugins = data.setdefault("plugins", []) +entry = { + "name": "odysseus", + "source": {"source": "local", "path": "./plugins/odysseus"}, + "policy": {"installation": "AVAILABLE", "authentication": "ON_INSTALL"}, + "category": "Productivity", +} +data["plugins"] = [item for item in plugins if item.get("name") != "odysseus"] + [entry] +p.write_text(json.dumps(data, indent=2) + "\n") +PY +codex plugin add odysseus@personal +``` + +6. Verify: + +```bash +python3 ~/plugins/odysseus/scripts/odysseus_api.py capabilities +``` + +Codex must use `/api/codex/*` endpoints. SSH, Docker, direct Python imports, database queries, and MCP internals bypass Odysseus Settings and must not be used for user data access. diff --git a/integrations/codex/scripts/odysseus_api.py b/integrations/codex/scripts/odysseus_api.py new file mode 100755 index 000000000..8a22eb494 --- /dev/null +++ b/integrations/codex/scripts/odysseus_api.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Small Odysseus scoped API helper for Codex terminal sessions.""" + +from __future__ import annotations + +import json +import os +import sys +import urllib.error +import urllib.request + + +def _usage() -> int: + print("usage:", file=sys.stderr) + print(" odysseus_api.py capabilities", file=sys.stderr) + print(" odysseus_api.py todos list", file=sys.stderr) + print(" odysseus_api.py todos add TITLE", file=sys.stderr) + print(" odysseus_api.py emails list [limit]", file=sys.stderr) + print(" odysseus_api.py emails read UID", file=sys.stderr) + print(" odysseus_api.py emails draft-doc JSON_PAYLOAD", file=sys.stderr) + print(" odysseus_api.py documents list [limit]", file=sys.stderr) + print(" odysseus_api.py documents read DOC_ID", file=sys.stderr) + print(" odysseus_api.py documents create JSON_PAYLOAD", file=sys.stderr) + print(" odysseus_api.py documents delete DOC_ID", file=sys.stderr) + print(" odysseus_api.py cookbook tasks", file=sys.stderr) + print(" odysseus_api.py cookbook servers", file=sys.stderr) + print(" odysseus_api.py cookbook cached [HOST]", file=sys.stderr) + print(" odysseus_api.py cookbook presets", file=sys.stderr) + print(" odysseus_api.py cookbook output SESSION_ID [tail]", file=sys.stderr) + print(" odysseus_api.py cookbook serve REPO_ID 'CMD' [REMOTE_HOST]", file=sys.stderr) + print(" odysseus_api.py cookbook preset NAME", file=sys.stderr) + print(" odysseus_api.py cookbook adopt SESSION_ID MODEL [HOST] [PORT]", file=sys.stderr) + print(" odysseus_api.py cookbook stop SESSION_ID", file=sys.stderr) + print(" odysseus_api.py METHOD /api/codex/path [json-body]", file=sys.stderr) + return 2 + + +def _config() -> tuple[str, str] | None: + base_url = os.environ.get("ODYSSEUS_URL", "").strip().rstrip("/") + token = os.environ.get("ODYSSEUS_API_TOKEN", "").strip() + missing = [] + if not base_url: + missing.append("ODYSSEUS_URL") + if not token: + missing.append("ODYSSEUS_API_TOKEN") + if missing: + print(f"missing {', '.join(missing)}; create a Codex Agent token in Odysseus Settings", file=sys.stderr) + return None + return base_url, token + + +def main() -> int: + if len(sys.argv) < 2: + return _usage() + + command = sys.argv[1].lower() + if command == "capabilities": + method = "GET" + path = "/api/codex/capabilities" + body = None + elif command == "todos": + if len(sys.argv) < 3: + return _usage() + action = sys.argv[2].lower() + path = "/api/codex/todos" + if action == "list": + method = "GET" + body = None + elif action == "add" and len(sys.argv) >= 4: + method = "POST" + body = json.dumps({"action": "add", "title": " ".join(sys.argv[3:])}) + else: + return _usage() + elif command == "emails": + if len(sys.argv) < 3: + return _usage() + action = sys.argv[2].lower() + if action == "list": + method = "GET" + limit = sys.argv[3] if len(sys.argv) >= 4 else "10" + path = f"/api/codex/emails?folder=INBOX&limit={limit}&offset=0&filter=all" + body = None + elif action == "read" and len(sys.argv) >= 4: + method = "GET" + path = f"/api/codex/emails/{sys.argv[3]}" + body = None + elif action in ("draft-doc", "draft_document") and len(sys.argv) >= 4: + method = "POST" + path = "/api/codex/emails/draft-document" + body = " ".join(sys.argv[3:]) + else: + return _usage() + elif command in ("documents", "docs"): + if len(sys.argv) < 3: + return _usage() + action = sys.argv[2].lower() + if action == "list": + method = "GET" + limit = sys.argv[3] if len(sys.argv) >= 4 else "50" + path = f"/api/codex/documents?limit={limit}" + body = None + elif action == "read" and len(sys.argv) >= 4: + method = "GET" + path = f"/api/codex/documents/{sys.argv[3]}" + body = None + elif action == "create" and len(sys.argv) >= 4: + method = "POST" + path = "/api/codex/documents" + body = " ".join(sys.argv[3:]) + elif action == "delete" and len(sys.argv) >= 4: + method = "DELETE" + path = f"/api/codex/documents/{sys.argv[3]}" + body = None + else: + return _usage() + elif command == "cookbook": + if len(sys.argv) < 3: + return _usage() + action = sys.argv[2].lower() + if action == "tasks": + method = "GET" + path = "/api/codex/cookbook/tasks" + body = None + elif action == "servers": + method = "GET" + path = "/api/codex/cookbook/servers" + body = None + elif action == "output" and len(sys.argv) >= 4: + method = "GET" + sid = sys.argv[3] + tail = sys.argv[4] if len(sys.argv) >= 5 else "400" + path = f"/api/codex/cookbook/output/{sid}?tail={tail}" + body = None + elif action == "cached": + method = "GET" + if len(sys.argv) >= 4: + from urllib.parse import quote + path = f"/api/codex/cookbook/cached?host={quote(sys.argv[3])}" + else: + path = "/api/codex/cookbook/cached" + body = None + elif action == "presets": + method = "GET" + path = "/api/codex/cookbook/presets" + body = None + elif action == "preset" and len(sys.argv) >= 4: + from urllib.parse import quote + method = "POST" + path = f"/api/codex/cookbook/preset/{quote(sys.argv[3])}" + body = None + elif action == "adopt" and len(sys.argv) >= 5: + method = "POST" + path = "/api/codex/cookbook/adopt" + payload = {"tmux_session": sys.argv[3], "model": sys.argv[4]} + if len(sys.argv) >= 6: payload["host"] = sys.argv[5] + if len(sys.argv) >= 7: payload["port"] = int(sys.argv[6]) + body = json.dumps(payload) + elif action == "serve" and len(sys.argv) >= 5: + method = "POST" + path = "/api/codex/cookbook/serve" + payload = {"repo_id": sys.argv[3], "cmd": sys.argv[4]} + if len(sys.argv) >= 6: + payload["remote_host"] = sys.argv[5] + body = json.dumps(payload) + elif action == "stop" and len(sys.argv) >= 4: + method = "POST" + path = f"/api/codex/cookbook/stop/{sys.argv[3]}" + body = None + else: + return _usage() + else: + if len(sys.argv) < 3: + return _usage() + method = sys.argv[1].upper() + path = sys.argv[2] + body = sys.argv[3] if len(sys.argv) > 3 else None + + if not path.startswith("/"): + path = "/" + path + if not path.startswith("/api/codex/"): + print("refusing non-/api/codex path; use scoped Odysseus integration endpoints only", file=sys.stderr) + return 2 + + config = _config() + if config is None: + return 2 + base_url, token = config + + data = None + headers = { + "Accept": "application/json", + "Authorization": f"Bearer {token}", + } + if body is not None: + try: + parsed = json.loads(body) + except json.JSONDecodeError as exc: + print(f"invalid json body: {exc}", file=sys.stderr) + return 2 + data = json.dumps(parsed).encode("utf-8") + headers["Content-Type"] = "application/json" + + req = urllib.request.Request(base_url + path, data=data, headers=headers, method=method) + try: + with urllib.request.urlopen(req, timeout=20) as resp: + print(resp.read().decode("utf-8")) + return 0 + except urllib.error.HTTPError as exc: + text = exc.read().decode("utf-8", errors="replace") + print(text or f"HTTP {exc.code}", file=sys.stderr) + return 1 + except OSError as exc: + print(f"request failed: {exc}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/integrations/codex/skills/odysseus/SKILL.md b/integrations/codex/skills/odysseus/SKILL.md new file mode 100644 index 000000000..d4cbdf726 --- /dev/null +++ b/integrations/codex/skills/odysseus/SKILL.md @@ -0,0 +1,142 @@ +--- +name: odysseus +description: Use when the user asks Codex to read or write Odysseus data (todos, email, calendar, memory, documents) or to launch/monitor/stop a Cookbook model-serve task through the scoped Codex Agent API. Requires ODYSSEUS_URL and ODYSSEUS_API_TOKEN. +--- + +# Odysseus + +Use this skill when a user asks to interact with Odysseus from Codex. + +## Configuration + +Expect these environment variables: + +- `ODYSSEUS_URL`: Base URL for the user's Odysseus instance, for example `http://127.0.0.1:7000`. +- `ODYSSEUS_API_TOKEN`: Scoped API token created in Odysseus Settings > Integrations > Add Integration > Codex Agent. + +If either value is missing, do not guess credentials. Tell the user to create a Codex Agent token in Odysseus Settings and expose both values to the terminal session. + +## When to use what + +- **Reminder ("remind me at 5pm to do X")** → TODO with `due_date`. The due_date IS the reminder — it fires a notification automatically via the user's configured channel (browser/email/ntfy). **Do NOT create a calendar event for a reminder.** Creating a calendar event named "Reminder" does NOT trigger a notification — it's just a time block on the calendar. +- **Calendar event ("meeting at 3pm", "dentist Tuesday 10am")** → calendar event. Use for scheduled time blocks, meetings, appointments, recurring schedules. These show up on the calendar grid; reminders for them are configured separately in Odysseus settings. +- **Note / freeform info ("note that the wifi password is ...")** → memory or todo without a due_date (depending on whether it's a fact about the user or an action item). +- **Persistent fact / preference about the user** → memory. + +If the user says "reminder" + a time, default to TODO with due_date. Only switch to calendar if the user explicitly says "calendar", "event", "meeting", "appointment", or describes a time *range*. + +## Safety + +- All Odysseus data access MUST go through the scoped HTTP API under `/api/codex/*`. +- Check `/api/codex/capabilities` before using a tool surface. +- Treat `403` as an intentional Settings restriction. Do not work around it. +- Do not use SSH, Docker, direct Python imports, SQLite queries, MCP internals, browser cookies, or local files to read/write Odysseus user data. +- Do not call helpers like `do_manage_notes`, email MCP internals, or database sessions directly for user data, even if shell access exists. +- Never send email directly unless the user explicitly asks to send and the token has a send-capable scope. +- Keep actions scoped to the token owner. + +## Todos + +The Codex API supports todos/checklists: + +- `GET /api/codex/todos` +- `POST /api/codex/todos` + +Use the bundled helper script when available: + +```bash +python3 integrations/codex/scripts/odysseus_api.py capabilities +python3 integrations/codex/scripts/odysseus_api.py todos list +python3 integrations/codex/scripts/odysseus_api.py todos add "Follow up" +``` + +Supported todo actions are `list`, `add`, `update`, `delete`, and `toggle_item`. + +**Reminders (todos with a due date)** — the backend parses natural language. Send `due_date` in the body via the generic POST so the time becomes a structured reminder, NOT a literal substring inside the title. The `todos add TITLE` shortcut only sets the title, so use the POST form for anything with a time: + +```bash +python3 integrations/codex/scripts/odysseus_api.py POST /api/codex/todos '{"action":"add","title":"Call dentist","due_date":"tomorrow at 5pm"}' +``` + +The backend accepts both ISO timestamps and natural language like `"tomorrow 5pm"`, `"next Monday 9am"`, `"in 2 hours"`. It anchors to the user's timezone. + +## Email + +The Codex API supports scoped email reads: + +- `GET /api/codex/emails?folder=INBOX&limit=10&offset=0&filter=all` +- `GET /api/codex/emails/{uid}?folder=INBOX` + +Use the bundled helper script when available: + +```bash +python3 integrations/codex/scripts/odysseus_api.py emails list 5 +python3 integrations/codex/scripts/odysseus_api.py emails read UID +``` + +If `/api/codex/capabilities` does not show `email.read: true`, do not inspect email. Ask the user to enable Email read in the Codex Agent settings. + +## Memory + +- `GET /api/codex/memory` — list memories for the token owner. +- `POST /api/codex/memory` — body `{"text": "...", "category": "fact", "source": "user", "session_id": null}`. Requires `memory:write`. +- `DELETE /api/codex/memory/{memory_id}` — remove a memory entry. Requires `memory:write`. + +```bash +python3 integrations/codex/scripts/odysseus_api.py GET /api/codex/memory +python3 integrations/codex/scripts/odysseus_api.py POST /api/codex/memory '{"text":"User prefers SI units","category":"preference"}' +``` + +## Calendar + +- `GET /api/codex/calendar/events?start=ISO&end=ISO` — list events in window. +- `POST /api/codex/calendar/events` — body matches `EventCreate` (`summary`, `dtstart`, `dtend`, `all_day`, `description`, `location`, `calendar_href`, `rrule`, `color`). Requires `calendar:write`. +- `DELETE /api/codex/calendar/events/{uid}` — delete event by uid (the value returned in the POST response). Requires `calendar:write`. + +## Documents + +- `GET /api/codex/documents?search=...&limit=50` — paginated library. +- `GET /api/codex/documents/{doc_id}` — fetch one document. +- `POST /api/codex/documents` — body `{"session_id": "...", "title": "...", "content": "...", "language": "markdown"}`. Requires `documents:write`. +- `DELETE /api/codex/documents/{doc_id}` — delete a document. Requires `documents:write`. + +## Email draft + send + +- Prefer `POST /api/codex/emails/draft-document` for Codex-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send. +- `POST /api/codex/emails/draft` — body matches `SendEmailRequest` (`to`, `cc`, `bcc`, `subject`, `body`, `body_html`, `attachments`, `account_id`, `in_reply_to`, `references`). Requires `email:draft` (or `email:send`). +- `POST /api/codex/emails/send` — same body. Requires `email:send`. Never send without explicit user instruction. + +## Cookbook serve (debug a failing model launch) + +The Cookbook surface lets you reproduce what a human would do in Odysseus → Cookbook: read which serves are running, tail their tmux output to see why they crashed, edit the launch command, relaunch, kill a stuck one. Use this when the user is debugging a model server that won't come up (compute-capability errors, OOM, missing kernels, wrong attention backend, etc.). + +- `GET /api/codex/cookbook/tasks` — list active serve/download/install tasks (sessionId, type, status, repo_id, remoteHost, payload._cmd). Requires `cookbook:read`. +- `GET /api/codex/cookbook/servers` — list configured servers (name, host, port, env type + path, model dirs). Requires `cookbook:read`. +- `GET /api/codex/cookbook/cached?host=` — list models already cached on the named server (HF cache + Ollama + extra modelDirs). Call BEFORE `serve` to see what's already on disk. Requires `cookbook:read`. +- `GET /api/codex/cookbook/presets` — list saved serve presets (model + host + port + cmd). The user's saved preset usually has a working cmd — try `preset NAME` before composing your own. Requires `cookbook:read`. +- `GET /api/codex/cookbook/output/{session_id}?tail=400` — read the last N lines of the task's persistent log file (preferred) or tmux pane (fallback). The log file persists across vllm crashes, so this returns the actual Python traceback even after the bash prompt + neofetch banner overwrites the pane. Default tail=400. Requires `cookbook:read`. +- `POST /api/codex/cookbook/serve` — launch a serve task. Body matches `ServeRequest`: `{ repo_id, cmd, remote_host?, ssh_port?, env_prefix?, gpus?, platform? }`. The `cmd` is validated: leading binary must be `vllm`/`python3`/`sglang`/`llama-server`/`ollama`/`node`/`npx`. NEVER prefix with `cd …`, `source …`, or chain with `&&`/`||`/`;`/`$(...)` — the validator rejects shell metacharacters. The venv activation (`env_prefix`) is added automatically from the host's saved settings, so pass the bare binary + args. Requires `cookbook:launch`. +- `POST /api/codex/cookbook/preset/{name}` — launch a saved preset by name. Reuses the working cmd + host the user already saved. Requires `cookbook:launch`. +- `POST /api/codex/cookbook/adopt` — register an externally-launched tmux session into cookbook tracking. Body: `{ tmux_session, model, host?, port? }`. Use this when serve_model rejected a cmd and you fell back to direct ssh+tmux — without adoption, the session is invisible to the UI. Requires `cookbook:launch`. +- `POST /api/codex/cookbook/stop/{session_id}` — kill the tmux session. Requires `cookbook:launch`. + +```bash +python3 ~/plugins/odysseus/scripts/odysseus_api.py cookbook tasks +python3 ~/plugins/odysseus/scripts/odysseus_api.py cookbook output serve-abc12345 400 +python3 ~/plugins/odysseus/scripts/odysseus_api.py cookbook stop serve-abc12345 +python3 ~/plugins/odysseus/scripts/odysseus_api.py cookbook serve \ + /mnt/HADES/models/Qwen3.5-397B-A17B-AWQ \ + "vllm serve /mnt/HADES/models/Qwen3.5-397B-A17B-AWQ --host 0.0.0.0 --port 8001 --tensor-parallel-size 8 --max-model-len 262144 --gpu-memory-utilization 0.90 --dtype auto --max-num-seqs 8 --trust-remote-code --enable-expert-parallel --enable-auto-tool-choice --tool-call-parser qwen3_coder --reasoning-parser qwen3" \ + pewds@192.168.1.12 +``` + +**Debug loop pattern:** `tasks` → `output SID 600` (find root cause; request larger `tail` if it references "above") → `stop SID` → `serve repo "new cmd"` → wait ~20s → `output` on the new sessionId. + +**Hard limits this surface enforces:** +- `cookbook serve` cmd allowlist + shell-metacharacter rejection. +- `cookbook stop` requires sessionIds matching `[a-zA-Z0-9_-]+`. +- Agent CAN spawn GPU-pinning long-lived processes — always `cookbook stop` your previous attempt before relaunching. + +## Forbidden Bypass Pattern + +If you are about to reach the Odysseus host/container, import app internals, query the database, or call MCP helper modules directly, stop. Those paths bypass Odysseus Settings and token scopes. Ask the user to enable the relevant Codex Agent tool toggle instead. diff --git a/launch-windows.ps1 b/launch-windows.ps1 new file mode 100644 index 000000000..263d95127 --- /dev/null +++ b/launch-windows.ps1 @@ -0,0 +1,169 @@ +#Requires -Version 5.1 +<# + Odysseus - native Windows launcher (no Docker). + + One command to: create a virtualenv, install dependencies, run first-time + setup (prints an admin password on first run), and start the server. + Safe to re-run - it skips whatever already exists. + + Usage: + powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1 + powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1 -Port 7000 -BindHost 127.0.0.1 + + Tip: bind 127.0.0.1 (default) for local-only use. Use 0.0.0.0 only when you + intentionally want other devices on your LAN to reach it. +#> +param( + [int]$Port = 7000, + [string]$BindHost = "127.0.0.1" +) + +$ErrorActionPreference = "Stop" +Set-Location -Path $PSScriptRoot + +function Write-Step($msg) { Write-Host ""; Write-Host ("==> " + $msg) -ForegroundColor Cyan } +function Fail($msg) { + Write-Host "" + Write-Host ("ERROR: " + $msg) -ForegroundColor Red + Write-Host "" + Read-Host "Press Enter to exit" + exit 1 +} + +function Test-WindowsBashStub($path) { + if (-not $path) { return $false } + $lowered = $path.ToLowerInvariant() + foreach ($stub in @("system32\bash.exe", "sysnative\bash.exe", "windowsapps\bash.exe")) { + if ($lowered.Contains($stub)) { return $true } + } + return $false +} + +function Find-GitBash { + $cmd = Get-Command bash -ErrorAction SilentlyContinue + if ($cmd -and -not (Test-WindowsBashStub $cmd.Source)) { return $cmd.Source } + + $roots = @() + foreach ($name in @("ProgramFiles", "ProgramW6432", "ProgramFiles(x86)", "LocalAppData")) { + $base = [Environment]::GetEnvironmentVariable($name) + if ($base) { + $roots += (Join-Path $base "Git") + if ($name -eq "LocalAppData") { $roots += (Join-Path $base "Programs\Git") } + } + } + $roots += @("C:\Program Files\Git", "C:\Program Files (x86)\Git") + + foreach ($root in ($roots | Select-Object -Unique)) { + foreach ($relative in @("bin\bash.exe", "usr\bin\bash.exe")) { + $candidate = Join-Path $root $relative + if (Test-Path $candidate) { return $candidate } + } + } + return $null +} + +# 1. Locate a Python interpreter (3.11+ required) +Write-Step "Checking for Python" +function Get-PythonVersionText($launcher, $launcherArgs) { + try { + return (& $launcher @launcherArgs -c "import sys; print('.'.join(map(str, sys.version_info[:3])))" 2>$null).Trim() + } catch { + return $null + } +} + +$pyExe = $null +$pyArgs = @() +$pyVersion = $null + +$pyLauncher = Get-Command py -ErrorAction SilentlyContinue +if ($pyLauncher) { + foreach ($v in @("-3.13", "-3.12", "-3.11")) { + $ver = Get-PythonVersionText $pyLauncher.Source @($v) + if ($ver) { + $pyExe = $pyLauncher.Source + $pyArgs = @($v) + $pyVersion = $ver + break + } + } +} + +if (-not $pyExe) { + $pythonCmd = Get-Command python -ErrorAction SilentlyContinue + if ($pythonCmd) { + $ver = Get-PythonVersionText $pythonCmd.Source @() + if ($ver) { + $versionParts = $ver.Split('.') + $major = [int]$versionParts[0] + $minor = [int]$versionParts[1] + if ($major -gt 3 -or ($major -eq 3 -and $minor -ge 11)) { + $pyExe = $pythonCmd.Source + $pyVersion = $ver + } + } + } +} + +if ($pyExe -like "*WindowsApps*python.exe") { + $pyCmd = Get-Command py -ErrorAction SilentlyContinue + if ($pyCmd) { + $pyExe = $pyCmd.Source + $pyArgs = @("-3.11") + } +} + +if (-not $pyExe) { + Fail "Couldn't find Python 3.11+ for Windows setup. Install Python 3.11+ (or open the Python launcher with 'py -3.11') from https://www.python.org/downloads/, then re-run this script." +} +$pythonLabel = ("Using Python {0}: {1} {2}" -f $pyVersion, $pyExe, ($pyArgs -join ' ')).TrimEnd() +Write-Host $pythonLabel + +# 2. Create the virtualenv if missing +$venvPy = Join-Path $PSScriptRoot "venv\Scripts\python.exe" +if (-not (Test-Path $venvPy)) { + Write-Step "Creating virtual environment (venv)" + & $pyExe @pyArgs -m venv venv + if ($LASTEXITCODE -ne 0 -or -not (Test-Path $venvPy)) { Fail "Failed to create the virtual environment." } +} else { + Write-Host "venv already exists - skipping creation." +} + +# 3. Install / update dependencies +Write-Step "Installing dependencies (first run can take a few minutes)" +& $venvPy -m pip install --upgrade pip --quiet +& $venvPy -m pip install -r requirements.txt +if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed. Scroll up for the pip error." } + +# 4. First-time setup (creates data dirs, DB, .env, admin user) +Write-Step "Running first-time setup" +& $venvPy setup.py +if ($LASTEXITCODE -ne 0) { Fail "setup.py failed." } + +# 5. Friendly note about Git Bash (full Cookbook / agent-shell parity) +if (-not (Find-GitBash)) { + Write-Host "" + Write-Host "NOTE: Git Bash (bash.exe) was not found on PATH." -ForegroundColor Yellow + Write-Host " The core app works without it. For full Cookbook background" -ForegroundColor Yellow + Write-Host " downloads and the agent shell tool, install Git for Windows:" -ForegroundColor Yellow + Write-Host " https://git-scm.com/download/win" -ForegroundColor Yellow +} + +# 6. Point CUDA_PATH at a real CUDA toolkit so GPU llama-cpp-python can import. +$cudaBase = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA" +if (Test-Path $cudaBase) { + $cudaBest = Get-ChildItem $cudaBase -Directory -ErrorAction SilentlyContinue | + Where-Object { Test-Path (Join-Path $_.FullName "bin") } | + Sort-Object { try { [version]($_.Name -replace "^v", "") } catch { [version]"0.0" } } -Descending | + Select-Object -First 1 + if ($cudaBest) { + $env:CUDA_PATH = $cudaBest.FullName + Write-Host ("Using CUDA_PATH = " + $cudaBest.FullName) -ForegroundColor Cyan + } +} + +# 7. Start the server (use `python -m uvicorn` - bare `uvicorn` may not be on PATH) +Write-Step ("Starting Odysseus at http://{0}:{1}" -f $BindHost, $Port) +Write-Host "Press Ctrl+C to stop." +Write-Host "" +& $venvPy -m uvicorn app:app --host $BindHost --port $Port diff --git a/launcher.py b/launcher.py new file mode 100644 index 000000000..ba158444f --- /dev/null +++ b/launcher.py @@ -0,0 +1,142 @@ +# launcher.py +"""Dedicated entrypoint for the standalone Windows portable launcher. + +Handles: +- Immediate GUI splash screen creation using tkinter. +- Suppressing console stream crashes in windowed GUI mode via NullWriter. +- Spawning system tray icon via pystray and Pillow (lazy-loaded). +- Auto-opening default browser pointing to the running backend. +- Launching the FastAPI server (importing and running app.py). +""" +import os +import sys +import threading +import time +import webbrowser + +# Define a dummy NullWriter to suppress standard stream crashes (isatty etc.) in GUI mode +class NullWriter: + def write(self, text): + pass + def flush(self): + pass + def isatty(self): + return False + +if sys.stdout is None: + sys.stdout = NullWriter() +if sys.stderr is None: + sys.stderr = NullWriter() + + +splash_root = None + +# If running from a frozen PyInstaller bundle, launch the splash screen IMMEDIATELY +if getattr(sys, 'frozen', False): + import tkinter as tk + + def show_splash_instantly(): + global splash_root + try: + splash_root = tk.Tk() + splash_root.title("Odysseus") + splash_root.overrideredirect(True) + splash_root.configure(bg="#1a1c23") + + # Accented borders + splash_root.config(highlightbackground="#e06c75", highlightcolor="#e06c75", highlightthickness=1) + + w, h = 360, 160 + ws = splash_root.winfo_screenwidth() + hs = splash_root.winfo_screenheight() + x = (ws - w) // 2 + y = (hs - h) // 2 + splash_root.geometry(f"{w}x{h}+{x}+{y}") + + tk.Label(splash_root, text="⛵ Odysseus", font=("Segoe UI", 22, "bold"), bg="#1a1c23", fg="#e06c75").pack(pady=(22, 2)) + tk.Label(splash_root, text="Launching background services...", font=("Segoe UI", 10), bg="#1a1c23", fg="#d1d4e0").pack(pady=2) + tk.Label(splash_root, text="Please wait, this will take a few seconds.", font=("Segoe UI", 8, "italic"), bg="#1a1c23", fg="#5c6370").pack(pady=(12, 0)) + + splash_root.attributes("-topmost", True) + splash_root.mainloop() + except Exception: + pass + + # Launch the GUI splash screen immediately on a background thread + threading.Thread(target=show_splash_instantly, daemon=True).start() + + +def create_tray_image(): + # Generate a beautiful 64x64 icon matching Odysseus brand red accent (#e06c75) + from PIL import Image, ImageDraw + image = Image.new('RGBA', (64, 64), (0, 0, 0, 0)) + dc = ImageDraw.Draw(image) + accent_red = (224, 108, 117, 255) + light_red = (224, 108, 117, 150) + + # Draw premium sailing boat + dc.polygon([(32, 10), (32, 45), (12, 45)], fill=accent_red) + dc.polygon([(32, 18), (32, 45), (48, 45)], fill=light_red) + dc.polygon([(8, 48), (56, 48), (44, 56), (20, 56)], fill=accent_red) + return image + + +def on_open_browser(icon, item, url): + webbrowser.open(url) + + +def on_exit(icon, item): + icon.stop() + os._exit(0) + + +def setup_system_tray(url): + try: + import pystray + icon_img = create_tray_image() + menu = ( + pystray.MenuItem('Open Odysseus', lambda icon, item: on_open_browser(icon, item, url), default=True), + pystray.MenuItem('Exit', on_exit) + ) + tray_icon = pystray.Icon( + "Odysseus", + icon_img, + "Odysseus", + menu + ) + tray_icon.run() + except Exception: + pass + + +def open_browser(url): + # Allow uvicorn and app lifecycles to complete warmups + time.sleep(3.5) + + # Safely close the splash screen + try: + global splash_root + if splash_root: + splash_root.after(0, splash_root.destroy) + except Exception: + pass + + webbrowser.open(url) + + +if __name__ == "__main__": + import uvicorn + # Import the FastAPI app from app.py + from app import app + + bind_host = os.getenv("APP_BIND", "127.0.0.1") + bind_port = int(os.getenv("APP_PORT", "7000")) + url = f"http://{bind_host}:{bind_port}" + + if getattr(sys, 'frozen', False): + # Start browser manager thread + threading.Thread(target=open_browser, args=(url,), daemon=True).start() + # Start system tray manager thread + threading.Thread(target=setup_system_tray, args=(url,), daemon=True).start() + + uvicorn.run(app, host=bind_host, port=bind_port, log_level="info") diff --git a/licenses/OpenDyslexic-OFL.txt b/licenses/OpenDyslexic-OFL.txt new file mode 100644 index 000000000..0a1d034b9 --- /dev/null +++ b/licenses/OpenDyslexic-OFL.txt @@ -0,0 +1,94 @@ +Copyright (c) 2019-07-29, Abbie Gonzalez (https://abbiecod.es|support@abbiecod.es), +with Reserved Font Name OpenDyslexic. +Copyright (c) 12/2012 - 2019 +This Font Software is licensed under the SIL Open Font License, Version 1.1. +This license is copied below, and is also available with a FAQ at: +http://scripts.sil.org/OFL + + +----------------------------------------------------------- +SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007 +----------------------------------------------------------- + +PREAMBLE +The goals of the Open Font License (OFL) are to stimulate worldwide +development of collaborative font projects, to support the font creation +efforts of academic and linguistic communities, and to provide a free and +open framework in which fonts may be shared and improved in partnership +with others. + +The OFL allows the licensed fonts to be used, studied, modified and +redistributed freely as long as they are not sold by themselves. The +fonts, including any derivative works, can be bundled, embedded, +redistributed and/or sold with any software provided that any reserved +names are not used by derivative works. The fonts and derivatives, +however, cannot be released under any other type of license. The +requirement for fonts to remain under this license does not apply +to any document created using the fonts or their derivatives. + +DEFINITIONS +"Font Software" refers to the set of files released by the Copyright +Holder(s) under this license and clearly marked as such. This may +include source files, build scripts and documentation. + +"Reserved Font Name" refers to any names specified as such after the +copyright statement(s). + +"Original Version" refers to the collection of Font Software components as +distributed by the Copyright Holder(s). + +"Modified Version" refers to any derivative made by adding to, deleting, +or substituting -- in part or in whole -- any of the components of the +Original Version, by changing formats or by porting the Font Software to a +new environment. + +"Author" refers to any designer, engineer, programmer, technical +writer or other person who contributed to the Font Software. + +PERMISSION & CONDITIONS +Permission is hereby granted, free of charge, to any person obtaining +a copy of the Font Software, to use, study, copy, merge, embed, modify, +redistribute, and sell modified and unmodified copies of the Font +Software, subject to the following conditions: + +1) Neither the Font Software nor any of its individual components, +in Original or Modified Versions, may be sold by itself. + +2) Original or Modified Versions of the Font Software may be bundled, +redistributed and/or sold with any software, provided that each copy +contains the above copyright notice and this license. These can be +included either as stand-alone text files, human-readable headers or +in the appropriate machine-readable metadata fields within text or +binary files as long as those fields can be easily viewed by the user. + +3) No Modified Version of the Font Software may use the Reserved Font +Name(s) unless explicit written permission is granted by the corresponding +Copyright Holder. This restriction only applies to the primary font name as +presented to the users. + +4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font +Software shall not be used to promote, endorse or advertise any +Modified Version, except to acknowledge the contribution(s) of the +Copyright Holder(s) and the Author(s) or with their explicit written +permission. + +5) The Font Software, modified or unmodified, in part or in whole, +must be distributed entirely under this license, and must not be +distributed under any other license. The requirement for fonts to +remain under this license does not apply to any document created +using the Font Software. + +TERMINATION +This license becomes null and void if any of the above conditions are +not met. + +DISCLAIMER +THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT +OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE +COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL +DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM +OTHER DEALINGS IN THE FONT SOFTWARE. diff --git a/mcp_servers/_common.py b/mcp_servers/_common.py deleted file mode 100644 index 641c8522d..000000000 --- a/mcp_servers/_common.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -_common.py - -Shared constants and helpers for built-in MCP servers. -""" - -MAX_OUTPUT_CHARS = 10_000 -MAX_READ_CHARS = 20_000 -SHELL_TIMEOUT = 60 -PYTHON_TIMEOUT = 30 -SEARCH_TIMEOUT = 30 - - -def truncate(text: str, limit: int = MAX_OUTPUT_CHARS) -> str: - """Truncate text to *limit* characters with a suffix note.""" - if len(text) > limit: - return text[:limit] + f"\n... (truncated, {len(text)} chars total)" - return text diff --git a/mcp_servers/email_server.py b/mcp_servers/email_server.py index f5b89ee07..e2bccfbfb 100644 --- a/mcp_servers/email_server.py +++ b/mcp_servers/email_server.py @@ -22,6 +22,8 @@ import os.path from pathlib import Path from datetime import datetime, timedelta +import uuid +from contextvars import ContextVar from mcp.server import Server from mcp.server.stdio import stdio_server @@ -31,13 +33,19 @@ server = Server("email") EMAIL_SOCKET_TIMEOUT = float(os.environ.get("EMAIL_SOCKET_TIMEOUT", "20")) -DATA_DIR = Path(__file__).resolve().parent.parent / "data" +from src.constants import DATA_DIR as _DATA_DIR, APP_DB, EMAIL_CACHE_DB, SETTINGS_FILE as _SETTINGS_FILE, MAIL_ATTACHMENTS_DIR +DATA_DIR = Path(_DATA_DIR) def _b(value) -> bytes: return str(value).encode() +def _q(name: str) -> str: + """Quote an IMAP mailbox name for commands that take mailbox args.""" + return '"' + (name or "").replace("\\", "\\\\").replace('"', '\\"') + '"' + + def _uid_fetch_rows(data) -> list: return [d for d in (data or []) if isinstance(d, bytes) and b"UID " in d] @@ -48,6 +56,13 @@ def _uid_fetch_rows(data) -> list: # flat keys when no DB row matches (legacy single-account behaviour). _ACCOUNT_CACHE: dict = {} # key = normalized account selector -> config dict +_MCP_OWNER_ARG = "_odysseus_owner" +_CURRENT_OWNER: ContextVar[str | None] = ContextVar("email_mcp_owner", default=None) +_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_EMAIL_OWNER", "ODYSSEUS_EMAIL_OWNER") +_OWNER_SCOPE_ERROR = ( + "Error: email MCP requires an authenticated owner or ODYSSEUS_MCP_EMAIL_OWNER " + "when owner-scoped email accounts are configured." +) def _clean_header_value(value) -> str: @@ -58,22 +73,130 @@ def _clean_header_value(value) -> str: def _db_path() -> Path: - return DATA_DIR / "app.db" + return Path(APP_DB) -def _list_accounts_raw() -> list: - """Return list of dicts from the email_accounts table. Empty list if table - missing or empty. Never raises.""" +def _configured_owner() -> str | None: + for key in _OWNER_ENV_KEYS: + owner = os.environ.get(key, "").strip() + if owner: + return owner + return None + + +def _current_owner() -> str: + owner = _CURRENT_OWNER.get() + return str(owner or _configured_owner() or "").strip() + + +def _account_owner(row: dict) -> str: + return str(row.get("owner") or "").strip() + + +def _has_owner_scoped_accounts(rows: list[dict]) -> bool: + return any(_account_owner(r) for r in rows) + + +def _account_visible_to_owner(row: dict, owner: str) -> bool: + row_owner = _account_owner(row) + if row_owner == owner: + return True + if row_owner: + return False + # Legacy ownerless accounts are only visible to a scoped caller when the + # mailbox itself matches the owner, mirroring the HTTP email route fallback. + owner_l = owner.lower() + return owner_l in { + str(row.get("imap_user") or "").strip().lower(), + str(row.get("from_address") or "").strip().lower(), + } + + +def _filter_accounts_for_owner(rows: list[dict]) -> list[dict]: + owner = _current_owner() + if owner: + return [r for r in rows if _account_visible_to_owner(r, owner)] + + if _has_owner_scoped_accounts(rows): + return [] + return rows + + +def _mcp_owner_required(rows: list[dict] | None = None) -> bool: + if _current_owner(): + return False + rows = rows if rows is not None else _read_accounts_from_db() + return _has_owner_scoped_accounts(rows) + + +def _load_email_writing_style() -> str: + """Return the existing Settings > Email > Writing Style value.""" + try: + settings_path = DATA_DIR / "settings.json" + if not settings_path.exists(): + return "" + settings = json.loads(settings_path.read_text(encoding="utf-8")) + return str(settings.get("email_writing_style") or "").strip() + except Exception: + return "" + + +def _writing_style_guidance() -> str: + style = _load_email_writing_style() + if not style: + return ( + "No saved writing style is configured in Settings > Email > Writing Style. " + "Use a concise, natural tone and do not invent facts." + ) + return ( + "Use this saved writing style from Settings > Email > Writing Style when " + "drafting the body. It overrides generic tone guidance:\n" + f"{style}" + ) + + +def _default_document_owner() -> str | None: + """Best-effort owner for MCP-created documents. + + MCP stdio tools do not receive the browser request's authenticated user, + but the document library is owner-filtered. Stamp drafts to the configured + single/default admin so assistant-created email drafts are visible. + """ + owner = os.environ.get("ODYSSEUS_DOCUMENT_OWNER", "").strip() + if owner: + return owner + try: + auth_path = DATA_DIR / "auth.json" + if not auth_path.exists(): + return None + users = (json.loads(auth_path.read_text(encoding="utf-8")).get("users") or {}) + if not isinstance(users, dict) or not users: + return None + admins = [name for name, data in users.items() if isinstance(data, dict) and data.get("is_admin")] + if len(admins) == 1: + return admins[0] + if len(users) == 1: + return next(iter(users)) + return admins[0] if admins else next(iter(users)) + except Exception: + return None + + +def _read_accounts_from_db() -> list: + """Return all enabled email account rows. Empty list if missing. Never raises.""" path = _db_path() if not path.exists(): return [] try: conn = sqlite3.connect(str(path)) conn.row_factory = sqlite3.Row - rows = conn.execute(""" - SELECT id, name, is_default, enabled, + columns = {r[1] for r in conn.execute("PRAGMA table_info(email_accounts)").fetchall()} + owner_select = "owner" if "owner" in columns else "NULL AS owner" + smtp_security_select = "smtp_security" if "smtp_security" in columns else "'' AS smtp_security" + rows = conn.execute(f""" + SELECT id, {owner_select}, name, is_default, enabled, imap_host, imap_port, imap_user, imap_password, imap_starttls, - smtp_host, smtp_port, smtp_user, smtp_password, from_address + smtp_host, smtp_port, {smtp_security_select}, smtp_user, smtp_password, from_address FROM email_accounts WHERE enabled = 1 ORDER BY is_default DESC, created_at ASC """).fetchall() @@ -85,11 +208,15 @@ def _list_accounts_raw() -> list: return [] -def _resolve_account(selector: str | None) -> dict | None: +def _list_accounts_raw() -> list: + """Return owner-visible email account rows for the active MCP call.""" + return _filter_accounts_for_owner(_read_accounts_from_db()) + + +def _resolve_account_from_rows(rows: list[dict], selector: str | None) -> dict | None: """Given a selector (None = default, or a name/user/id string), return the matching row or None. Matching is case-insensitive substring on name + imap_user + from_address, plus exact id match.""" - rows = _list_accounts_raw() if not rows: return None if not selector: @@ -124,6 +251,10 @@ def _resolve_account(selector: str | None) -> dict | None: return None +def _resolve_account(selector: str | None) -> dict | None: + return _resolve_account_from_rows(_list_accounts_raw(), selector) + + def _load_config(account: str | None = None) -> dict: """Return the full config dict for the requested account (or default). @@ -132,7 +263,7 @@ def _load_config(account: str | None = None) -> dict: 2. env vars + settings.json flat keys (legacy) 3. hardcoded fallbacks (localhost:31143 etc.) """ - cache_key = (account or "").strip().lower() or "__default__" + cache_key = (_current_owner(), (account or "").strip().lower() or "__default__") if cache_key in _ACCOUNT_CACHE: return _ACCOUNT_CACHE[cache_key] @@ -145,6 +276,7 @@ def _load_config(account: str | None = None) -> dict: "imap_starttls": os.environ.get("IMAP_STARTTLS", "true").lower() == "true", "smtp_host": os.environ.get("SMTP_HOST", ""), "smtp_port": int(os.environ.get("SMTP_PORT", "465")), + "smtp_security": os.environ.get("SMTP_SECURITY", ""), "smtp_user": os.environ.get("SMTP_USER", ""), "smtp_password": os.environ.get("SMTP_PASSWORD", ""), "smtp_starttls": os.environ.get("SMTP_STARTTLS", "false").lower() == "true", @@ -154,14 +286,19 @@ def _load_config(account: str | None = None) -> dict: "trash_folder": os.environ.get("TRASH_FOLDER", "Trash"), "cache_db": os.environ.get( "EMAIL_CACHE_DB", - str(DATA_DIR / "email_cache.db"), + EMAIL_CACHE_DB, ), "account_id": None, "account_name": None, } - rows = _list_accounts_raw() - row = _resolve_account(account) + raw_rows = _read_accounts_from_db() + if _mcp_owner_required(raw_rows): + raise ValueError(_OWNER_SCOPE_ERROR) + rows = _filter_accounts_for_owner(raw_rows) + row = _resolve_account_from_rows(rows, account) + if _current_owner() and raw_rows and not rows: + raise ValueError("No email account is configured for the authenticated owner") if account and rows and not row: available = ", ".join( f"{r.get('name') or r.get('imap_user')} <{r.get('imap_user') or r.get('from_address') or '?'}>" @@ -189,15 +326,16 @@ def _load_config(account: str | None = None) -> dict: cfg["imap_ssl"] = int(cfg["imap_port"]) == 993 and not cfg["imap_starttls"] cfg["smtp_host"] = row["smtp_host"] or cfg["smtp_host"] cfg["smtp_port"] = int(row["smtp_port"] or cfg["smtp_port"]) + cfg["smtp_security"] = row["smtp_security"] or cfg["smtp_security"] or ("starttls" if int(cfg["smtp_port"]) == 587 else "ssl") cfg["smtp_user"] = row["smtp_user"] or cfg["smtp_user"] cfg["smtp_password"] = _decrypt(row["smtp_password"]) if row["smtp_password"] else cfg["smtp_password"] cfg["from_address"] = row["from_address"] or row["imap_user"] or cfg["from_address"] else: # Legacy fallback: settings.json flat keys try: - settings_path = Path(__file__).resolve().parent.parent / "data" / "settings.json" + settings_path = Path(_SETTINGS_FILE) if settings_path.exists(): - settings = json.loads(settings_path.read_text()) + settings = json.loads(settings_path.read_text(encoding="utf-8")) for key in ( "imap_host", "imap_port", "imap_user", "imap_password", "smtp_host", "smtp_port", "smtp_user", "smtp_password", @@ -235,10 +373,27 @@ def _imap_connect(account: str | None = None): timeout=EMAIL_SOCKET_TIMEOUT, ) if cfg["imap_starttls"]: - conn.starttls() + try: + conn.starttls() + except Exception: + # Don't leak the open plain socket on a rejected STARTTLS. (#3174) + try: + conn.shutdown() + except Exception: + pass + raise if getattr(conn, "sock", None): conn.sock.settimeout(EMAIL_SOCKET_TIMEOUT) - conn.login(cfg["imap_user"], cfg["imap_password"]) + try: + conn.login(cfg["imap_user"], cfg["imap_password"]) + except Exception: + # A failed login otherwise orphans the connected socket; close it + # before propagating (shutdown() is the pre-auth low-level close). (#3174) + try: + conn.shutdown() + except Exception: + pass + raise return conn @@ -333,14 +488,25 @@ def _decode_header(raw): """Decode MIME encoded header.""" if not raw: return "" - parts = email.header.decode_header(raw) - decoded = [] - for data, charset in parts: - if isinstance(data, bytes): - decoded.append(data.decode(charset or "utf-8", errors="replace")) - else: - decoded.append(data) - return " ".join(decoded) + try: + # make_header concatenates per RFC 2047: no spurious space between an + # encoded-word and adjacent plain text (plain runs keep their own + # whitespace), and whitespace between two adjacent encoded-words is + # dropped. The old " ".join produced "Re: Jose" style double spaces + # on every non-ASCII subject or sender. + return str(email.header.make_header(email.header.decode_header(raw))) + except Exception: + # Malformed header or unknown charset: lossy per-part decode + decoded = [] + for data, charset in email.header.decode_header(raw): + if isinstance(data, bytes): + try: + decoded.append(data.decode(charset or "utf-8", errors="replace")) + except LookupError: + decoded.append(data.decode("utf-8", errors="replace")) + else: + decoded.append(data) + return "".join(decoded) def _extract_text(msg): @@ -393,6 +559,148 @@ def _get_cached_summaries(): return {} +def _fixture_email_file() -> Path: + return DATA_DIR / "fixture_email_messages.json" + + +def _fixture_email_enabled() -> bool: + return _fixture_email_file().exists() + + +def _parse_fixture_date(raw_date: str) -> tuple[str, float]: + if not raw_date: + return "", 0.0 + parsed = None + try: + parsed = datetime.fromisoformat(str(raw_date).replace("Z", "+00:00")) + except Exception: + try: + parsed = email.utils.parsedate_to_datetime(str(raw_date)) + except Exception: + parsed = None + if parsed: + return parsed.isoformat(), parsed.timestamp() + return str(raw_date), 0.0 + + +def _fixture_email_record(row: dict, uid_num: int, owner: str) -> dict: + sender = str(row.get("from") or "Fixture Sender ") + sender_name, sender_addr = email.utils.parseaddr(sender) + date_str, date_epoch = _parse_fixture_date(str(row.get("date") or "")) + subject = str(row.get("subject") or "(no subject)") + body = str(row.get("body") or "") + owner_key = re.sub(r"[^A-Za-z0-9_.-]", "-", owner or "default") + uid = str(uid_num) + return { + "uid": uid, + "message_id": f"", + "subject": subject, + "from": sender_name or sender_addr or sender, + "from_address": sender_addr, + "date": date_str, + "date_epoch": date_epoch, + "summary": body[:240], + "body": body, + "account": "Fixture Inbox", + "account_email": owner or str(row.get("owner") or ""), + "account_id": "fixture-email", + "attachments": [], + } + + +def _fixture_email_rows(owner: str | None = None) -> list[dict]: + path = _fixture_email_file() + if not path.exists(): + return [] + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return [] + rows = raw.get("messages") if isinstance(raw, dict) else raw + out = [] + owner = str(owner or "").strip() + for i, row in enumerate(rows if isinstance(rows, list) else [], start=1): + if not isinstance(row, dict): + continue + row_owner = str(row.get("owner") or "").strip() + if owner and row_owner and row_owner != owner: + continue + out.append(_fixture_email_record(row, i, owner or row_owner)) + out.sort(key=lambda item: item.get("date_epoch") or 0, reverse=True) + return out + + +def _fixture_account_rows() -> list[dict]: + if not _fixture_email_enabled(): + return [] + owner = _current_owner() + owners = [] + for row in _fixture_email_rows(owner or None): + email_addr = row.get("account_email") or owner or "fixture@fixtures.odysseus.local" + if email_addr not in owners: + owners.append(email_addr) + if not owners: + owners = [owner or "fixture@fixtures.odysseus.local"] + return [ + { + "id": "fixture-email", + "owner": owner or owners[0], + "name": "Fixture Inbox", + "is_default": True, + "imap_user": owners[0], + "from_address": owners[0], + } + ] + + +def _fixture_email_matches(item: dict, query: str) -> bool: + if not query: + return True + terms = [term for term in re.split(r"\W+", str(query).lower()) if term] + haystack = "\n".join( + str(item.get(key) or "") + for key in ("subject", "from", "from_address", "body", "summary") + ).lower() + return all(term in haystack for term in terms) + + +def _fixture_list_emails(folder="INBOX", max_results=20, unresponded_only=False, + unread_only=False, account=None) -> list[dict] | None: + if not _fixture_email_enabled(): + return None + if account and str(account).strip().lower() not in { + "fixture-email", + "fixture inbox", + "fixture", + str(_current_owner()).lower(), + }: + return [] + if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}: + return [] + return _fixture_email_rows(_current_owner())[: int(max_results or 20)] + + +def _fixture_search_emails(query, folders=None, max_results=20, account=None) -> list[dict] | None: + if not _fixture_email_enabled(): + return None + rows = _fixture_list_emails("INBOX", max_results=1000, account=account) or [] + out = [dict(row, _folder="INBOX") for row in rows if _fixture_email_matches(row, str(query or ""))] + return out[: int(max_results or 20)] + + +def _fixture_read_email(uid=None, message_id=None, folder="INBOX", account=None) -> dict | None: + if not _fixture_email_enabled(): + return None + if (folder or "INBOX").upper() not in {"INBOX", "ALL", "ALL MAIL"}: + return {"error": f"Email UID {uid or message_id} not found"} + for item in _fixture_email_rows(_current_owner()): + if uid and str(item.get("uid")) == str(uid): + return item + if message_id and str(item.get("message_id")) == str(message_id): + return item + return {"error": f"Email not found with UID/Message-ID: {uid or message_id}"} + + # ── Tool implementations ── @@ -403,63 +711,74 @@ def _list_emails(folder="INBOX", max_results=20, unresponded_only=False, Pass unread_only=True and/or unresponded_only=True for attention scans. account selects mailbox (None = default). """ - conn = _imap_connect(account) - select_status, _ = conn.select(folder, readonly=True) - if select_status != "OK": - conn.logout() - raise ValueError(f"IMAP folder not found: {folder}") - - if unread_only and unresponded_only: - status, data = conn.uid("SEARCH", None, "(UNSEEN UNANSWERED)") - elif unread_only: - status, data = conn.uid("SEARCH", None, "(UNSEEN)") - else: - # Include read too — IMAP search "ALL" returns the entire folder - status, data = conn.uid("SEARCH", None, "ALL") + fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, account) + if fixture is not None: + return fixture + conn = None + try: + conn = _imap_connect(account) + select_status, _ = conn.select(_q(folder), readonly=True) + if select_status != "OK": + raise ValueError(f"IMAP folder not found: {folder}") + + if unread_only and unresponded_only: + status, data = conn.uid("SEARCH", None, "(UNSEEN UNANSWERED)") + elif unread_only: + status, data = conn.uid("SEARCH", None, "(UNSEEN)") + elif unresponded_only: + # Was missing — unresponded_only=True (without unread_only) fell through + # to "ALL" and returned answered mail too, despite the documented + # "emails without replies" behaviour. + status, data = conn.uid("SEARCH", None, "(UNANSWERED)") + else: + # Include read too — IMAP search "ALL" returns the entire folder + status, data = conn.uid("SEARCH", None, "ALL") - if status != "OK" or not data[0]: - conn.logout() - return [] + if status != "OK" or not data[0]: + return [] - uid_list = list(reversed(data[0].split()))[:max_results] - cache = _get_cached_summaries() - results = [] + uid_list = list(reversed(data[0].split()))[:max_results] + cache = _get_cached_summaries() + results = [] - for uid in uid_list: - try: - status, msg_data = conn.uid("FETCH", uid, "(RFC822.HEADER)") - if status != "OK": + for uid in uid_list: + try: + status, msg_data = conn.uid("FETCH", uid, "(RFC822.HEADER)") + if status != "OK": + continue + raw_header = msg_data[0][1] + msg = email.message_from_bytes(raw_header) + + subject = _decode_header(msg.get("Subject", "(no subject)")) + sender = _decode_header(msg.get("From", "unknown")) + date_str = msg.get("Date", "") + message_id = msg.get("Message-ID", "") + + # Parse sender name + sender_name, sender_addr = email.utils.parseaddr(sender) + sender_display = sender_name or sender_addr + + # Check cache for summary + cached = cache.get(subject, {}) + summary = cached.get("summary", "") + + results.append({ + "uid": uid.decode(), + "message_id": message_id, + "subject": subject, + "from": sender_display, + "from_address": sender_addr, + "date": date_str, + "summary": summary, + }) + except Exception: continue - raw_header = msg_data[0][1] - msg = email.message_from_bytes(raw_header) - - subject = _decode_header(msg.get("Subject", "(no subject)")) - sender = _decode_header(msg.get("From", "unknown")) - date_str = msg.get("Date", "") - message_id = msg.get("Message-ID", "") - - # Parse sender name - sender_name, sender_addr = email.utils.parseaddr(sender) - sender_display = sender_name or sender_addr - - # Check cache for summary - cached = cache.get(subject, {}) - summary = cached.get("summary", "") - - results.append({ - "uid": uid.decode(), - "message_id": message_id, - "subject": subject, - "from": sender_display, - "from_address": sender_addr, - "date": date_str, - "summary": summary, - }) - except Exception: - continue - conn.logout() - return results + return results + finally: + if conn: + try: conn.logout() + except Exception: pass def _result_sort_time(result: dict) -> datetime: @@ -476,6 +795,9 @@ def _result_sort_time(result: dict) -> datetime: def _list_emails_across_accounts(folder="INBOX", max_results=20, unresponded_only=False, unread_only=False): + fixture = _fixture_list_emails(folder, max_results, unresponded_only, unread_only, None) + if fixture is not None: + return fixture, [] rows = _list_accounts_raw() combined = [] errors = [] @@ -509,6 +831,9 @@ def _search_emails(query, folders=None, max_results=20, account=None): _list_emails plus an `_folder` tag.""" if not query or not str(query).strip(): return [] + fixture = _fixture_search_emails(query, folders=folders, max_results=max_results, account=account) + if fixture is not None: + return fixture q = str(query).replace("\\", "\\\\").replace('"', '\\"') # Mail clients commonly use OR FROM/SUBJECT/TEXT to match either field. # IMAP SEARCH OR is binary, so we nest it. @@ -522,7 +847,7 @@ def _search_emails(query, folders=None, max_results=20, account=None): try: for folder in folders: try: - status, _ = conn.select(folder, readonly=True) + status, _ = conn.select(_q(folder), readonly=True) if status != "OK": continue status, data = conn.uid("SEARCH", None, search_cmd) @@ -631,58 +956,65 @@ def _extract_attachment_to_disk(msg, index, target_dir): def _read_email(uid=None, message_id=None, folder="INBOX", account=None): """Read full email content by UID or message-ID. account = mailbox selector.""" + fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=account) + if fixture is not None: + return fixture cfg = _load_config(account) - conn = _imap_connect(account) - conn.select(folder, readonly=True) + conn = None + try: + conn = _imap_connect(account) + conn.select(_q(folder), readonly=True) - if message_id and not uid: - status, data = conn.uid("SEARCH", None, f'(HEADER Message-ID "{message_id}")') - if status != "OK" or not data[0]: - conn.logout() - return {"error": f"Email not found with Message-ID: {message_id}"} - uid = data[0].split()[-1] + if message_id and not uid: + status, data = conn.uid("SEARCH", None, f'(HEADER Message-ID "{message_id}")') + if status != "OK" or not data[0]: + return {"error": f"Email not found with Message-ID: {message_id}"} + uid = data[0].split()[-1] - if not uid: - conn.logout() - return {"error": "No UID or Message-ID provided"} + if not uid: + return {"error": "No UID or Message-ID provided"} - status, msg_data = conn.uid("FETCH", _b(uid), "(RFC822)") - if status != "OK": - conn.logout() - return {"error": f"Failed to fetch email UID {uid}"} - if not msg_data or not msg_data[0] or not isinstance(msg_data[0], tuple) or len(msg_data[0]) < 2: - conn.logout() - return {"error": f"Email not found with UID {uid}"} + status, msg_data = conn.uid("FETCH", _b(uid), "(BODY.PEEK[])") + if status != "OK": + return {"error": f"Failed to fetch email UID {uid}"} + if not msg_data or not msg_data[0] or not isinstance(msg_data[0], tuple) or len(msg_data[0]) < 2: + return {"error": f"Email not found with UID {uid}"} - raw = msg_data[0][1] - msg = email.message_from_bytes(raw) + raw = msg_data[0][1] + msg = email.message_from_bytes(raw) - subject = _decode_header(msg.get("Subject", "(no subject)")) - sender = _decode_header(msg.get("From", "unknown")) - date_str = msg.get("Date", "") - message_id_header = msg.get("Message-ID", "") - body = _extract_text(msg) - attachments = _list_attachments_from_msg(msg) + subject = _decode_header(msg.get("Subject", "(no subject)")) + sender = _decode_header(msg.get("From", "unknown")) + date_str = msg.get("Date", "") + message_id_header = msg.get("Message-ID", "") + body = _extract_text(msg) + attachments = _list_attachments_from_msg(msg) - sender_name, sender_addr = email.utils.parseaddr(sender) + sender_name, sender_addr = email.utils.parseaddr(sender) - conn.logout() - return { - "uid": uid.decode() if isinstance(uid, bytes) else str(uid), - "account": cfg.get("account_name") or cfg.get("imap_user") or "default", - "account_email": cfg.get("imap_user") or cfg.get("from_address") or "", - "account_id": cfg.get("account_id"), - "message_id": message_id_header, - "subject": subject, - "from": sender_name or sender_addr, - "from_address": sender_addr, - "date": date_str, - "body": body[:8000], - "attachments": attachments, - } + return { + "uid": uid.decode() if isinstance(uid, bytes) else str(uid), + "account": cfg.get("account_name") or cfg.get("imap_user") or "default", + "account_email": cfg.get("imap_user") or cfg.get("from_address") or "", + "account_id": cfg.get("account_id"), + "message_id": message_id_header, + "subject": subject, + "from": sender_name or sender_addr, + "from_address": sender_addr, + "date": date_str, + "body": body[:8000], + "attachments": attachments, + } + finally: + if conn: + try: conn.logout() + except Exception: pass def _read_email_across_accounts(uid=None, message_id=None, folder="INBOX"): + fixture = _fixture_read_email(uid=uid, message_id=message_id, folder=folder, account=None) + if fixture is not None: + return fixture rows = _list_accounts_raw() matches = [] errors = [] @@ -739,17 +1071,26 @@ def _smtp_connect(account=None, cfg=None): if not _smtp_ready(cfg): raise ValueError(f"Email account {cfg.get('account_name') or account or 'default'} has no SMTP configured") port = int(cfg.get("smtp_port") or 465) - # Account rows only store host/port, not the legacy env-level smtp_ssl - # toggle. Infer the conventional TLS mode from the port so MCP tools match - # the web send path: 465 = implicit SSL, 587 = STARTTLS. - if port == 587: + security = str(cfg.get("smtp_security") or "").strip().lower() + if security not in {"ssl", "starttls", "none"}: + security = "starttls" if port == 587 else "ssl" + if security == "starttls": conn = smtplib.SMTP( cfg["smtp_host"], port, timeout=EMAIL_SOCKET_TIMEOUT, ) - conn.starttls() - elif cfg.get("smtp_ssl", True): + try: + conn.starttls() + except Exception: + # Don't leak the open plain socket on a rejected STARTTLS. SMTP has + # no shutdown(); close() is the low-level socket close (no QUIT). (#3174) + try: + conn.close() + except Exception: + pass + raise + elif security == "ssl": conn = smtplib.SMTP_SSL( cfg["smtp_host"], port, @@ -761,15 +1102,127 @@ def _smtp_connect(account=None, cfg=None): port, timeout=EMAIL_SOCKET_TIMEOUT, ) - if cfg["smtp_starttls"]: - conn.starttls() if cfg["smtp_user"] and cfg["smtp_password"]: - conn.login(cfg["smtp_user"], cfg["smtp_password"]) + try: + conn.login(cfg["smtp_user"], cfg["smtp_password"]) + except Exception: + # A failed login otherwise orphans the connected socket; close it + # before propagating (SMTP has no shutdown(); close() = socket close). (#3174) + try: + conn.close() + except Exception: + pass + raise return conn +def _read_agent_email_confirm_setting() -> bool: + """True if the user wants agent send_email/reply_to_email calls to be + queued for manual approval instead of SMTPed immediately. Defaults to + True so a fresh install is safe — agents have been observed inventing + signatures and sending to real recipients without the user's review.""" + try: + from src.settings import get_setting + return bool(get_setting("agent_email_confirm", True)) + except Exception: + return True + + +def _stash_agent_draft(*, to, subject, body, in_reply_to=None, references=None, + cc=None, bcc=None, account=None) -> dict: + """Insert the composed email into scheduled_emails with status + 'agent_draft' and a far-future send_at so the scheduled-send poller + never picks it up. Returns the pending payload the model surfaces to + the user (and that the chat UI can render as an approval card).""" + try: + from src.constants import SCHEDULED_EMAILS_DB + except Exception: + return {"success": False, "error": "Pending-email storage unavailable"} + pending_id = uuid.uuid4().hex[:16] + far_future = "9999-12-31T00:00:00" + now = datetime.utcnow().isoformat() + try: + conn = sqlite3.connect(SCHEDULED_EMAILS_DB) + # Touch the schema in case the email-routes init hasn't run yet + # (MCP server can boot independently). + conn.execute(""" + CREATE TABLE IF NOT EXISTS scheduled_emails ( + id TEXT PRIMARY KEY, + to_addr TEXT NOT NULL, + cc TEXT, + bcc TEXT, + subject TEXT, + body TEXT NOT NULL, + in_reply_to TEXT, + references_hdr TEXT, + attachments TEXT, + send_at TEXT NOT NULL, + created_at TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending', + error TEXT, + owner TEXT DEFAULT '', + account_id TEXT, + odysseus_kind TEXT + ) + """) + conn.execute(""" + INSERT INTO scheduled_emails + (id, to_addr, cc, bcc, subject, body, in_reply_to, references_hdr, + attachments, send_at, created_at, status, account_id, odysseus_kind, owner) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'agent_draft', ?, ?, ?) + """, ( + pending_id, + to if isinstance(to, str) else ", ".join(to), + cc if isinstance(cc, str) else (", ".join(cc) if cc else None), + bcc if isinstance(bcc, str) else (", ".join(bcc) if bcc else None), + subject or "", + body or "", + in_reply_to or None, + references if isinstance(references, str) else (" ".join(references) if references else None), + "[]", + far_future, + now, + account or None, + "agent_draft", + _current_owner(), + )) + conn.commit() + conn.close() + except Exception as e: + return {"success": False, "error": f"Failed to stash draft: {e}"} + return { + "success": True, + "pending": True, + "pending_id": pending_id, + "to": to if isinstance(to, str) else ", ".join(to), + "subject": subject or "", + "body": body or "", + "message": ( + "✋ Draft staged for your approval — nothing has been sent yet.\n" + "Review the To/Subject/Body above. Reply 'send' to deliver, or " + "'cancel' to discard." + ), + } + + def _send_email(to, subject, body, in_reply_to=None, references=None, cc=None, bcc=None, account=None): - """Send an email via SMTP. Returns dict with status.""" + """Send an email via SMTP. Returns dict with status. + + When the `agent_email_confirm` setting is on (the default), the email + is NOT SMTPed — instead it lands in scheduled_emails as an + `agent_draft` row and the user reviews + approves it from the chat + UI. This closes the auto-send hole that let earlier models invent + signatures and ship them to real recipients without confirmation.""" + if _read_agent_email_confirm_setting(): + # Even confirmation-first sends must resolve the selected account now. + # Otherwise a caller could stage a pending draft against another + # owner's account selector before browser approval handles it. + cfg = _load_config(account) + return _stash_agent_draft( + to=to, subject=subject, body=body, + in_reply_to=in_reply_to, references=references, + cc=cc, bcc=bcc, account=cfg.get("account_id") or account, + ) send_account, cfg = _resolve_send_config(account) msg = EmailMessage() msg["From"] = _clean_header_value(cfg["from_address"]) @@ -809,7 +1262,7 @@ def _send_email(to, subject, body, in_reply_to=None, references=None, cc=None, b imap = _imap_connect(send_account) try: sent_folder = _detect_sent_folder(imap) - append_st, append_data = imap.append(sent_folder, "\\Seen", None, msg.as_bytes()) + append_st, append_data = imap.append(_q(sent_folder), "\\Seen", None, msg.as_bytes()) if append_st == "OK" and append_data: m = re.search(rb"APPENDUID\s+\d+\s+(\d+)", append_data[0] or b"") if m: @@ -833,11 +1286,188 @@ def _send_email(to, subject, body, in_reply_to=None, references=None, cc=None, b } -def _reply_to_email(uid, body, folder="INBOX", reply_all=False, account=None): - """Reply to an existing email by UID. Threads via In-Reply-To/References.""" +def _build_email_document_content( + to, + subject, + body, + *, + cc=None, + bcc=None, + in_reply_to=None, + references=None, + source_uid=None, + source_folder=None, +): + header_lines = [f"To: {to or ''}"] + if cc: + header_lines.append(f"Cc: {cc}") + if bcc: + header_lines.append(f"Bcc: {bcc}") + header_lines.append(f"Subject: {subject or ''}") + if in_reply_to: + header_lines.append(f"In-Reply-To: {in_reply_to}") + if references: + header_lines.append(f"References: {references}") + if source_uid: + header_lines.append(f"X-Source-UID: {source_uid}") + if source_folder: + header_lines.append(f"X-Source-Folder: {source_folder}") + return "\n".join(header_lines) + "\n---\n" + (body or "") + + +def _merge_email_reply_body(existing_content: str, reply_body: str) -> str: + """Preserve email headers and quoted chain while replacing the editable reply body.""" + if "\n---\n" not in (existing_content or ""): + return reply_body or "" + head, body = existing_content.split("\n---\n", 1) + quote_markers = ( + "---------- Previous message ----------", + "-----Original Message-----", + "----- Original Message -----", + ) + quote_index = -1 + for marker in quote_markers: + idx = body.find(marker) + if idx != -1 and (quote_index == -1 or idx < quote_index): + quote_index = idx + quote = body[quote_index:].strip() if quote_index != -1 else "" + merged_body = (reply_body or "").strip() + if quote: + merged_body = f"{merged_body}\n\n{quote}" if merged_body else quote + return f"{head}\n---\n{merged_body}" + + +def _create_email_draft_document( + *, + to, + subject, + body, + title=None, + cc=None, + bcc=None, + in_reply_to=None, + references=None, + source_uid=None, + source_folder=None, + account=None, + source_message_id=None, +): + """Create an Odysseus email compose document for user review. Does not send.""" + from core.database import SessionLocal, Document, DocumentVersion + try: + from src.event_bus import fire_event + except Exception: + fire_event = None + + cfg = _load_config(account) if account else _load_config(None) + content = _build_email_document_content( + to, + subject, + body, + cc=cc, + bcc=bcc, + in_reply_to=in_reply_to, + references=references, + source_uid=source_uid, + source_folder=source_folder, + ) + doc_id = str(uuid.uuid4()) + ver_id = str(uuid.uuid4()) + doc_title = (title or subject or "Email draft").strip() or "Email draft" + doc_owner = _current_owner() or _default_document_owner() + + db = SessionLocal() + try: + if source_uid and source_folder: + existing = ( + db.query(Document) + .filter(Document.is_active == True) + .filter(Document.language == "email") + .filter(Document.owner == doc_owner) + .filter(Document.source_email_uid == str(source_uid)) + .filter(Document.source_email_folder == source_folder) + .order_by(Document.updated_at.desc()) + .first() + ) + if existing and "\n---\n" in (existing.current_content or ""): + existing.current_content = _merge_email_reply_body(existing.current_content, body or "") + existing.version_count = (existing.version_count or 0) + 1 + ver = DocumentVersion( + id=ver_id, + document_id=existing.id, + version_number=existing.version_count, + content=existing.current_content, + summary="Updated by email MCP draft tool", + source="ai", + ) + db.add(ver) + db.commit() + if fire_event: + try: + fire_event("document_updated", doc_owner) + except Exception: + pass + return { + "draft": True, + "updated": True, + "doc_id": existing.id, + "title": existing.title, + "language": existing.language, + "account": cfg.get("account_name"), + "account_id": cfg.get("account_id"), + "to": to, + "subject": subject, + } + + doc = Document( + id=doc_id, + session_id=None, + title=doc_title, + language="email", + current_content=content, + version_count=1, + is_active=True, + owner=doc_owner, + source_email_uid=source_uid, + source_email_folder=source_folder, + source_email_account_id=cfg.get("account_id"), + source_email_message_id=source_message_id, + ) + ver = DocumentVersion( + id=ver_id, + document_id=doc_id, + version_number=1, + content=content, + summary="Created by email MCP draft tool", + source="ai", + ) + db.add(doc) + db.add(ver) + db.commit() + if fire_event: + try: + fire_event("document_created", doc_owner) + except Exception: + pass + return { + "draft": True, + "doc_id": doc_id, + "title": doc_title, + "language": "email", + "account": cfg.get("account_name"), + "account_id": cfg.get("account_id"), + "to": to, + "subject": subject, + } + finally: + db.close() + + +def _draft_reply_to_email(uid, body, folder="INBOX", reply_all=False, account=None, title=None): + """Create a threaded Odysseus reply draft document. Does not send.""" conn = _imap_connect(account) - conn.select(folder, readonly=True) - status, msg_data = conn.uid("FETCH", _b(uid), "(RFC822)") + conn.select(_q(folder), readonly=True) + status, msg_data = conn.uid("FETCH", _b(uid), "(BODY.PEEK[])") conn.logout() if status != "OK" or not msg_data or not msg_data[0]: return {"error": f"Failed to fetch email UID {uid}"} @@ -854,6 +1484,168 @@ def _reply_to_email(uid, body, folder="INBOX", reply_all=False, account=None): _, sender_addr = email.utils.parseaddr(sender) to_addrs = sender_addr + cc = None + if reply_all: + cc_addrs = [] + cfg = _load_config(account) + own_addrs = { + (cfg.get("imap_user") or "").strip().lower(), + (cfg.get("from_address") or "").strip().lower(), + } + for header_name in ("To", "Cc"): + for _, addr in email.utils.getaddresses([orig.get(header_name, "")]): + addr_l = (addr or "").strip().lower() + if addr and addr != sender_addr and addr_l not in own_addrs: + cc_addrs.append(addr) + if cc_addrs: + cc = ", ".join(dict.fromkeys(cc_addrs)) + + return _create_email_draft_document( + to=to_addrs, + subject=reply_subject, + body=body, + title=title or reply_subject, + cc=cc, + in_reply_to=orig_message_id, + references=new_references, + source_uid=uid, + source_folder=folder, + account=account, + source_message_id=orig_message_id, + ) + + +async def _ai_draft_reply_to_email(uid, folder="INBOX", reply_all=False, account=None, title=None): + """Generate a reply with Odysseus' AI-reply prompt/style, then create a compose doc.""" + read_result = _read_email(uid=uid, folder=folder, account=account) + if "error" in read_result: + return read_result + + to_addr = read_result.get("from_address") or email.utils.parseaddr(read_result.get("from") or "")[1] + subject = read_result.get("subject") or "" + reply_subject = subject if subject.lower().startswith("re:") else f"Re: {subject}" + original_body = read_result.get("body") or "" + message_id = read_result.get("message_id") or "" + + if not original_body.strip(): + return {"error": "No email body available for AI reply"} + + try: + from routes.email_helpers import ( + _EMAIL_REPLY_SYS_PROMPT_BASE, + _apply_email_style_mechanics, + _extract_reply, + _load_settings, + ) + from src.endpoint_resolver import ( + resolve_endpoint, + resolve_utility_fallback_candidates, + resolve_chat_fallback_candidates, + ) + from src.llm_core import llm_call_async_with_fallback + except Exception as exc: + return {"error": f"AI reply helpers unavailable: {exc}"} + + settings = _load_settings() + style = settings.get("email_writing_style", "") + system_prompt = _EMAIL_REPLY_SYS_PROMPT_BASE + if style: + system_prompt += f"\n\nWRITING STYLE TO MATCH:\n{style}" + + user_msg = ( + f"Recipient: {to_addr}\nSubject: {reply_subject}\n\n" + f"Original email and any current draft:\n{original_body[:6000]}\n\n" + "Draft a reply. Return only the reply body text." + ) + + candidates = [] + seen = set() + + def _add(url, model, headers): + key = (url or "", model or "") + if not url or not model or key in seen: + return + seen.add(key) + candidates.append((url, model, headers)) + + try: + _add(*resolve_endpoint("utility", owner=None)) + except Exception: + pass + try: + _add(*resolve_endpoint("default", owner=None)) + except Exception: + pass + try: + utility_fallbacks = resolve_utility_fallback_candidates(owner=None) or [] + except TypeError: + utility_fallbacks = resolve_utility_fallback_candidates() or [] + for cand in utility_fallbacks: + _add(*cand) + try: + chat_fallbacks = resolve_chat_fallback_candidates(owner=None) or [] + except TypeError: + chat_fallbacks = resolve_chat_fallback_candidates() or [] + for cand in chat_fallbacks: + _add(*cand) + + if not candidates: + return {"error": "No LLM endpoint configured for AI reply"} + + try: + raw_reply = await llm_call_async_with_fallback( + candidates, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_msg}, + ], + temperature=0.7, + max_tokens=1024, + timeout=60, + ) + except Exception as exc: + return {"error": f"AI reply generation failed: {exc}"} + + reply = _apply_email_style_mechanics(_extract_reply(raw_reply or "")) + if not reply: + return {"error": "AI reply generation returned an empty response"} + + return _draft_reply_to_email( + uid=uid, + body=reply, + folder=folder, + reply_all=reply_all, + account=account, + title=title or reply_subject, + ) + + +def _reply_to_email(uid, body, folder="INBOX", reply_all=False, account=None): + """Reply to an existing email by UID. Threads via In-Reply-To/References.""" + conn = None + try: + conn = _imap_connect(account) + conn.select(_q(folder), readonly=True) + status, msg_data = conn.uid("FETCH", _b(uid), "(BODY.PEEK[])") + finally: + if conn: + try: conn.logout() + except Exception: pass + if status != "OK" or not msg_data or not msg_data[0]: + return {"error": f"Failed to fetch email UID {uid}"} + raw = msg_data[0][1] + orig = email.message_from_bytes(raw) + + orig_subject = _decode_header(orig.get("Subject", "")) + reply_subject = orig_subject if orig_subject.lower().startswith("re:") else f"Re: {orig_subject}" + orig_message_id = orig.get("Message-ID", "") + orig_references = orig.get("References", "") + new_references = (orig_references + " " + orig_message_id).strip() if orig_references else orig_message_id + + sender = _decode_header(orig.get("From", "")) + _, sender_addr = email.utils.parseaddr(sender) + to_addrs = sender_addr + cc = None if reply_all: cc_addrs = [] @@ -878,7 +1670,7 @@ def _reply_to_email(uid, body, folder="INBOX", reply_all=False, account=None): def _set_flag(uid, folder, flag, add=True, account=None): """Add or remove an IMAP flag (e.g. \\Seen, \\Answered, \\Deleted).""" conn = _imap_connect(account) - conn.select(folder) + conn.select(_q(folder)) op = "+FLAGS" if add else "-FLAGS" try: status, data = conn.uid("STORE", _b(uid), op, flag) @@ -900,7 +1692,7 @@ def _bulk_set_flag(uids, folder, flag, add=True, account=None): conn = _imap_connect(account) touched = [] try: - conn.select(folder) + conn.select(_q(folder)) op = "+FLAGS" if add else "-FLAGS" msg_set = ",".join(str(u) for u in uids) try: @@ -927,7 +1719,7 @@ def _bulk_move(uids, source_folder, dest_folder, account=None, role: str = ""): conn = _imap_connect(account) moved = 0 try: - conn.select(source_folder) + conn.select(_q(source_folder)) dest_folder = _resolve_folder(conn, dest_folder, role or _folder_role_from_name(dest_folder)) msg_set = ",".join(str(u) for u in uids) try: @@ -938,10 +1730,11 @@ def _bulk_move(uids, source_folder, dest_folder, account=None, role: str = ""): if not existing: return 0 moved = len(existing) - status, _ = conn.uid("MOVE", _b(msg_set), dest_folder) + dest_arg = _q(dest_folder) + status, _ = conn.uid("MOVE", _b(msg_set), dest_arg) if status != "OK": # Fallback: UID copy + flag-delete + expunge - status, _ = conn.uid("COPY", _b(msg_set), dest_folder) + status, _ = conn.uid("COPY", _b(msg_set), dest_arg) if status != "OK": return 0 status, _ = conn.uid("STORE", _b(msg_set), "+FLAGS", "\\Deleted") @@ -958,7 +1751,7 @@ def _search_uids(folder="INBOX", criteria="UNSEEN", account=None): ALL, ANSWERED). Used to resolve selectors like all_unread → uids.""" conn = _imap_connect(account) try: - conn.select(folder, readonly=True) + conn.select(_q(folder), readonly=True) status, data = conn.uid("SEARCH", None, criteria) if status != "OK" or not data or not data[0]: return [] @@ -970,7 +1763,7 @@ def _search_uids(folder="INBOX", criteria="UNSEEN", account=None): def _move_message(uid, source_folder, dest_folder, account=None, role: str = ""): """Move a message between folders. Tries IMAP MOVE, falls back to copy+delete.""" conn = _imap_connect(account) - conn.select(source_folder) + conn.select(_q(source_folder)) try: dest_folder = _resolve_folder(conn, dest_folder, role or _folder_role_from_name(dest_folder)) try: @@ -980,11 +1773,12 @@ def _move_message(uid, source_folder, dest_folder, account=None, role: str = "") existing = _uid_fetch_rows(data) if status != "OK" or not existing: return False - status, _ = conn.uid("MOVE", _b(uid), dest_folder) + dest_arg = _q(dest_folder) + status, _ = conn.uid("MOVE", _b(uid), dest_arg) if status == "OK": return True # Fallback: UID copy + delete - status, _ = conn.uid("COPY", _b(uid), dest_folder) + status, _ = conn.uid("COPY", _b(uid), dest_arg) if status != "OK": return False status, _ = conn.uid("STORE", _b(uid), "+FLAGS", "\\Deleted") @@ -1013,16 +1807,21 @@ def _archive_email(uid, folder="INBOX", account=None): def _download_attachment(uid, index, folder="INBOX", account=None): """Extract a specific attachment to disk and return its local path.""" - conn = _imap_connect(account) - conn.select(folder, readonly=True) - status, msg_data = conn.uid("FETCH", _b(uid), "(RFC822)") - conn.logout() + conn = None + try: + conn = _imap_connect(account) + conn.select(_q(folder), readonly=True) + status, msg_data = conn.uid("FETCH", _b(uid), "(BODY.PEEK[])") + finally: + if conn: + try: conn.logout() + except Exception: pass if status != "OK": return {"error": f"Failed to fetch email UID {uid}"} raw = msg_data[0][1] msg = email.message_from_bytes(raw) - target_dir = DATA_DIR / "mail-attachments" / f"{folder}_{uid}" + target_dir = Path(MAIL_ATTACHMENTS_DIR) / f"{folder}_{uid}" filepath = _extract_attachment_to_disk(msg, index, target_dir) if not filepath: return {"error": f"Attachment index {index} not found"} @@ -1114,6 +1913,8 @@ async def list_tools() -> list[Tool]: name="send_email", description=( "Send a new email via SMTP. Provide recipient(s), subject, and body. " + "This sends immediately; for normal assistant-written email, prefer " + "draft_email so the user can review and send from Odysseus. " "For replying to an existing thread, use reply_to_email instead. " "Pass `account` to send from a non-default mailbox." ), @@ -1130,10 +1931,36 @@ async def list_tools() -> list[Tool]: "required": ["to", "subject", "body"], }, ), + Tool( + name="draft_email", + description=( + "Create a new Odysseus email compose draft document. This DOES NOT send. " + "Use this as the default way to write an email for the user: it opens " + "a reviewable email document with To/Cc/Bcc/Subject/body, and the user " + "can edit or press Send in Odysseus. " + f"{_writing_style_guidance()}" + ), + inputSchema={ + "type": "object", + "properties": { + "to": {"type": "string", "description": "Recipient email address(es), comma-separated"}, + "subject": {"type": "string", "description": "Email subject line"}, + "body": {"type": "string", "description": "Draft body"}, + "cc": {"type": "string", "description": "CC address(es), comma-separated (optional)"}, + "bcc": {"type": "string", "description": "BCC address(es), comma-separated (optional)"}, + "title": {"type": "string", "description": "Optional Odysseus document title"}, + **ACCOUNT_PROP, + }, + "required": ["to", "subject", "body"], + }, + ), Tool( name="reply_to_email", description=( - "Reply to an existing email by UID. Automatically threads the reply with " + "Reply to an existing email by UID. This sends immediately. Do NOT use " + "for normal 'write/draft a reply saying X' requests; use " + "draft_email_reply so the user can review and send from Odysseus. " + "Only use this when the user explicitly says to send now. Automatically threads the reply with " "In-Reply-To and References headers, prefixes 'Re:' on the subject, and " "uses the original sender as the recipient. Set reply_all=true to also CC " "the original To/Cc recipients. For follow-up 'reply ...' requests, use " @@ -1151,6 +1978,49 @@ async def list_tools() -> list[Tool]: "required": ["uid", "body"], }, ), + Tool( + name="draft_email_reply", + description=( + "Create an Odysseus email reply draft document for an existing email UID. " + "This DOES NOT send. It threads the draft with In-Reply-To/References, " + "prefills the recipient and subject, and stores source email metadata so " + "the user can review and send from the normal email composer. " + f"{_writing_style_guidance()}" + ), + inputSchema={ + "type": "object", + "properties": { + "uid": {"type": "string", "description": "Exact Email UID from list_emails/read_email; never invent UID 1"}, + "body": {"type": "string", "description": "Draft reply body text"}, + "folder": {"type": "string", "description": "IMAP folder (default: INBOX)", "default": "INBOX"}, + "reply_all": {"type": "boolean", "description": "Reply to all recipients (default: false)", "default": False}, + "title": {"type": "string", "description": "Optional Odysseus document title"}, + **ACCOUNT_PROP, + }, + "required": ["uid", "body"], + }, + ), + Tool( + name="ai_draft_email_reply", + description=( + "Generate an AI reply using Odysseus' existing AI Reply behavior, " + "including Settings > Email > Writing Style, then create an email " + "compose document for review. This DOES NOT send and does NOT save " + "to the mailbox Drafts folder. Use this when the user asks you to " + "write or draft a reply to an email without dictating the exact body." + ), + inputSchema={ + "type": "object", + "properties": { + "uid": {"type": "string", "description": "Exact Email UID from list_emails/read_email; never invent UID 1"}, + "folder": {"type": "string", "description": "IMAP folder (default: INBOX)", "default": "INBOX"}, + "reply_all": {"type": "boolean", "description": "Reply to all recipients (default: false)", "default": False}, + "title": {"type": "string", "description": "Optional Odysseus document title"}, + **ACCOUNT_PROP, + }, + "required": ["uid"], + }, + ), Tool( name="archive_email", description="Move an email out of the inbox into the Archive folder. Use after handling an email you want to keep but no longer need in the inbox.", @@ -1291,10 +2161,21 @@ async def list_tools() -> list[Tool]: @server.call_tool() async def call_tool(name: str, arguments: dict) -> list[TextContent]: + arguments = dict(arguments) if isinstance(arguments, dict) else {} + owner = str(arguments.pop(_MCP_OWNER_ARG, "") or "").strip() + owner_token = _CURRENT_OWNER.set(owner or None) try: + all_db_accounts = _read_accounts_from_db() + if _mcp_owner_required(all_db_accounts): + return [TextContent(type="text", text=_OWNER_SCOPE_ERROR)] + if name == "list_email_accounts": - rows = _list_accounts_raw() + rows = _filter_accounts_for_owner(all_db_accounts) + if not rows: + rows = _fixture_account_rows() if not rows: + if all_db_accounts and owner: + return [TextContent(type="text", text="No email accounts configured for this owner.")] return [TextContent(type="text", text="No email accounts configured. Legacy single-account mode active.")] lines = [f"Found {len(rows)} email account(s):\n"] for r in rows: @@ -1474,9 +2355,44 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: bcc=arguments.get("bcc"), account=acct, ) + if "error" in result: + return [TextContent(type="text", text=f"Error: {result['error']}")] + if result.get("pending"): + return [TextContent( + type="text", + text=( + f"Draft staged for approval (pending id: {result.get('pending_id')}). " + "Nothing has been sent yet. Review and approve it in Odysseus before delivery." + ), + )] acct_note = f" (from {result['account']})" if result.get("account") else "" return [TextContent(type="text", text=f"Sent email to {result['to']} with subject '{result['subject']}'{acct_note}.")] + elif name == "draft_email": + to = arguments.get("to") + subject = arguments.get("subject") + body = arguments.get("body") + if not to or not subject or body is None: + return [TextContent(type="text", text="Error: to, subject, and body are required")] + result = _create_email_draft_document( + to=to, + subject=subject, + body=body, + title=arguments.get("title"), + cc=arguments.get("cc"), + bcc=arguments.get("bcc"), + account=acct, + ) + acct_note = f" from {result['account']}" if result.get("account") else "" + return [TextContent( + type="text", + text=( + f"Created Odysseus email draft `{result['title']}` " + f"(document ID: {result['doc_id']}){acct_note}. " + "It has not been sent; open the document in Odysseus to review and send." + ), + )] + elif name == "reply_to_email": uid = arguments.get("uid") body = arguments.get("body") @@ -1498,6 +2414,54 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: pass return [TextContent(type="text", text=f"Replied to UID {uid}: '{result['subject']}' → {result['to']}")] + elif name == "draft_email_reply": + uid = arguments.get("uid") + body = arguments.get("body") + if not uid or body is None: + return [TextContent(type="text", text="Error: uid and body are required")] + result = _draft_reply_to_email( + uid=uid, + body=body, + folder=arguments.get("folder", "INBOX"), + reply_all=bool(arguments.get("reply_all", False)), + account=acct, + title=arguments.get("title"), + ) + if "error" in result: + return [TextContent(type="text", text=f"Error: {result['error']}")] + acct_note = f" from {result['account']}" if result.get("account") else "" + return [TextContent( + type="text", + text=( + f"Created Odysseus reply draft `{result['title']}` for UID {uid} " + f"(document ID: {result['doc_id']}){acct_note}. " + "It has not been sent; open the document in Odysseus to review and send." + ), + )] + + elif name == "ai_draft_email_reply": + uid = arguments.get("uid") + if not uid: + return [TextContent(type="text", text="Error: uid is required")] + result = await _ai_draft_reply_to_email( + uid=uid, + folder=arguments.get("folder", "INBOX"), + reply_all=bool(arguments.get("reply_all", False)), + account=acct, + title=arguments.get("title"), + ) + if "error" in result: + return [TextContent(type="text", text=f"Error: {result['error']}")] + acct_note = f" from {result['account']}" if result.get("account") else "" + return [TextContent( + type="text", + text=( + f"Generated AI reply and created Odysseus compose draft " + f"`{result['title']}` for UID {uid} (document ID: {result['doc_id']}){acct_note}. " + "It has not been sent; open the document in Odysseus to review and send." + ), + )] + elif name == "archive_email": uid = arguments.get("uid") if not uid: @@ -1576,6 +2540,8 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: except Exception as e: return [TextContent(type="text", text=f"Error: {e}")] + finally: + _CURRENT_OWNER.reset(owner_token) # ── Main ── diff --git a/mcp_servers/image_gen_server.py b/mcp_servers/image_gen_server.py index 872ccd681..6cb77f780 100644 --- a/mcp_servers/image_gen_server.py +++ b/mcp_servers/image_gen_server.py @@ -16,6 +16,8 @@ sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) +from src.constants import GENERATED_IMAGES_DIR + server = Server("image_gen") @@ -71,7 +73,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: if not model_spec: for candidate in ("gpt-image-1.5", "gpt-image-1", "dall-e-3"): try: - _resolve_model(candidate) + await asyncio.to_thread(_resolve_model, candidate) model_spec = candidate break except ValueError: @@ -79,7 +81,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: if not model_spec: return [TextContent(type="text", text="Error: No image model found. Configure one in Admin.")] - url, model_id, headers = _resolve_model(model_spec) + url, model_id, headers = await asyncio.to_thread(_resolve_model, model_spec) is_gpt_image = "gpt-image" in model_id.lower() base_url = url.replace("/chat/completions", "").replace("/v1/messages", "").rstrip("/") @@ -115,14 +117,18 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: img = images[0] image_url = None + # Prefix the instance's public base URL (existing app_public_url setting) so the + # link is fully-qualified and clickable when the model echoes it. Empty = relative + # same-origin path (unchanged default). + _pub_base = (get_setting("app_public_url", "") or "").rstrip("/") if img.get("b64_json"): - img_dir = Path("data/generated_images") + img_dir = Path(GENERATED_IMAGES_DIR) img_dir.mkdir(parents=True, exist_ok=True) filename = f"{uuid.uuid4().hex[:12]}.png" img_path = img_dir / filename img_path.write_bytes(base64.b64decode(img["b64_json"])) - image_url = f"/api/generated-image/{filename}" + image_url = f"{_pub_base}/api/generated-image/{filename}" # Save to gallery try: @@ -146,7 +152,13 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: else: return [TextContent(type="text", text="Error: Unexpected image API response format")] - result = f"Generated image for: {prompt[:100]}\nimage_url: {image_url}\nmodel: {model_id}\nsize: {size}" + # "Direct link:" rather than an "image_url:" label — small models copied the + # label token ("image_url") into the link href, producing a broken link. + result = ( + f"Generated image for: {prompt[:100]}\n" + f"Direct link: {image_url}\n" + f"model: {model_id}\nsize: {size}" + ) return [TextContent(type="text", text=result)] except httpx.TimeoutException: diff --git a/mcp_servers/memory_server.py b/mcp_servers/memory_server.py index c2812e1c0..fafbcfc2b 100644 --- a/mcp_servers/memory_server.py +++ b/mcp_servers/memory_server.py @@ -6,6 +6,7 @@ """ import asyncio +import os import sys import time from pathlib import Path @@ -23,6 +24,55 @@ _memory_vector = None _initialized = False +_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_MEMORY_OWNER", "ODYSSEUS_MEMORY_OWNER") +_OWNER_SCOPE_ERROR = ( + "Error: Memory MCP owner is not configured for an owner-scoped memory store. " + "Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool." +) + + +def _configured_owner() -> str | None: + for key in _OWNER_ENV_KEYS: + owner = os.environ.get(key, "").strip() + if owner: + return owner + return None + + +def _entry_owner(entry: dict) -> str | None: + owner = entry.get("owner") + if owner is None: + return None + owner_text = str(owner).strip() + return owner_text or None + + +def _owner_scoped_store(entries: list[dict]) -> bool: + return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict)) + + +def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]: + """Return configured owner, all entries, visible entries, and optional error.""" + entries = _memory_manager.load_all() + owner = _configured_owner() + if owner is None and _owner_scoped_store(entries): + return None, entries, [], _OWNER_SCOPE_ERROR + if owner is None: + visible = [ + entry for entry in entries + if isinstance(entry, dict) and _entry_owner(entry) is None + ] + else: + visible = [ + entry for entry in entries + if isinstance(entry, dict) and _entry_owner(entry) == owner + ] + return owner, entries, visible, None + + +def _text_result(text: str) -> list[TextContent]: + return [TextContent(type="text", text=text)] + def _ensure_init(): """Lazy-init memory managers on first use.""" @@ -75,43 +125,46 @@ async def list_tools() -> list[Tool]: @server.call_tool() async def call_tool(name: str, arguments: dict) -> list[TextContent]: if name != "manage_memory": - return [TextContent(type="text", text=f"Unknown tool: {name}")] + return _text_result(f"Unknown tool: {name}") _ensure_init() if not _memory_manager: - return [TextContent(type="text", text="Error: Memory manager not available")] + return _text_result("Error: Memory manager not available") action = arguments.get("action", "") if action == "list": category_filter = arguments.get("category", "") - memories = _memory_manager.load() + _owner, _all_memories, memories, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) if category_filter: memories = [m for m in memories if m.get("category", "").lower() == category_filter.lower()] if not memories: msg = "No memories found" if category_filter: msg += f" in category '{category_filter}'" - return [TextContent(type="text", text=msg + ".")] + return _text_result(msg + ".") + lines = [f"Found {len(memories)} memory entries:\n"] - for m in memories[:100]: + for m in memories: cat = m.get("category", "fact") mid = m.get("id", "?")[:8] text = m.get("text", "") if len(text) > 150: text = text[:150] + "..." lines.append(f"- [{cat}] `{mid}` — {text}") - if len(memories) > 100: - lines.append(f"... and {len(memories) - 100} more") - return [TextContent(type="text", text="\n".join(lines))] + return _text_result("\n".join(lines)) elif action == "add": text = arguments.get("text", "") category = arguments.get("category", "fact") if not text: - return [TextContent(type="text", text="Error: Memory text cannot be empty")] - entry = _memory_manager.add_entry(text, source="ai_agent", category=category) - memories = _memory_manager.load_all() + return _text_result("Error: Memory text cannot be empty") + owner, memories, _visible, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) + entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner) memories.append(entry) _memory_manager.save(memories) if _memory_vector and _memory_vector.healthy: @@ -119,25 +172,28 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: _memory_vector.add(entry["id"], text) except Exception: pass - return [TextContent(type="text", text=f"Memory added: [{category}] {text} (id: {entry['id'][:8]})")] + return _text_result(f"Memory added: [{category}] {text} (id: {entry['id'][:8]})") elif action == "edit": memory_id = arguments.get("memory_id", "") new_text = arguments.get("text", "") if not memory_id or not new_text: - return [TextContent(type="text", text="Error: edit needs memory_id and text")] - memories = _memory_manager.load_all() - found = False + return _text_result("Error: edit needs memory_id and text") + _owner, memories, visible, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) full_id = None - for m in memories: + for m in visible: if m.get("id", "").startswith(memory_id): + full_id = m["id"] + break + if not full_id: + return _text_result(f"Error: Memory '{memory_id}' not found") + for m in memories: + if m.get("id") == full_id: m["text"] = new_text m["timestamp"] = int(time.time()) - found = True - full_id = m["id"] break - if not found: - return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")] _memory_manager.save(memories) if _memory_vector and _memory_vector.healthy and full_id: try: @@ -145,26 +201,27 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: _memory_vector.add(full_id, new_text) except Exception: pass - return [TextContent(type="text", text=f"Memory updated: {new_text}")] + return _text_result(f"Memory updated: {new_text}") elif action == "delete": memory_id = arguments.get("memory_id", "") if not memory_id: - return [TextContent(type="text", text="Error: delete needs memory_id")] - memories = _memory_manager.load_all() + return _text_result("Error: delete needs memory_id") + _owner, memories, visible, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) full_id = None deleted_text = "" deleted_category = "" - for m in memories: + for m in visible: if m.get("id", "").startswith(memory_id): full_id = m["id"] deleted_text = m.get("text", "") deleted_category = m.get("category", "") break - original_len = len(memories) - memories = [m for m in memories if not m.get("id", "").startswith(memory_id)] - if len(memories) == original_len: - return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")] + if not full_id: + return _text_result(f"Error: Memory '{memory_id}' not found") + memories = [m for m in memories if m.get("id") != full_id] _memory_manager.save(memories) if _memory_vector and _memory_vector.healthy and full_id: try: @@ -173,30 +230,32 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: pass cat = f"[{deleted_category}] " if deleted_category else "" snippet = deleted_text if len(deleted_text) <= 120 else deleted_text[:117] + "..." - return [TextContent(type="text", text=f"Memory deleted: {cat}{snippet} (id: {memory_id})")] + return _text_result(f"Memory deleted: {cat}{snippet} (id: {memory_id})") elif action == "search": query = arguments.get("text", "") if not query: - return [TextContent(type="text", text="Error: search needs text (query)")] - memories = _memory_manager.load() + return _text_result("Error: search needs text (query)") + _owner, _all_memories, memories, scope_error = _scope_entries() + if scope_error: + return _text_result(scope_error) if hasattr(_memory_manager, 'get_relevant_memories'): results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20) else: query_lower = query.lower() results = [m for m in memories if query_lower in m.get("text", "").lower()][:20] if not results: - return [TextContent(type="text", text=f"No memories found matching '{query}'.")] + return _text_result(f"No memories found matching '{query}'.") lines = [f"Found {len(results)} matching memories:\n"] for m in results: cat = m.get("category", "fact") mid = m.get("id", "?")[:8] text = m.get("text", "") lines.append(f"- [{cat}] `{mid}` — {text}") - return [TextContent(type="text", text="\n".join(lines))] + return _text_result("\n".join(lines)) else: - return [TextContent(type="text", text=f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search")] + return _text_result(f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search") async def run(): diff --git a/mcp_servers/rag_server.py b/mcp_servers/rag_server.py index 2d50b4b4f..71aa1b60b 100644 --- a/mcp_servers/rag_server.py +++ b/mcp_servers/rag_server.py @@ -101,10 +101,13 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: return [TextContent(type="text", text=f"Error: {e}")] elif action == "add_directory": - directory = arguments.get("directory", "").strip() + _dir = arguments.get("directory") + directory = _dir.strip() if isinstance(_dir, str) else "" if not directory: return [TextContent(type="text", text="Error: add_directory needs a directory path")] - directory = os.path.expanduser(directory) + # Store an absolute path so indexed `source` metadata is absolute and + # remove_directory (which abspath-normalizes) can match it later (#1660). + directory = os.path.abspath(os.path.expanduser(directory)) if not os.path.isdir(directory): return [TextContent(type="text", text=f"Error: Directory not found: {directory}")] if not _rag_manager: @@ -112,14 +115,27 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: try: result = _rag_manager.index_personal_documents(directory) indexed = result.get("indexed_count", 0) if isinstance(result, dict) else 0 + # Record the directory so `list` and `remove_directory` can see it. + # Indexing was just done above, so pass index=False to avoid a second + # (ownerless) pass. Without this the directory was indexed but never + # tracked in indexed_directories, so it was invisible/unremovable. + if _personal_docs_manager and hasattr(_personal_docs_manager, "add_directory"): + try: + _personal_docs_manager.add_directory(directory, index=False) + except Exception: + pass return [TextContent(type="text", text=f"Directory '{directory}' added to RAG index ({indexed} chunks indexed)")] except Exception as e: return [TextContent(type="text", text=f"Error: Failed to index directory: {e}")] elif action == "remove_directory": - directory = arguments.get("directory", "").strip() + _dir = arguments.get("directory") + directory = _dir.strip() if isinstance(_dir, str) else "" if not directory: return [TextContent(type="text", text="Error: remove_directory needs a directory path")] + # Expand ~ to match add_directory, which indexes the expanded path. + # Without this, removing "~/docs" never matches the stored absolute path. + directory = os.path.expanduser(directory) if not _personal_docs_manager: return [TextContent(type="text", text="Error: Personal docs manager not available")] try: diff --git a/odysseus-ui.service b/odysseus-ui.service index fea436398..835c8cc5a 100644 --- a/odysseus-ui.service +++ b/odysseus-ui.service @@ -9,7 +9,7 @@ Type=simple # CHANGE THESE to match your user and install path: User=YOURUSER WorkingDirectory=/home/YOURUSER/odysseus-ui -ExecStart=/home/YOURUSER/odysseus-ui/venv/bin/uvicorn app:app --port 8000 --host 0.0.0.0 +ExecStart=/home/YOURUSER/odysseus-ui/venv/bin/uvicorn app:app --port 7000 --host 0.0.0.0 Restart=always RestartSec=3 EnvironmentFile=-/home/YOURUSER/odysseus-ui/.env diff --git a/package-lock.json b/package-lock.json index 80eac7ebf..eac6229e7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,93 +1,22 @@ { - "name": "odysseus-ui", + "name": "odysseus", "lockfileVersion": 3, "requires": true, "packages": { "": { - "dependencies": { - "@anthropic-ai/sdk": "^0.98.0" - }, "devDependencies": { - "@antithesishq/bombadil": "^0.3.2" - } - }, - "node_modules/@anthropic-ai/sdk": { - "version": "0.98.0", - "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.98.0.tgz", - "integrity": "sha512-N7aXtCvC5g6T1Y4V29lJjceu/zTkVkIZF0jdBvagr0TRFHuKeImffalGWEfqZKrvjH+IQbzJWw6TmSmUzrlMgg==", - "license": "MIT", - "dependencies": { - "json-schema-to-ts": "^3.1.1", - "standardwebhooks": "^1.0.0" - }, - "bin": { - "anthropic-ai-sdk": "bin/cli" - }, - "peerDependencies": { - "zod": "^3.25.0 || ^4.0.0" - }, - "peerDependenciesMeta": { - "zod": { - "optional": true - } + "@antithesishq/bombadil": "^0.6.1" } }, "node_modules/@antithesishq/bombadil": { - "version": "0.3.2", - "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.3.2.tgz", - "integrity": "sha512-ATy1w9ZY5gbny1H8DFc7rxZitT7DLLLFDiGcRZe+8TQiUrV5tLO+IJGOVNNLp3RpCqjZqSsxGiKoQsx31ipV1g==", + "version": "0.6.1", + "resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.6.1.tgz", + "integrity": "sha512-d1iufG3MI7gSMSiSmMeNdcMW+qR0yQXL2zdkVynC3n3DYgFJYlYXKUQzygmqU12m4RWlR5iOdQU1hsx5UT6+IA==", "dev": true, - "license": "MIT" - }, - "node_modules/@babel/runtime": { - "version": "7.29.7", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", - "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", "license": "MIT", - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@stablelib/base64": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", - "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", - "license": "MIT" - }, - "node_modules/fast-sha256": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", - "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", - "license": "Unlicense" - }, - "node_modules/json-schema-to-ts": { - "version": "3.1.1", - "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", - "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", - "license": "MIT", - "dependencies": { - "@babel/runtime": "^7.18.3", - "ts-algebra": "^2.0.0" - }, - "engines": { - "node": ">=16" - } - }, - "node_modules/standardwebhooks": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", - "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", - "license": "MIT", - "dependencies": { - "@stablelib/base64": "^1.0.0", - "fast-sha256": "^1.3.0" + "bin": { + "bombadil": "bin/bombadil.js" } - }, - "node_modules/ts-algebra": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", - "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", - "license": "MIT" } } } diff --git a/package.json b/package.json index c14f9abbb..0236252de 100644 --- a/package.json +++ b/package.json @@ -1,8 +1,9 @@ { - "devDependencies": { - "@antithesishq/bombadil": "^0.3.2" + "repository": { + "type": "git", + "url": "https://github.com/odysseus-dev/odysseus.git" }, - "dependencies": { - "@anthropic-ai/sdk": "^0.98.0" + "devDependencies": { + "@antithesishq/bombadil": "^0.6.1" } } diff --git a/pyproject.toml b/pyproject.toml index 116b1376c..da00ee259 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,22 @@ [tool.pytest.ini_options] testpaths = ["tests"] asyncio_mode = "auto" +# Test-taxonomy markers added at collection time by tests/conftest.py. The +# stable area_* markers are declared here; the dynamic sub_ +# markers are registered before collection by pytest_configure in +# tests/conftest.py, so unknown-mark warnings still flag genuine typos outside +# the taxonomy. See tests/_taxonomy.py and tests/README.md. +markers = [ + "area_security: tests covering auth, owner-scope, SSRF, XSS, confinement, redaction", + "area_routes: tests covering HTTP route / API behavior", + "area_services: tests covering service-layer behavior (llm, cookbook, email, calendar, ...)", + "area_cli: tests covering CLI / script behavior", + "area_js: JavaScript / Node-backed tests", + "area_helpers: self-tests for the shared test helpers in tests/helpers/", + "area_unit: pure parser / utility tests that do not clearly belong elsewhere", + "area_uncategorized: tests not yet matched by the taxonomy (fallback)", + # Fast-lane marker (issue #3443). Opt-in and orthogonal to the area_*/sub_* + # taxonomy. The fast lane runs `not slow`; mark a test slow only with + # duration evidence (see tests/run_focus.py --durations and tests/README.md). + "slow: opt-in marker for known-slow tests; excluded by the fast lane (not slow)", +] diff --git a/requirements-optional.txt b/requirements-optional.txt index 72d9f7e69..ab21e81ee 100644 --- a/requirements-optional.txt +++ b/requirements-optional.txt @@ -4,10 +4,18 @@ # Note: chromadb-client + fastembed moved to requirements.txt — RAG, semantic # memory, and tool selection are core paths, so they ship by default now. +# Local speech-to-text (microphone -> text) via faster-whisper, for the +# "local" STT provider. Runs on CPU out of the box (CTranslate2 backend, no +# torch needed). Install if you want to dictate/transcribe with the mic +# without sending audio to an external endpoint. +# Optional extra: install `torch` too if you have a CUDA GPU and want +# GPU-accelerated transcription — it's auto-detected, CPU is used otherwise. +faster-whisper + # DuckDuckGo as a search provider option. # Install if you want DDG in the search-provider dropdown. # Alternatives: SearXNG, Brave, Tavily, Serper, Google PSE. -duckduckgo-search +ddgs # PDF form-filling feature (fillable AcroForm detection, field extraction, # value/annotation/signature stamping, page rendering for the form overlay). @@ -15,3 +23,14 @@ duckduckgo-search # network-served app — see ACKNOWLEDGMENTS.md. The MIT core (PDF *text* # extraction via pypdf) works without it; this only unlocks form-filling. PyMuPDF + +# Office / EPUB document text extraction (chat attachments + the personal-docs +# RAG index). markitdown (MIT, Microsoft) converts .docx/.xlsx/.pptx/.xls/.epub +# to Markdown — more token-efficient and model-legible than a raw dump. Optional +# and lazy-imported via src/markitdown_runtime.py; without it those formats fall +# back to a friendly "install to extract" banner and the core stays pure-MIT. +# Extras pull mammoth/lxml/python-pptx/pandas/openpyxl/xlrd; the base also pulls +# magika (onnxruntime), already a core dep via fastembed. We avoid the +# [all]/Azure/audio extras (cloud + heavy). Pinned to a release >30 days old per +# the dependency-age discussion in issue #485. +markitdown[docx,pptx,xlsx,xls]==0.1.6 diff --git a/requirements.txt b/requirements.txt index 1bf1e9bb9..be5f5d450 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,9 @@ uvicorn python-multipart python-dotenv httpx -pydantic -pydantic-settings +httpcore>=1.0,<2.0 +pydantic>=2.13.4 +pydantic-settings>=2.14.1 SQLAlchemy pypdf beautifulsoup4 @@ -21,8 +22,16 @@ youtube-transcript-api # Markdown rendering for research reports (src/visual_report.py). # Imported at module-top so it's a hard core dep, not optional. markdown +# HTML sanitizer for rendered research reports (src/visual_report.py). Report +# content is untrusted (LLM output over crawled pages) and report pages run +# under a relaxed CSP, so the rendered HTML is allowlist-sanitized. +nh3 # Calendar .ics import/export (routes/calendar_routes.py). icalendar +# Recurrence rule expansion for calendar events (routes/calendar_routes.py). +# Imported directly as dateutil.rrule — make it explicit even though caldav +# pulls it in transitively. +python-dateutil # CalDAV sync (src/caldav_sync.py). Handles PROPFIND discovery + REPORT # fetch across Radicale, Nextcloud, Apple, Fastmail; we'd be reinventing # the protocol without it. @@ -35,3 +44,7 @@ qrcode[pil] croniter pytest pytest-asyncio +# starlette.testclient prefers httpx2 since Starlette 1.2.0 and warns on every +# TestClient import when only classic httpx is present. Runtime code keeps +# using `httpx` above; this is test-client only. +httpx2 diff --git a/routes/_validators.py b/routes/_validators.py new file mode 100644 index 000000000..aa4cf00cc --- /dev/null +++ b/routes/_validators.py @@ -0,0 +1,31 @@ +import re + +from fastapi import HTTPException + + +_REMOTE_HOST_RE = re.compile( + r"^(?:[A-Za-z0-9][A-Za-z0-9._-]*@)?[A-Za-z0-9][A-Za-z0-9._-]*$" +) +_SSH_PORT_RE = re.compile(r"^\d{1,5}$") + + +def validate_remote_host(v: str | None) -> str | None: + if v is None or v == "": + return None + if not _REMOTE_HOST_RE.match(v): + raise HTTPException( + 400, + "Invalid remote_host — must be host or user@host, no SSH option syntax", + ) + return v + + +def validate_ssh_port(v: str | None) -> str | None: + if v is None or v == "": + return None + if not _SSH_PORT_RE.fullmatch(str(v)): + raise HTTPException(400, "Invalid ssh_port") + port = int(v) + if port < 1 or port > 65535: + raise HTTPException(400, "Invalid ssh_port") + return str(port) diff --git a/routes/admin_wipe/__init__.py b/routes/admin_wipe/__init__.py new file mode 100644 index 000000000..9d5fa1a52 --- /dev/null +++ b/routes/admin_wipe/__init__.py @@ -0,0 +1,5 @@ +"""Admin wipe route domain package (slice 2h, #4082/#4071). + +Contains admin_wipe_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/admin_wipe_routes.py re-exports from here. +""" diff --git a/routes/admin_wipe/admin_wipe_routes.py b/routes/admin_wipe/admin_wipe_routes.py new file mode 100644 index 000000000..212e2a768 --- /dev/null +++ b/routes/admin_wipe/admin_wipe_routes.py @@ -0,0 +1,176 @@ +"""Admin Danger Zone — per-category wipes. + +Each endpoint is admin-only and truncates exactly one domain so the +user can selectively reset memory / skills / notes / etc. without +nuking everything. The catch-all `chats` endpoint mirrors the +existing /api/sessions/all so the Danger Zone speaks one URL pattern. + +URL shape: DELETE /api/admin/wipe/{kind} +Kinds: chats, memory, skills, notes, tasks, documents, gallery, calendar. +""" + +import json +import logging +import os +import shutil +from fastapi import APIRouter, HTTPException, Request + +from core.middleware import require_admin +from core.database import ( + SessionLocal, + Session as DbSession, + ChatMessage as DbChatMessage, + Memory, + Note, + ScheduledTask, + TaskRun, + Document, + DocumentVersion, + GalleryImage, + GalleryAlbum, + CalendarEvent, + CalendarCal, +) +from src.constants import DATA_DIR, SKILLS_DIR, SKILLS_FILE, GALLERY_DIR, GALLERY_UPLOADS_DIR + +logger = logging.getLogger(__name__) + + +def _wipe_memory_files(): + """Blank memory.json + drop the per-owner tidy-state sidecar so the + next audit doesn't try to diff against gone memories.""" + for name in ("memory.json", "memory_tidy_state.json"): + p = os.path.join(DATA_DIR, name) + if not os.path.exists(p): + continue + try: + if name == "memory.json": + with open(p, "w", encoding="utf-8") as f: + json.dump([], f) + else: + os.remove(p) + except OSError as e: + logger.warning(f"Could not reset {name}: {e}") + + +def _rmtree_quiet(path: str): + """rmtree that doesn't crash if the path doesn't exist.""" + if os.path.isdir(path): + try: + shutil.rmtree(path) + except OSError as e: + logger.warning(f"Could not remove {path}: {e}") + + +def setup_admin_wipe_routes(session_manager): + """The session_manager is passed in so we can also clear its + in-memory cache when wiping chats — without it the DB is empty + but the next /api/sessions returns stale entries.""" + router = APIRouter(prefix="/api/admin") + + @router.delete("/wipe/{kind}") + def wipe(kind: str, request: Request): + require_admin(request) + kind = (kind or "").strip().lower() + + db = SessionLocal() + try: + if kind == "chats": + count = db.query(DbSession).count() + db.query(DbChatMessage).delete() + db.query(DbSession).delete() + db.commit() + try: + session_manager.sessions.clear() + except Exception: + pass + return {"status": "deleted", "kind": kind, "count": count} + + if kind == "memory": + count = db.query(Memory).count() + db.query(Memory).delete() + db.commit() + _wipe_memory_files() + # Drop the vector store too so semantic search doesn't + # return ghosts. Lazy import — chromadb may not be + # initialised in every deployment. + try: + from src.memory_vector import get_memory_vector_store + mv = get_memory_vector_store() + if mv and hasattr(mv, "clear"): + mv.clear() + except Exception as e: + logger.info(f"Memory vector clear skipped: {e}") + return {"status": "deleted", "kind": kind, "count": count} + + if kind == "skills": + # Skills live as SKILL.md files under data/skills/. Drop + # the entire directory; the SkillsManager re-creates the + # tree on next write. + skills_dir = SKILLS_DIR + count = 0 + if os.path.isdir(skills_dir): + # Count SKILL.md files for the response — quick walk. + for _, _, files in os.walk(skills_dir): + count += sum(1 for f in files if f == "SKILL.md") + _rmtree_quiet(skills_dir) + # Legacy fallback file + legacy = SKILLS_FILE + if os.path.exists(legacy): + try: + os.remove(legacy) + except OSError: + pass + return {"status": "deleted", "kind": kind, "count": count} + + if kind == "notes": + count = db.query(Note).count() + db.query(Note).delete() + db.commit() + return {"status": "deleted", "kind": kind, "count": count} + + if kind == "tasks": + # TaskRun rows reference tasks via FK — clear them first. + db.query(TaskRun).delete() + count = db.query(ScheduledTask).count() + db.query(ScheduledTask).delete() + db.commit() + return {"status": "deleted", "kind": kind, "count": count} + + if kind == "documents": + # DocumentVersion FKs Document — clear children first. + db.query(DocumentVersion).delete() + count = db.query(Document).count() + db.query(Document).delete() + db.commit() + return {"status": "deleted", "kind": kind, "count": count} + + if kind == "gallery": + count = db.query(GalleryImage).count() + db.query(GalleryAlbum).count() + db.query(GalleryImage).delete() + db.query(GalleryAlbum).delete() + db.commit() + # Also drop the upload dir so disk doesn't keep orphans. + _rmtree_quiet(GALLERY_DIR) + _rmtree_quiet(GALLERY_UPLOADS_DIR) + return {"status": "deleted", "kind": kind, "count": count} + + if kind == "calendar": + # Events FK calendars — clear children first, then both. + db.query(CalendarEvent).delete() + count = db.query(CalendarCal).count() + db.query(CalendarCal).delete() + db.commit() + return {"status": "deleted", "kind": kind, "count": count} + + raise HTTPException(400, f"Unknown wipe kind: {kind!r}") + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.exception(f"Wipe {kind} failed") + raise HTTPException(500, f"Wipe {kind} failed: {e}") + finally: + db.close() + + return router diff --git a/routes/admin_wipe_routes.py b/routes/admin_wipe_routes.py index 89d8ed0ea..a57c72df6 100644 --- a/routes/admin_wipe_routes.py +++ b/routes/admin_wipe_routes.py @@ -1,174 +1,17 @@ -"""Admin Danger Zone — per-category wipes. - -Each endpoint is admin-only and truncates exactly one domain so the -user can selectively reset memory / skills / notes / etc. without -nuking everything. The catch-all `chats` endpoint mirrors the -existing /api/sessions/all so the Danger Zone speaks one URL pattern. - -URL shape: DELETE /api/admin/wipe/{kind} -Kinds: chats, memory, skills, notes, tasks, documents, gallery, calendar. +"""Backward-compat shim — canonical location is routes/admin_wipe/admin_wipe_routes.py. + +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.admin_wipe_routes``, ``from routes.admin_wipe_routes +import X``, ``importlib.import_module("routes.admin_wipe_routes")``, and the +``import ... as admin_wipe_routes`` + ``monkeypatch.setattr(admin_wipe_routes, +"SessionLocal", ...)`` / ``"require_admin"`` pattern used by +test_admin_wipe_gallery.py all operate on the *same* object the application +actually uses. Keeps existing import paths working after slice 2h +(#4082/#4071). """ -import json -import logging -import os -import shutil -from fastapi import APIRouter, HTTPException, Request - -from core.middleware import require_admin -from core.database import ( - SessionLocal, - Session as DbSession, - ChatMessage as DbChatMessage, - Memory, - Note, - ScheduledTask, - TaskRun, - Document, - DocumentVersion, - GalleryImage, - CalendarEvent, - CalendarCal, -) -from src.constants import DATA_DIR - -logger = logging.getLogger(__name__) - - -def _wipe_memory_files(): - """Blank memory.json + drop the per-owner tidy-state sidecar so the - next audit doesn't try to diff against gone memories.""" - for name in ("memory.json", "memory_tidy_state.json"): - p = os.path.join(DATA_DIR, name) - if not os.path.exists(p): - continue - try: - if name == "memory.json": - with open(p, "w") as f: - json.dump([], f) - else: - os.remove(p) - except OSError as e: - logger.warning(f"Could not reset {name}: {e}") - - -def _rmtree_quiet(path: str): - """rmtree that doesn't crash if the path doesn't exist.""" - if os.path.isdir(path): - try: - shutil.rmtree(path) - except OSError as e: - logger.warning(f"Could not remove {path}: {e}") - - -def setup_admin_wipe_routes(session_manager): - """The session_manager is passed in so we can also clear its - in-memory cache when wiping chats — without it the DB is empty - but the next /api/sessions returns stale entries.""" - router = APIRouter(prefix="/api/admin") - - @router.delete("/wipe/{kind}") - def wipe(kind: str, request: Request): - require_admin(request) - kind = (kind or "").strip().lower() - - db = SessionLocal() - try: - if kind == "chats": - count = db.query(DbSession).count() - db.query(DbChatMessage).delete() - db.query(DbSession).delete() - db.commit() - try: - session_manager.sessions.clear() - except Exception: - pass - return {"status": "deleted", "kind": kind, "count": count} - - if kind == "memory": - count = db.query(Memory).count() - db.query(Memory).delete() - db.commit() - _wipe_memory_files() - # Drop the vector store too so semantic search doesn't - # return ghosts. Lazy import — chromadb may not be - # initialised in every deployment. - try: - from src.memory_vector import get_memory_vector_store - mv = get_memory_vector_store() - if mv and hasattr(mv, "clear"): - mv.clear() - except Exception as e: - logger.info(f"Memory vector clear skipped: {e}") - return {"status": "deleted", "kind": kind, "count": count} - - if kind == "skills": - # Skills live as SKILL.md files under data/skills/. Drop - # the entire directory; the SkillsManager re-creates the - # tree on next write. - skills_dir = os.path.join(DATA_DIR, "skills") - count = 0 - if os.path.isdir(skills_dir): - # Count SKILL.md files for the response — quick walk. - for _, _, files in os.walk(skills_dir): - count += sum(1 for f in files if f == "SKILL.md") - _rmtree_quiet(skills_dir) - # Legacy fallback file - legacy = os.path.join(DATA_DIR, "skills.json") - if os.path.exists(legacy): - try: - os.remove(legacy) - except OSError: - pass - return {"status": "deleted", "kind": kind, "count": count} - - if kind == "notes": - count = db.query(Note).count() - db.query(Note).delete() - db.commit() - return {"status": "deleted", "kind": kind, "count": count} - - if kind == "tasks": - # TaskRun rows reference tasks via FK — clear them first. - db.query(TaskRun).delete() - count = db.query(ScheduledTask).count() - db.query(ScheduledTask).delete() - db.commit() - return {"status": "deleted", "kind": kind, "count": count} - - if kind == "documents": - # DocumentVersion FKs Document — clear children first. - db.query(DocumentVersion).delete() - count = db.query(Document).count() - db.query(Document).delete() - db.commit() - return {"status": "deleted", "kind": kind, "count": count} - - if kind == "gallery": - count = db.query(GalleryImage).count() - db.query(GalleryImage).delete() - db.commit() - # Also drop the upload dir so disk doesn't keep orphans. - _rmtree_quiet(os.path.join(DATA_DIR, "gallery")) - _rmtree_quiet(os.path.join(DATA_DIR, "gallery_uploads")) - return {"status": "deleted", "kind": kind, "count": count} - - if kind == "calendar": - # Events FK calendars — clear children first, then both. - db.query(CalendarEvent).delete() - count = db.query(CalendarCal).count() - db.query(CalendarCal).delete() - db.commit() - return {"status": "deleted", "kind": kind, "count": count} +import sys as _sys - raise HTTPException(400, f"Unknown wipe kind: {kind!r}") - except HTTPException: - raise - except Exception as e: - db.rollback() - logger.exception(f"Wipe {kind} failed") - raise HTTPException(500, f"Wipe {kind} failed: {e}") - finally: - db.close() +from routes.admin_wipe import admin_wipe_routes as _canonical # noqa: F401 - return router +_sys.modules[__name__] = _canonical diff --git a/routes/api_token_routes.py b/routes/api_token_routes.py index ba412a48f..cbc828731 100644 --- a/routes/api_token_routes.py +++ b/routes/api_token_routes.py @@ -12,6 +12,65 @@ MAX_NAME_LEN = 100 DEFAULT_SCOPES = "chat" +ALLOWED_SCOPES = { + "chat", + "todos:read", + "todos:write", + "documents:read", + "documents:write", + "email:read", + "email:draft", + "email:send", + "calendar:read", + "calendar:write", + "memory:read", + "memory:write", + "cookbook:read", + "cookbook:launch", +} +TOKEN_PROFILES = { + "chat": ["chat"], + "codex_todos": ["todos:read", "todos:write"], + "codex_documents": ["documents:read", "documents:write"], + "codex_email_drafts": ["email:read", "email:draft", "documents:read", "documents:write"], +} + + +def _normalize_scopes(scopes: str | list[str] | None = None, profile: str | None = None) -> list[str]: + profile = profile if isinstance(profile, str) else None + profile_key = (profile or "").strip() + if profile_key: + if profile_key not in TOKEN_PROFILES: + raise HTTPException(400, "Unknown token profile") + requested = list(TOKEN_PROFILES[profile_key]) + elif isinstance(scopes, list): + requested = [str(s).strip() for s in scopes if str(s).strip()] + elif isinstance(scopes, str) and scopes: + requested = [s.strip() for s in scopes.replace(" ", ",").split(",") if s.strip()] + else: + requested = [DEFAULT_SCOPES] + + normalized = [] + for scope in requested: + if scope not in ALLOWED_SCOPES: + raise HTTPException(400, f"Unknown token scope: {scope}") + if scope not in normalized: + normalized.append(scope) + + def ensure_before(write_scope: str, read_scope: str): + if write_scope not in normalized or read_scope in normalized: + return + idx = normalized.index(write_scope) + normalized.insert(idx, read_scope) + + ensure_before("todos:write", "todos:read") + ensure_before("documents:write", "documents:read") + ensure_before("calendar:write", "calendar:read") + ensure_before("memory:write", "memory:read") + ensure_before("email:draft", "email:read") + ensure_before("cookbook:launch", "cookbook:read") + + return normalized or [DEFAULT_SCOPES] def setup_api_token_routes() -> APIRouter: @@ -45,13 +104,28 @@ def _invalidate_cache(request: Request): except Exception: pass + @router.get("/tokens/profiles") + def token_profiles(request: Request): + require_admin(request) + return { + "profiles": TOKEN_PROFILES, + "allowed_scopes": sorted(ALLOWED_SCOPES), + } + @router.post("/tokens") - def create_token(request: Request, name: str = Form("")): + def create_token( + request: Request, + name: str = Form(""), + scopes: str = Form(None), + profile: str = Form(None), + ): require_admin(request) name = name.strip()[:MAX_NAME_LEN] if not name: raise HTTPException(400, "Token name is required") owner = get_current_user(request) + scope_list = _normalize_scopes(scopes, profile) + scopes_value = ",".join(scope_list) raw_token = "ody_" + secrets.token_urlsafe(32) token_hash = bcrypt.hashpw(raw_token.encode(), bcrypt.gensalt()).decode() @@ -64,7 +138,7 @@ def create_token(request: Request, name: str = Form("")): name=name, token_hash=token_hash, token_prefix=raw_token[:8], - scopes=DEFAULT_SCOPES, + scopes=scopes_value, is_active=True, )) _invalidate_cache(request) @@ -75,16 +149,60 @@ def create_token(request: Request, name: str = Form("")): "owner": owner, "token": raw_token, "token_prefix": raw_token[:8], - "scopes": DEFAULT_SCOPES.split(","), + "scopes": scope_list, } + @router.patch("/tokens/{token_id}") + async def update_token(request: Request, token_id: str): + require_admin(request) + current_user = get_current_user(request) + try: + payload = await request.json() + except Exception: + payload = {} + if not isinstance(payload, dict): + payload = {} + with get_db_session() as db: + token = db.query(ApiToken).filter(ApiToken.id == token_id).first() + if not token: + raise HTTPException(404, "Token not found") + if current_user and token.owner != current_user: + raise HTTPException(403, "Not your token") + if isinstance(payload.get("name"), str) and payload["name"].strip(): + token.name = payload["name"].strip()[:MAX_NAME_LEN] + # Only touch scopes when the caller actually sent them. A partial + # update such as a rename ({"name": ...} with no "scopes" key) must + # not silently reset the token to the default scope — that dropped + # every previously granted scope. + if "scopes" in payload: + token.scopes = ",".join(_normalize_scopes(payload.get("scopes"))) + db.add(token) + current_scopes = [ + s.strip() + for s in (getattr(token, "scopes", "") or DEFAULT_SCOPES).split(",") + if s.strip() + ] + response = { + "id": token_id, + "name": getattr(token, "name", ""), + "owner": getattr(token, "owner", None), + "token_prefix": getattr(token, "token_prefix", ""), + "scopes": current_scopes, + } + _invalidate_cache(request) + return response + @router.delete("/tokens/{token_id}") def delete_token(request: Request, token_id: str): require_admin(request) + current_user = get_current_user(request) with get_db_session() as db: - deleted = db.query(ApiToken).filter(ApiToken.id == token_id).delete() - if not deleted: + token = db.query(ApiToken).filter(ApiToken.id == token_id).first() + if not token: raise HTTPException(404, "Token not found") + if current_user and token.owner != current_user: + raise HTTPException(403, "Not your token") + db.delete(token) _invalidate_cache(request) return {"status": "deleted"} diff --git a/routes/assistant_routes.py b/routes/assistant_routes.py index 17c50163d..0b609e37f 100644 --- a/routes/assistant_routes.py +++ b/routes/assistant_routes.py @@ -16,6 +16,7 @@ from core.database import SessionLocal, CrewMember, ScheduledTask from src.auth_helpers import get_current_user +from core.auth import RESERVED_USERNAMES from src.task_scheduler import compute_next_run @@ -89,11 +90,11 @@ def _owner(request: Request) -> str: # check-in tasks seeded. Hitting any /assistant route under one of these # used to seed a full CrewMember + Morning/Midday/Evening tasks under that # owner, which then double-fired alongside the real user's check-ins. - _SYNTHETIC_OWNERS = frozenset({"internal-tool", "api", "demo", "system", ""}) + # RESERVED_USERNAMES covers the same set; the `not owner` guard handles "". async def _get_or_create(owner: str) -> CrewMember: """Return the per-owner assistant CrewMember, creating it on demand.""" - if not owner or owner in _SYNTHETIC_OWNERS: + if not owner or owner in RESERVED_USERNAMES: raise HTTPException(status_code=400, detail=f"Cannot seed assistant for {owner!r}") db = SessionLocal() try: diff --git a/routes/auth_routes.py b/routes/auth_routes.py index 00a298adb..5c7a4e04a 100644 --- a/routes/auth_routes.py +++ b/routes/auth_routes.py @@ -3,11 +3,19 @@ from fastapi import APIRouter, Request, Response, HTTPException from pydantic import BaseModel from typing import Optional +import asyncio import logging import os -from core.auth import AuthManager +import json +import re +from pathlib import Path + +from core.atomic_io import atomic_write_json, atomic_write_text +from core.auth import AuthManager, RESERVED_USERNAMES, SetAdminResult, TOKEN_TTL +from src.constants import DEEP_RESEARCH_DIR, MEMORY_FILE, PASSWORD_MIN_LENGTH, SKILLS_DIR from src.rate_limiter import RateLimiter +from src.settings_scrub import scrub_settings from src.settings import ( load_settings as _load_settings, save_settings as _save_settings, @@ -21,6 +29,7 @@ update_integration, delete_integration, get_integration, + mask_integration_secret, execute_api_call, INTEGRATION_PRESETS, migrate_from_settings, @@ -61,6 +70,17 @@ class DeleteUserRequest(BaseModel): username: str +class RenameUserRequest(BaseModel): + username: str + + +class SetAdminRequest(BaseModel): + is_admin: bool + + +class SetOpenRegistrationRequest(BaseModel): + enabled: bool + SESSION_COOKIE = "odysseus_session" @@ -82,9 +102,13 @@ async def first_run_setup(body: SetupRequest, request: Request): raise HTTPException(429, "Too many requests — try again later") if auth_manager.is_configured: raise HTTPException(400, "Already configured") - if len(body.password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") - ok = auth_manager.setup(body.username, body.password) + if len(body.password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") + if len(body.username.strip()) < 1: + raise HTTPException(400, "Username is required") + if body.username.lower() in RESERVED_USERNAMES: + raise HTTPException(403, "Username is reserved") + ok = await asyncio.to_thread(auth_manager.setup, body.username, body.password) if not ok: raise HTTPException(500, "Setup failed") return {"ok": True, "message": "Admin account created"} @@ -98,11 +122,13 @@ async def signup(body: SignupRequest, request: Request): raise HTTPException(400, "Run setup first") if not auth_manager.signup_enabled: raise HTTPException(403, "Registration is disabled. Ask an admin for an account.") - if len(body.password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") + if len(body.password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") if len(body.username.strip()) < 1: raise HTTPException(400, "Username is required") - ok = auth_manager.create_user(body.username, body.password, is_admin=False) + if body.username.lower() in RESERVED_USERNAMES: + raise HTTPException(403, "Username is reserved") + ok = await asyncio.to_thread(auth_manager.create_user, body.username, body.password, is_admin=False) if not ok: raise HTTPException(409, "Username already taken") return {"ok": True, "message": "Account created"} @@ -113,7 +139,7 @@ async def login(body: LoginRequest, request: Request, response: Response): raise HTTPException(429, "Too many requests — try again later") # Verify password first username = body.username.strip().lower() - if not auth_manager.verify_password(username, body.password): + if not await asyncio.to_thread(auth_manager.verify_password, username, body.password): raise HTTPException(401, "Invalid credentials") # Check 2FA if enabled if auth_manager.totp_enabled(username): @@ -122,8 +148,8 @@ async def login(body: LoginRequest, request: Request, response: Response): return {"ok": False, "requires_totp": True, "username": username} if not auth_manager.totp_verify(username, body.totp_code): raise HTTPException(401, "Invalid 2FA code") - # All checks passed — create session - token = auth_manager.create_session(username, body.password) + # All checks passed — create session (password already verified above) + token = await asyncio.to_thread(auth_manager.create_session_trusted, username) if not token: raise HTTPException(401, "Invalid credentials") cookie_kwargs = dict( @@ -135,7 +161,7 @@ async def login(body: LoginRequest, request: Request, response: Response): path="/", ) if body.remember: - cookie_kwargs["max_age"] = 60 * 60 * 24 * 7 # 7 days + cookie_kwargs["max_age"] = TOKEN_TTL response.set_cookie(**cookie_kwargs) return {"ok": True, "username": username} @@ -164,16 +190,23 @@ async def auth_status(request: Request): pass return result + @router.get("/policy") + async def auth_policy(): + """Return public auth policy constants for the frontend.""" + return auth_manager.policy() + @router.post("/change-password") async def change_password(body: ChangePasswordRequest, request: Request): user = _get_current_user(request) if not user: raise HTTPException(401, "Not authenticated") - if len(body.new_password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") - ok = auth_manager.change_password(user, body.current_password, body.new_password) + if len(body.new_password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") + current_token = request.cookies.get(SESSION_COOKIE) + ok = await asyncio.to_thread(auth_manager.change_password, user, body.current_password, body.new_password) if not ok: raise HTTPException(400, "Current password is incorrect") + await asyncio.to_thread(auth_manager.revoke_user_sessions, user, current_token) return {"ok": True} # ------------------------------------------------------------------ @@ -248,8 +281,12 @@ async def admin_create_user(body: CreateUserRequest, request: Request): user = _get_current_user(request) if not user or not auth_manager.is_admin(user): raise HTTPException(403, "Admin only") - if len(body.password) < 8: - raise HTTPException(400, "Password must be at least 8 characters") + if len(body.password) < PASSWORD_MIN_LENGTH: + raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters") + if len(body.username.strip()) < 1: + raise HTTPException(400, "Username is required") + if body.username.lower() in RESERVED_USERNAMES: + raise HTTPException(403, "Username is reserved") ok = auth_manager.create_user(body.username, body.password, body.is_admin) if not ok: raise HTTPException(409, "Username already taken") @@ -266,23 +303,309 @@ async def update_user_privileges(username: str, request: Request): raise HTTPException(404, "User not found or is admin") return {"ok": True, "privileges": auth_manager.get_privileges(username)} - @router.post("/signup-toggle") + @router.put("/users/{username}/rename") + async def rename_user(username: str, body: RenameUserRequest, request: Request): + user = _get_current_user(request) + if not user or not auth_manager.is_admin(user): + raise HTTPException(403, "Admin only") + old_username = (username or "").strip().lower() + new_username = (body.username or "").strip().lower() + if not new_username: + raise HTTPException(400, "Username required") + if old_username == new_username: + return {"ok": True, "username": new_username, "renamed_self": old_username == user} + if old_username not in auth_manager.users: + raise HTTPException(404, "User not found") + if new_username in auth_manager.users: + raise HTTPException(409, "Username already taken") + + # Gate on auth first. Every mutation below is contingent on this + # succeeding — doing it last meant a rejected rename (e.g. reserved + # username) left file-backed owner fields already rewritten with no + # way to roll them back. + ok = auth_manager.rename_user(old_username, new_username, user) + if not ok: + raise HTTPException(400, "Cannot rename user") + + def _rollback_auth_rename() -> bool: + # On self-rename the admin session has already moved to the new + # username, so the rollback must authenticate as the new user. + rollback_user = new_username if user == old_username else user + try: + return bool(auth_manager.rename_user(new_username, old_username, rollback_user)) + except Exception as rollback_err: + logger.error( + "Failed to roll back auth rename %s -> %s after owner migration failure: %s", + new_username, old_username, rollback_err, + ) + return False + + # Usernames are ownership keys for user data. Rename the common + # owner-scoped DB rows so the account keeps access to its sessions, + # docs, email accounts, tasks, etc. + try: + from sqlalchemy import func + from core.database import Base, SessionLocal + db = SessionLocal() + try: + for mapper in Base.registry.mappers: + model = mapper.class_ + if not hasattr(model, "owner"): + continue + ( + db.query(model) + .filter(func.lower(model.owner) == old_username) + .update({"owner": new_username}, synchronize_session=False) + ) + db.commit() + except Exception: + db.rollback() + raise + finally: + db.close() + except Exception as e: + logger.error("Failed to rename owner references %s -> %s: %s", old_username, new_username, e) + if not _rollback_auth_rename(): + logger.error( + "Auth rename %s -> %s could not be rolled back after owner migration failure", + old_username, new_username, + ) + raise HTTPException(500, "Failed to rename user data") + + # Per-user prefs are JSON-backed, not SQL-backed. + try: + from routes.prefs_routes import _load as _load_prefs, _save as _save_prefs + prefs = _load_prefs() + users = prefs.get("_users") if isinstance(prefs, dict) else None + if isinstance(users, dict): + prefs_key = next( + (k for k in users if str(k).strip().lower() == old_username), + None, + ) + new_taken = any(str(k).strip().lower() == new_username for k in users) + if prefs_key is not None and not new_taken: + users[new_username] = users.pop(prefs_key) + _save_prefs(prefs) + except Exception as e: + logger.warning("Failed to rename user prefs %s -> %s: %s", old_username, new_username, e) + + # In-flight deep-research tasks live in the process-local + # ResearchHandler registry. They are not covered by the persisted JSON + # migration above, but the research routes filter and cancel by this + # owner field while the job is running. Do this before sweeping + # completed JSON files so a job that finishes during the rename saves + # with the new owner or is caught by the disk sweep below. + try: + rh = getattr(request.app.state, "research_handler", None) + rename_owner = getattr(rh, "rename_owner", None) + if callable(rename_owner): + rename_owner(old_username, new_username) + except Exception as e: + logger.warning("Failed to rename active research tasks %s -> %s: %s", old_username, new_username, e) + + # deep_research: each completed report is a standalone JSON file with + # an `owner` field. research_routes filters by d.get("owner") == user, + # so a stale owner makes every report invisible to the renamed user. + try: + dr_dir = Path(DEEP_RESEARCH_DIR) + if dr_dir.is_dir(): + for p in dr_dir.glob("*.json"): + try: + d = json.loads(p.read_text(encoding="utf-8")) + if str(d.get("owner", "")).strip().lower() == old_username: + d["owner"] = new_username + atomic_write_json(str(p), d) + except Exception as err: + logger.warning("Failed to update research owner in %s: %s", p.name, err) + except Exception as e: + logger.warning("Failed to rename research owner references %s -> %s: %s", old_username, new_username, e) + + # memory.json: a flat JSON array where each entry carries an `owner` + # field. memory_manager.load(owner=user) filters on it, so stale + # entries disappear from the memory panel. + try: + if os.path.isfile(MEMORY_FILE): + with open(MEMORY_FILE, encoding="utf-8") as fh: + entries = json.loads(fh.read()) + if isinstance(entries, list): + changed = False + for entry in entries: + if isinstance(entry, dict) and str(entry.get("owner", "")).strip().lower() == old_username: + entry["owner"] = new_username + changed = True + if changed: + atomic_write_json(MEMORY_FILE, entries) + except Exception as e: + logger.warning("Failed to rename memory.json owner references %s -> %s: %s", old_username, new_username, e) + + # uploads.json: upload rows use owner metadata for access checks and + # owner-prefixed index keys for dedupe. Rename both so attachments keep + # resolving after the account username changes. + try: + upload_handler = getattr(request.app.state, "upload_handler", None) + rename_owner = getattr(upload_handler, "rename_owner", None) + if callable(rename_owner): + rename_owner(old_username, new_username) + except Exception as e: + logger.warning("Failed to rename upload owner references %s -> %s: %s", old_username, new_username, e) + + # direct personal RAG uploads live in per-owner directories and the + # vector metadata also carries the username used for owner-filtered + # search. Keep both in sync with the auth rename. + try: + from routes.personal_routes import rename_personal_upload_owner + personal_docs_manager = getattr(request.app.state, "personal_docs_manager", None) + if personal_docs_manager is not None: + rag_manager = getattr(personal_docs_manager, "rag_manager", None) + rename_personal_upload_owner( + old_username, + new_username, + personal_docs_manager=personal_docs_manager, + rag_manager=rag_manager, + ) + except Exception as e: + logger.warning("Failed to rename personal RAG upload owner references %s -> %s: %s", old_username, new_username, e) + + # skills: SKILL.md frontmatter carries owner: ; the usage + # sidecar (_usage.json) keys entries as owner::skill-name. Both must + # be updated or the renamed user's Skills panel goes empty. + try: + skills_root = Path(SKILLS_DIR) + if skills_root.is_dir(): + _owner_re = re.compile( + r'(?m)^(owner:\s*)' + re.escape(old_username) + r'\s*$', + re.IGNORECASE, + ) + for p in skills_root.rglob("SKILL.md"): + try: + text = p.read_text(encoding="utf-8") + new_text = _owner_re.sub(r'\g<1>' + new_username, text) + if new_text != text: + atomic_write_text(str(p), new_text) + except Exception as err: + logger.warning("Failed to update skill owner in %s: %s", p, err) + usage_path = skills_root / "_usage.json" + if usage_path.is_file(): + try: + usage = json.loads(usage_path.read_text(encoding="utf-8")) + if isinstance(usage, dict): + new_usage = {} + changed = False + for k, v in usage.items(): + owner_part, sep, skill_part = k.partition("::") + if sep and owner_part.lower() == old_username: + new_usage[new_username + "::" + skill_part] = v + changed = True + else: + new_usage[k] = v + if changed: + atomic_write_json(str(usage_path), new_usage) + except Exception as err: + logger.warning("Failed to update skills usage keys %s -> %s: %s", old_username, new_username, err) + except Exception as e: + logger.warning("Failed to rename skills owner references %s -> %s: %s", old_username, new_username, e) + + # The in-memory session cache (session_manager.sessions) stores each + # session's owner at load time. Without this patch the renamed user's + # sessions are invisible on the next /api/sessions call because + # get_sessions_for_user does an exact `s.owner == username` comparison + # against stale in-memory values. + sm = getattr(request.app.state, "session_manager", None) + if sm is not None: + for sess in list(getattr(sm, "sessions", {}).values()): + if str(getattr(sess, "owner", None) or "").strip().lower() == old_username: + sess.owner = new_username + + # The owner-rename loop above updated ApiToken.owner in the DB, but the + # bearer-token cache still maps each token to the OLD owner. Without + # refreshing it, the renamed user's API tokens resolve to the old (now + # non-existent) owner and stop reaching their data until the cache next + # goes dirty. Invalidate it now, like the token CRUD routes do. + invalidator = getattr(request.app.state, "invalidate_token_cache", None) + if callable(invalidator): + invalidator() + return {"ok": True, "username": new_username, "renamed_self": old_username == user} + + @router.put("/users/{username}/admin") + async def set_user_admin(username: str, body: SetAdminRequest, request: Request): + """Promote/demote a user to/from admin. Admin only. + + The last remaining admin can't be demoted (no lockout). Self-demotion + is allowed while another admin exists; the `self` flag tells the UI to + reload the acting user into the normal-user view. + """ + user = _get_current_user(request) + if not user or not auth_manager.is_admin(user): + raise HTTPException(403, "Admin only") + result = auth_manager.set_admin(username, body.is_admin, user) + if result is SetAdminResult.USER_NOT_FOUND: + raise HTTPException(404, "User not found") + if result is SetAdminResult.NOT_AUTHORIZED: + raise HTTPException(403, "Admin only") + if result is SetAdminResult.LAST_ADMIN: + raise HTTPException(400, "Cannot demote the last admin") + target = (username or "").strip().lower() + return { + "ok": True, + "is_admin": body.is_admin, + "self": target == (user or "").strip().lower(), + } + + @router.post("/signup-toggle", deprecated=True) async def toggle_signup(request: Request): - """Toggle open registration on/off. Admin only.""" + """ + Toggle open registration on/off. Admin only. + + DEPRECATED: This endpoint uses toggle semantics which can lead to unsafe state changes. + Use PUT /open-signup instead. + + This endpoint is kept for backward compatibility and may be removed in future versions. + """ user = _get_current_user(request) if not user or not auth_manager.is_admin(user): raise HTTPException(403, "Admin only") auth_manager.signup_enabled = not auth_manager.signup_enabled return {"ok": True, "signup_enabled": auth_manager.signup_enabled} + @router.put("/open-signup") + async def set_signup_enabled(body: SetOpenRegistrationRequest, request: Request): + """Set open signup enabled state. Admin only.""" + user = _get_current_user(request) + if not user or not auth_manager.is_admin(user): + raise HTTPException(403, "Admin only") + auth_manager.signup_enabled = body.enabled + return {"ok": True,"signup_enabled": auth_manager.signup_enabled} + @router.delete("/users") async def admin_delete_user(body: DeleteUserRequest, request: Request): user = _get_current_user(request) if not user or not auth_manager.is_admin(user): raise HTTPException(403, "Admin only") - ok = auth_manager.delete_user(body.username, user) + + def _invalidate_api_token_cache(): + try: + invalidator = getattr(request.app.state, "invalidate_token_cache", None) + if invalidator: + invalidator() + except Exception: + pass + + try: + ok = auth_manager.delete_user(body.username, user) + except Exception: + # delete_user can touch ApiToken rows before a later auth-store write + # fails. Dirty the bearer cache anyway so a partial token purge does + # not leave already-cached tokens authenticating until restart. + _invalidate_api_token_cache() + raise if not ok: raise HTTPException(400, "Cannot delete user") + # delete_user removes the user's ApiToken rows, but the bearer-auth + # middleware serves from an in-memory prefix->token cache that only + # rebuilds when flagged dirty. Without this, a deleted user's already + # cached token keeps authenticating until some other token op or a + # restart clears the cache. Mirror what the token routes do. + _invalidate_api_token_cache() return {"ok": True} # ---- Feature visibility (admin-managed) ---- @@ -308,29 +631,6 @@ async def set_features(request: Request): # ---- App settings (admin-managed) ---- - _SECRET_KEY_PATTERNS = ("_api_key", "_password", "_secret", "_token", "_key") - - def _is_secret_key(name: str) -> bool: - n = (name or "").lower() - if n in ("google_pse_cx",): # public identifier, not a secret - return False - return any(n.endswith(p) or n == p.lstrip("_") for p in _SECRET_KEY_PATTERNS) - - def _scrub_settings(settings: dict) -> dict: - """Return a copy of settings with secret-shaped values masked. - - Frontend reads /settings without auth for things like keybinds + TTS - prefs. Secrets (search-provider keys, IMAP/SMTP passwords) must NOT - be exposed to non-admin callers. - """ - scrubbed = {} - for k, v in (settings or {}).items(): - if _is_secret_key(k) and isinstance(v, str) and v: - scrubbed[k] = "" # presence preserved, value blanked - else: - scrubbed[k] = v - return scrubbed - @router.get("/settings") async def get_settings(request: Request): """Returns app settings. Admins get the full set; non-admins get @@ -340,7 +640,7 @@ async def get_settings(request: Request): settings = _load_settings() if user and auth_manager.is_admin(user): return settings - return _scrub_settings(settings) + return scrub_settings(settings) @router.post("/settings") async def set_settings(request: Request): @@ -350,9 +650,24 @@ async def set_settings(request: Request): raise HTTPException(403, "Admin only") body = await request.json() current = _load_settings() + # Per-key validation for numeric settings: coerce to int and clamp to a + # sane range so a bad value can't disable the agent or let it run away. + _INT_RANGES = { + "agent_max_rounds": (1, 200), + "agent_max_tool_calls": (0, 1000), # 0 = unlimited + } for key in DEFAULT_SETTINGS: - if key in body: - current[key] = body[key] + if key not in body: + continue + val = body[key] + if key in _INT_RANGES: + lo, hi = _INT_RANGES[key] + try: + val = int(val) + except (TypeError, ValueError): + raise HTTPException(400, f"{key} must be an integer") + val = max(lo, min(val, hi)) + current[key] = val _save_settings(current) return current @@ -369,12 +684,7 @@ async def list_integrations_route(request: Request): raise HTTPException(403, "Admin only") items = load_integrations() # Mask API keys for frontend display - safe = [] - for item in items: - copy = dict(item) - if copy.get("api_key"): - copy["api_key"] = copy["api_key"][:4] + "****" - safe.append(copy) + safe = [mask_integration_secret(item) for item in items] return {"integrations": safe} @router.get("/integrations/presets") @@ -390,7 +700,7 @@ async def create_integration(request: Request): raise HTTPException(403, "Admin only") body = await request.json() item = add_integration(body) - return {"ok": True, "integration": item} + return {"ok": True, "integration": mask_integration_secret(item)} @router.put("/integrations/{integration_id}") async def update_integration_route(integration_id: str, request: Request): @@ -402,7 +712,7 @@ async def update_integration_route(integration_id: str, request: Request): item = update_integration(integration_id, body) if not item: raise HTTPException(404, "Integration not found") - return {"ok": True, "integration": item} + return {"ok": True, "integration": mask_integration_secret(item)} @router.delete("/integrations/{integration_id}") async def delete_integration_route(integration_id: str, request: Request): @@ -487,6 +797,27 @@ async def test_integration_route(integration_id: str, request: Request): hint = " If this is Docker Compose ntfy, set NTFY_BIND to that host/Tailscale IP and NTFY_BASE_URL to the same server URL in .env, then recreate ntfy." return {"ok": False, "message": f"ntfy publish to {full_url} failed: {e}.{hint}"[:500]} + if preset == "discord_webhook": + import httpx + webhook_url = (integ.get("base_url") or "").strip() + if not webhook_url: + return {"ok": False, "message": "No webhook URL set — paste the full Discord webhook URL into the Base URL field."} + payload = { + "embeds": [{ + "title": "Odysseus connectivity test", + "description": "If you see this, your Discord Webhook integration is wired up correctly.", + "color": 5793266, + }] + } + try: + async with httpx.AsyncClient(timeout=8.0) as client: + r = await client.post(webhook_url, json=payload) + if r.is_success: + return {"ok": True, "message": "Test embed sent — check your Discord channel to confirm it arrived."} + return {"ok": False, "message": f"Discord returned HTTP {r.status_code}: {r.text[:200]}"} + except Exception as e: + return {"ok": False, "message": f"Request failed: {e}"[:400]} + # All other presets: GET against a known health endpoint. # Fall back to detecting from name if preset is missing. health_paths = { diff --git a/routes/backup_routes.py b/routes/backup_routes.py index b165fcce7..313369370 100644 --- a/routes/backup_routes.py +++ b/routes/backup_routes.py @@ -77,7 +77,12 @@ async def import_data(request: Request): # ── Memories ── if "memories" in body and isinstance(body["memories"], list): existing = memory_manager.load_all() - existing_texts = {e.get("text", "").strip().lower() for e in existing} + # Dedup against THIS user's own memories only. Using every tenant's + # rows (load_all) meant a memory whose text matched any other + # user's was silently skipped, so the importing user lost their own + # data. The full store is still saved back below. + existing_texts = {e.get("text", "").strip().lower() + for e in existing if e.get("owner") == user} added = 0 for mem in body["memories"]: if not isinstance(mem, dict) or not mem.get("text"): @@ -96,24 +101,74 @@ async def import_data(request: Request): # ── Skills ── if "skills" in body and isinstance(body["skills"], list): existing = skills_manager.load_all() - existing_ids = {s.get("id") for s in existing} - existing_titles = {s.get("title", "").strip().lower() for s in existing} + # Dedup against THIS user's own skills only. Using every tenant's + # rows (load_all) meant a skill whose id/name/title matched any + # other user's was silently skipped, so the importing user lost + # their own data — same cross-tenant bug fixed for memories above. + # The full store is still saved back below. + own = [s for s in existing if s.get("owner") == user] + existing_names = {s.get("name") for s in own if s.get("name")} + existing_ids = {s.get("id") for s in own if s.get("id")} + existing_titles = { + (s.get("title") or s.get("description") or "").strip().lower() + for s in own + } added = 0 for skill in body["skills"]: - if not isinstance(skill, dict) or not skill.get("title"): + if not isinstance(skill, dict): continue - # Skip if same id or same title already exists - if skill.get("id") in existing_ids: + title = ( + skill.get("title") or skill.get("description") + or skill.get("name") or "" + ).strip() + if not title: continue - if skill["title"].strip().lower() in existing_titles: + sid = skill.get("id") or skill.get("name") + if sid and sid in existing_ids: continue - if user and not skill.get("owner"): - skill["owner"] = user - existing.append(skill) - existing_ids.add(skill.get("id")) - existing_titles.add(skill["title"].strip().lower()) + nm = skill.get("name") + if nm and nm in existing_names: + continue + if title.lower() in existing_titles: + continue + owner = skill.get("owner") + if user and not owner: + owner = user + # Skills live on disk as SKILL.md files; the old JSON-era + # skills_manager.save() no longer exists. Write each new skill + # via add_skill (source="user" skips auto-dedup — this is an + # explicit backup restore). + result = skills_manager.add_skill( + title=title, + name=skill.get("name"), + description=skill.get("description"), + problem=skill.get("problem", ""), + solution=skill.get("solution", ""), + steps=skill.get("steps"), + tags=skill.get("tags"), + source="user", + teacher_model=skill.get("teacher_model"), + confidence=skill.get("confidence", 0.8), + owner=owner, + category=skill.get("category", "general"), + when_to_use=skill.get("when_to_use"), + procedure=skill.get("procedure"), + pitfalls=skill.get("pitfalls"), + verification=skill.get("verification"), + platforms=skill.get("platforms"), + requires_toolsets=skill.get("requires_toolsets"), + fallback_for_toolsets=skill.get("fallback_for_toolsets"), + status=skill.get("status", "draft"), + version=skill.get("version", "1.0.0"), + ) + if result.get("_deduped"): + continue + if result.get("name"): + existing_names.add(result["name"]) + if result.get("id"): + existing_ids.add(result["id"]) + existing_titles.add(title.lower()) added += 1 - skills_manager.save(existing) imported.append(f"{added} skills") # ── Presets ── diff --git a/routes/calendar_routes.py b/routes/calendar_routes.py index faff70ffc..6e0ee124c 100644 --- a/routes/calendar_routes.py +++ b/routes/calendar_routes.py @@ -1,18 +1,59 @@ """Calendar routes — local SQLite-backed calendar CRUD.""" import logging +import json +import re import uuid from datetime import datetime, date, timedelta -from typing import Optional +from typing import Optional, List from fastapi import APIRouter, HTTPException, Request, UploadFile, File from pydantic import BaseModel +from sqlalchemy import or_, and_ +from dateutil.rrule import rrulestr -from core.database import SessionLocal, CalendarCal, CalendarEvent -from src.auth_helpers import get_current_user +from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent +from src.auth_helpers import effective_user, require_user +from src.upload_limits import read_upload_limited, ICS_MAX_BYTES +from src.upload_handler import reserve_upload_references logger = logging.getLogger(__name__) + +def _ics_naive_dtstart(dt): + """Naive value matching how import_ics STORES CalendarEvent.dtstart. + + Timed tz-aware events are stored as UTC with tzinfo stripped, all-day + dates as midnight datetimes, naive datetimes unchanged. The ICS dedup + must compute the same value or a re-import never matches the stored row. + """ + if isinstance(dt, datetime): + if dt.tzinfo is not None: + from datetime import timezone as _tz + return dt.astimezone(_tz.utc).replace(tzinfo=None) + return dt + if isinstance(dt, date): + return datetime(dt.year, dt.month, dt.day) + return dt + + +def _ensure_positive_duration(start_dt, end_dt, all_day): + """Clamp an imported event's end so it has a positive duration. + + Some .ics exporters write a single-day all-day event with DTEND equal to + DTSTART (treating DTEND as inclusive rather than the RFC 5545 exclusive + bound). Stored verbatim that produces a zero-duration row, which the + list_events overlap filter (dtstart < end AND dtend > start) silently + drops — the event never appears on the calendar even though the web UI + would otherwise show it. Normalize a non-positive end to the same default + span used when DTEND is absent: one day for all-day events, one hour + otherwise. + """ + if end_dt <= start_dt: + return start_dt + (timedelta(days=1) if all_day else timedelta(hours=1)) + return end_dt + + # Single-user fallback identity. Used only when: # 1. The app is configured for single-user (no auth middleware), AND # 2. The request didn't resolve to an authenticated user. @@ -25,16 +66,17 @@ def _require_user(request: Request) -> str: - """Return the authenticated user. In multi-user mode an unauthenticated - request raises 401; in single-user mode it falls through to - FALLBACK_OWNER. Prevents the silent cross-user data write that would - happen if a request slipped past auth middleware in a real deployment.""" - u = get_current_user(request) - if u: - return u - if _SINGLE_USER_MODE: - return FALLBACK_OWNER - raise HTTPException(401, "Authentication required") + """Return the authenticated user. Uses require_user so AUTH_ENABLED=false + and single-user mode both work: require_user returns "" when auth is + disabled or unconfigured, and only raises 401 when auth is configured but + the caller is unauthenticated. Falls back to FALLBACK_OWNER for calendar + writes so data isn't stored under an empty owner in single-user mode.""" + user = require_user(request) + if user: + return user + # require_user returned "" — auth is off or unconfigured (single-user). + # Use FALLBACK_OWNER so calendar rows have a stable owner for filtering. + return FALLBACK_OWNER def _get_or_404_calendar(db, cal_id: str, owner: str) -> CalendarCal: @@ -60,6 +102,98 @@ def _get_or_404_event(db, uid: str, owner: str) -> CalendarEvent: raise HTTPException(404, "Event not found") return ev + +def _ics_escape(text: str) -> str: + """Escape a value for an iCalendar TEXT field (RFC 5545 §3.3.11). + + Backslash, semicolon and comma are structural in TEXT values and must be + escaped, and newlines become a literal ``\\n``. Backslash is escaped first + so the escapes we add aren't re-escaped. + """ + return ( + (text or "") + .replace("\\", "\\\\") + .replace(";", "\\;") + .replace(",", "\\,") + .replace("\r\n", "\\n") + .replace("\n", "\\n") + .replace("\r", "\\n") + ) + + +def _safe_ics_filename(name: str) -> str: + """Return a conservative .ics filename safe for Content-Disposition.""" + stem = name if isinstance(name, str) else "" + stem = re.sub(r"[^A-Za-z0-9._-]", "_", stem).strip("._-") + if not stem: + stem = "calendar" + return f"{stem[:128]}.ics" + + +def _resolve_base_uid(uid: str) -> str: + """Extract the base series UID from a compound occurrence UID. + + Compound UIDs have the form ``{base_uid}::{date_suffix}``. + For plain UIDs (no ``::``), returns the UID unchanged. + """ + if not uid: + raise ValueError("empty uid") + idx = uid.find("::") + if idx == -1: + return uid # plain UID — no suffix + base = uid[:idx] + if not base: + raise ValueError("malformed compound UID: missing base before ::") + return base + + +async def _push_caldav_event_after_commit(owner: str, uid: str, action: str): + """Best-effort CalDAV write-through. Local writes stay authoritative if + the remote server is unreachable; pending flags let /sync retry later.""" + try: + result = {"ok": True} + if action == "create": + from src.caldav_sync import push_event_create + result = await push_event_create(owner, uid) + elif action == "update": + from src.caldav_sync import push_event_update + result = await push_event_update(owner, uid) + elif action == "delete": + from src.caldav_sync import push_event_delete + result = await push_event_delete(owner, uid) + if result and not result.get("ok") and not result.get("skipped"): + raise RuntimeError(result.get("error") or result) + except Exception as e: + logger.warning("CalDAV %s push failed for uid=%s: %s", action, uid, e) + if action in {"create", "update"}: + db = SessionLocal() + try: + ev = _get_or_404_event(db, uid, owner) + ev.caldav_sync_pending = action + db.commit() + except Exception: + db.rollback() + finally: + db.close() + + +def _record_caldav_delete_tombstone(db, ev: CalendarEvent, owner: str) -> None: + if not (ev.calendar and ev.calendar.source == "caldav"): + return + tombstone = db.query(CalendarDeletedEvent).filter( + CalendarDeletedEvent.uid == ev.uid, + CalendarDeletedEvent.owner == owner, + ).first() + if not tombstone: + tombstone = CalendarDeletedEvent(uid=ev.uid, owner=owner) + db.add(tombstone) + tombstone.calendar_id = ev.calendar_id + tombstone.remote_href = ev.remote_href + tombstone.remote_etag = ev.remote_etag + tombstone.caldav_base_url = getattr(ev.calendar, "caldav_base_url", None) + tombstone.summary = ev.summary or "" + tombstone.last_error = None + # ── Pydantic models ── class EventCreate(BaseModel): @@ -105,26 +239,18 @@ def _ensure_default_calendar(db, owner: str = None) -> CalendarCal: return cal -# Per-request user UTC offset (in minutes east of UTC). chat_routes sets this -# from the `X-Tz-Offset` header so naive natural-language times the LLM -# emits ("today at 9pm") are parsed in the USER's timezone, not the server's -# clock. None = unknown, fall back to legacy server-local behavior. -from contextvars import ContextVar -_USER_TZ_OFFSET_MIN: ContextVar = ContextVar("user_tz_offset_min", default=None) - - -def set_user_tz_offset(offset_min): - """Set the current user's UTC offset for this async context.""" - try: - v = int(offset_min) - except (TypeError, ValueError): - return - _USER_TZ_OFFSET_MIN.set(v) - - -def get_user_tz_offset(): - """Read the current user's UTC offset (minutes east of UTC), or None.""" - return _USER_TZ_OFFSET_MIN.get() +# Per-request user time context. chat_routes sets this from browser timezone +# headers so natural-language times the LLM emits ("today at 9pm") are parsed +# in the user's timezone, not the server's clock. None = unknown, fall back to +# legacy server-local behavior. +from src.user_time import ( + get_user_tz_name, + get_user_tz_offset, + now_user_local, + set_user_tz_name, + set_user_tz_offset, + user_timezone, +) def parse_due_for_user(s: str) -> str: @@ -143,6 +269,7 @@ def parse_due_for_user(s: str) -> str: """ from datetime import timezone as _tz, timedelta as _td offset = get_user_tz_offset() + tz_name = get_user_tz_name() s = (s or "").strip() if not s: return s @@ -156,11 +283,11 @@ def parse_due_for_user(s: str) -> str: except ValueError: parsed = None - if offset is None: + if offset is None and not tz_name: # No user tz known — preserve legacy behavior (naive server-local). return _parse_dt(s).isoformat() - user_tz = _tz(_td(minutes=offset)) + user_tz = user_timezone() # Naive ISO → tag with user tz. if parsed is not None and parsed.tzinfo is None: @@ -168,7 +295,7 @@ def parse_due_for_user(s: str) -> str: # Natural language — evaluate against user's "now". server_now_utc = datetime.now(_tz.utc) - user_now = server_now_utc.astimezone(user_tz) + user_now = now_user_local(server_now_utc) # Patch datetime.now() inside _parse_dt by leveraging the user's clock: # we re-implement the small natural-language phrases here against user_now # so the result is naturally in the user's tz. @@ -176,6 +303,7 @@ def parse_due_for_user(s: str) -> str: lower = s.lower().strip() def _parse_time(t): + t = _re.sub(r'\b([ap])\s*\.?\s*m\.?\b', r'\1m', t.strip(), flags=_re.IGNORECASE) m = _re.match(r'^\s*(\d{1,2})(?::(\d{2}))?\s*(am|pm)?\s*$', t, _re.IGNORECASE) if not m: return None h = int(m.group(1)); mn = int(m.group(2) or 0); ampm = (m.group(3) or "").lower() @@ -198,6 +326,17 @@ def _parse_time(t): if t is not None: return base.replace(hour=t[0], minute=t[1]).isoformat() + # Time-first: "3pm today", "11pm today", "9am tomorrow" + m = _re.match(r'^(.+?)\s+(today|tonight|tomorrow|tmrw|yesterday)$', lower) + if m: + time_part, word = m.group(1).strip(), m.group(2) + base = today + if word in ("tomorrow", "tmrw"): base = today + _td(days=1) + elif word == "yesterday": base = today - _td(days=1) + t = _parse_time(time_part) + if t is not None: + return base.replace(hour=t[0], minute=t[1]).isoformat() + m = _re.match(r'^in\s+(\d+)\s*(hour|hr|minute|min|day)s?\s*$', lower) if m: n = int(m.group(1)); unit = m.group(2) @@ -285,6 +424,7 @@ def _parse_dt(s: str) -> datetime: def _parse_time(t: str): """Return (hour, minute) from '1pm', '1:30 PM', '13:00', etc., or None.""" + t = _re.sub(r'\b([ap])\s*\.?\s*m\.?\b', r'\1m', t.strip(), flags=_re.IGNORECASE) m = _re.match(r'^\s*(\d{1,2})(?::(\d{2}))?\s*(am|pm)?\s*$', t, _re.IGNORECASE) if not m: return None @@ -299,8 +439,8 @@ def _parse_time(t: str): return None return h, mn - # today/tomorrow/yesterday [at] TIME - m = _re.match(r'^(today|tomorrow|tmrw|yesterday)(?:\s+at)?\s*(.*)$', lower) + # today/tonight/tomorrow/yesterday [at] TIME + m = _re.match(r'^(today|tonight|tomorrow|tmrw|yesterday)(?:\s+at)?\s*(.*)$', lower) if m: word, rest = m.group(1), m.group(2).strip() base = today @@ -314,6 +454,20 @@ def _parse_time(t: str): if t is not None: return base.replace(hour=t[0], minute=t[1]) + # time-first: "3pm today", "9am tomorrow", "11pm tonight" + # (parity with parse_due_for_user, which handles these via the same form) + m = _re.match(r'^(.+?)\s+(today|tonight|tomorrow|tmrw|yesterday)$', lower) + if m: + time_part, word = m.group(1).strip(), m.group(2) + base = today + if word in ("tomorrow", "tmrw"): + base = today + timedelta(days=1) + elif word == "yesterday": + base = today - timedelta(days=1) + t = _parse_time(time_part) + if t is not None: + return base.replace(hour=t[0], minute=t[1]) + # next [at] TIME weekdays = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"] m = _re.match(r'^next\s+(\w+)(?:\s+at)?\s*(.*)$', lower) @@ -348,7 +502,17 @@ def _parse_time(t: str): # Last resort: dateutil's fuzzy parser try: from dateutil import parser as _du - return _du.parse(s) + parsed = _du.parse(s) + # Strip tz like every other return path above — this function's + # contract is naive datetimes (CalendarEvent.dtstart is naive). An + # offset-bearing non-ISO input (e.g. RFC-2822 "Mon, 05 Jan 2026 + # 14:00:00 +0900") otherwise leaked tz-aware into the naive column and + # crashed read-back comparisons in _expand_rrule with "can't compare + # offset-naive and offset-aware datetimes". + if parsed.tzinfo is not None: + from datetime import timezone as _tz + return parsed.astimezone(_tz.utc).replace(tzinfo=None) + return parsed except Exception: raise ValueError(f"could not parse datetime: {s!r}") @@ -379,6 +543,7 @@ def _event_to_dict(ev: CalendarEvent) -> dict: "description": ev.description or "", "location": ev.location or "", "rrule": ev.rrule or "", + "recurrence_exdates": _recurrence_exdates(ev), "calendar": ev.calendar.name if ev.calendar else "", "calendar_href": ev.calendar_id, "color": ev.color or (ev.calendar.color if ev.calendar else ""), @@ -387,62 +552,336 @@ def _event_to_dict(ev: CalendarEvent) -> dict: } +# ── Recurrence expansion ── + +_RRULE_EXPANSION_LIMIT = 1000 + + +def _recurrence_exdates(ev: CalendarEvent) -> list[str]: + raw = getattr(ev, "recurrence_exdates", "") or "" + if not raw: + return [] + try: + values = json.loads(raw) + except Exception: + return [] + if not isinstance(values, list): + return [] + return [str(v) for v in values if isinstance(v, str) and v.strip()] + + +def _occurrence_exdate_key(uid: str, ev: CalendarEvent) -> str: + if "::" not in uid: + return "" + suffix = uid.split("::", 1)[1] + if ev.all_day: + return suffix[:10] + return suffix[:16] + + +def _expand_rrule( + ev: CalendarEvent, start: datetime, end: datetime +) -> List[dict]: + """Expand a single recurring CalendarEvent into occurrence dicts. + + Each occurrence gets a stable compound UID of the form + ``{base_uid}::{date_or_datetime}`` so the frontend can tell + occurrences apart while the series UID is still recoverable + for edit/delete targeting. + + Non-recurring events (empty rrule) are returned as a single-item + list — the caller doesn't need to branch. + """ + duration = ev.dtend - ev.dtstart + + if not ev.rrule or not ev.rrule.strip(): + # Non-recurring — return the base event as-is. list_events + # already filters non-recurring rows with the overlap check + # in SQL, so we don't re-check here. + d = _event_to_dict(ev) + d["is_recurrence"] = False + d["series_uid"] = ev.uid + d["truncated"] = False + return [d] + + # Parse the rrule, applying it to the base dtstart. + rrule_str = ev.rrule + if ev.dtstart is not None and getattr(ev.dtstart, "tzinfo", None) is None: + # Events are stored with a naive (UTC) dtstart, but standard .ics + # exporters (Google/Apple/Outlook/Fastmail) write the bound as an + # absolute UTC value, e.g. UNTIL=20240105T090000Z. dateutil refuses to + # mix a tz-aware UNTIL with a naive DTSTART ("RRULE UNTIL values must be + # specified in UTC when DTSTART is timezone-aware"), so the except branch + # below would silently collapse the whole series to a single event. + # Drop the trailing Z so UNTIL matches the naive DTSTART. + import re as _re + rrule_str = _re.sub( + r"(UNTIL=\d{8}(?:T\d{6})?)Z", r"\1", rrule_str, flags=_re.IGNORECASE + ) + try: + rule = rrulestr(rrule_str, dtstart=ev.dtstart) + except Exception as ex: + logger.warning( + "Failed to parse rrule=%r for event %s: %s", ev.rrule, ev.uid, ex + ) + d = _event_to_dict(ev) + d["is_recurrence"] = False + d["series_uid"] = ev.uid + d["truncated"] = False + # Malformed RRULE rows are fetched by the recurring SQL branch + # with only dtstart < end_dt — the base event may not actually + # overlap the window. Only return if it does. + if ev.dtstart < end and ev.dtend > start: + return [d] + return [] + + # Expand from start - duration so multi-day / overnight occurrences + # that start before the window but end inside it are captured + # (matching non-recurring overlap semantics: dtstart < end AND + # dtend > start). + expand_start = start - duration + results = [] + truncated = False + base = _event_to_dict(ev) + exdates = set(_recurrence_exdates(ev)) + + for occ_start in rule.xafter(expand_start, inc=True): + if occ_start >= end: + break + + occ_end = occ_start + duration + + # Overlap filter: occurrence must intersect [start, end). + # This enforces exclusive-end semantics (occ_start >= end is + # excluded) and includes multi-day crossings (occ_end > start). + if occ_end <= start: + continue + + if len(results) >= _RRULE_EXPANSION_LIMIT: + truncated = True + break + + # Build the compound uid: {base_uid}::{date} or ::{datetime} + if ev.all_day: + occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%d')}" + exdate_key = occ_start.strftime("%Y-%m-%d") + else: + occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%dT%H:%M')}" + exdate_key = occ_start.strftime("%Y-%m-%dT%H:%M") + + if exdate_key in exdates: + continue + + d = dict(base) + d["uid"] = occ_uid + d["series_uid"] = ev.uid + d["is_recurrence"] = True + d["truncated"] = False + + if ev.all_day: + d["dtstart"] = occ_start.strftime("%Y-%m-%d") + d["dtend"] = occ_end.strftime("%Y-%m-%d") + else: + suffix = "Z" if getattr(ev, "is_utc", False) else "" + d["dtstart"] = occ_start.isoformat() + suffix + d["dtend"] = occ_end.isoformat() + suffix + d["is_utc"] = bool(getattr(ev, "is_utc", False)) + + results.append(d) + + if truncated: + for d in results: + d["truncated"] = True + + return results + + # ── Routes ── -def setup_calendar_routes() -> APIRouter: +def setup_calendar_routes(upload_handler=None) -> APIRouter: router = APIRouter(prefix="/api/calendar", tags=["calendar"]) - # CalDAV connect form (Integrations → Calendar). Storage is local - # SQLite; sync (src/caldav_sync.py) pulls remote events into it on - # calendar open and periodically via the scheduler. + def _reserve_calendar_uploads(request: Request, *values) -> None: + missing_id = reserve_upload_references( + upload_handler, + effective_user(request), + *values, + ) + if missing_id: + raise HTTPException(409, f"Referenced upload is no longer available: {missing_id}") + + # ── CalDAV multi-account helpers ───────────────────────────────────────── + + def _get_caldav_accounts(owner: str) -> list: + from src.caldav_sync import _load_caldav_accounts + return _load_caldav_accounts(owner) + + def _save_caldav_accounts(owner: str, accounts: list) -> None: + from routes.prefs_routes import _load_for_user, _save_for_user + prefs = _load_for_user(owner) or {} + prefs["caldav_accounts"] = accounts + prefs.pop("caldav", None) + _save_for_user(owner, prefs) + + # ── CalDAV config routes (backward-compat single-account API) ──────────── + @router.get("/config") async def get_config(request: Request): + """Legacy single-account endpoint — returns the first configured account.""" owner = _require_user(request) - from routes.prefs_routes import _load_for_user - cfg = (_load_for_user(owner) or {}).get("caldav", {}) or {} - # Surface url+username but never hand the password back to the - # client — saved-state UI shouldn't leak the credential. + accounts = _get_caldav_accounts(owner) + if not accounts: + return {"url": "", "username": "", "password": "", "has_password": False, "local": True} + first = accounts[0] + pw = first.get("password") or "" + has_pw = False + if pw: + try: + from src.secret_storage import decrypt + has_pw = bool(decrypt(pw)) + except Exception: + has_pw = bool(pw) return { - "url": cfg.get("url", "") or "", - "username": cfg.get("username", "") or "", + "url": first.get("url", "") or "", + "username": first.get("username", "") or "", "password": "", - "has_password": bool(cfg.get("password")), - "local": not bool(cfg.get("url")), + "has_password": has_pw, + "local": not bool(first.get("url")), } @router.post("/config") async def save_config(request: Request): + """Legacy single-account endpoint — upserts the first account.""" owner = _require_user(request) - from routes.prefs_routes import _load_for_user, _save_for_user try: body = await request.json() except Exception: body = {} - prefs = _load_for_user(owner) or {} - cfg = dict(prefs.get("caldav") or {}) - # Empty url => clear the whole entry (treat as "remove integration"). + accounts = _get_caldav_accounts(owner) if not (body.get("url") or "").strip(): - prefs.pop("caldav", None) - _save_for_user(owner, prefs) + _save_caldav_accounts(owner, []) return {"ok": True, "cleared": True} - cfg["url"] = body.get("url", "").strip() - cfg["username"] = (body.get("username") or "").strip() - # Preserve the stored password when the client sends an empty - # one (edit form re-submitted without re-typing the password). + from src.caldav_sync import validate_caldav_url + try: + validated_url = validate_caldav_url(body.get("url", "")) + except ValueError as e: + raise HTTPException(400, str(e)) + if accounts: + acc = dict(accounts[0]) + else: + import uuid as _uuid + acc = {"id": str(_uuid.uuid4()), "label": "CalDAV"} + acc["url"] = validated_url + acc["username"] = (body.get("username") or "").strip() if body.get("password"): - cfg["password"] = body["password"] - prefs["caldav"] = cfg - _save_for_user(owner, prefs) + from src.secret_storage import encrypt + acc["password"] = encrypt(body["password"]) + new_accounts = [acc] + (accounts[1:] if len(accounts) > 1 else []) + _save_caldav_accounts(owner, new_accounts) + return {"ok": True} + + # ── CalDAV multi-account CRUD ───────────────────────────────────────────── + + @router.get("/config/accounts") + async def list_caldav_accounts(request: Request): + """Return all configured CalDAV accounts (passwords never returned).""" + owner = _require_user(request) + accounts = _get_caldav_accounts(owner) + safe = [] + for acc in accounts: + pw = acc.get("password") or "" + has_pw = False + if pw: + try: + from src.secret_storage import decrypt + has_pw = bool(decrypt(pw)) + except Exception: + has_pw = bool(pw) + safe.append({ + "id": acc.get("id", ""), + "label": acc.get("label", "") or acc.get("url", ""), + "url": acc.get("url", "") or "", + "username": acc.get("username", "") or "", + "has_password": has_pw, + }) + return {"accounts": safe} + + @router.post("/config/accounts") + async def add_caldav_account(request: Request): + """Add a new CalDAV account.""" + import uuid as _uuid + owner = _require_user(request) + try: + body = await request.json() + except Exception: + body = {} + from src.caldav_sync import validate_caldav_url + try: + url = validate_caldav_url(body.get("url", "")) + except ValueError as e: + raise HTTPException(400, str(e)) + if not body.get("password"): + raise HTTPException(400, "Password is required") + from src.secret_storage import encrypt + new_acc = { + "id": str(_uuid.uuid4()), + "label": (body.get("label") or "").strip() or "CalDAV", + "url": url, + "username": (body.get("username") or "").strip(), + "password": encrypt(body["password"]), + } + accounts = _get_caldav_accounts(owner) + accounts.append(new_acc) + _save_caldav_accounts(owner, accounts) + return {"ok": True, "id": new_acc["id"]} + + @router.put("/config/accounts/{account_id}") + async def update_caldav_account(account_id: str, request: Request): + """Update an existing CalDAV account by id.""" + owner = _require_user(request) + try: + body = await request.json() + except Exception: + body = {} + accounts = _get_caldav_accounts(owner) + idx = next((i for i, a in enumerate(accounts) if a.get("id") == account_id), None) + if idx is None: + raise HTTPException(404, "Account not found") + acc = dict(accounts[idx]) + if body.get("url"): + from src.caldav_sync import validate_caldav_url + try: + acc["url"] = validate_caldav_url(body["url"]) + except ValueError as e: + raise HTTPException(400, str(e)) + if body.get("label") is not None: + acc["label"] = (body.get("label") or "").strip() or "CalDAV" + if body.get("username") is not None: + acc["username"] = (body.get("username") or "").strip() + if body.get("password"): + from src.secret_storage import encrypt + acc["password"] = encrypt(body["password"]) + accounts[idx] = acc + _save_caldav_accounts(owner, accounts) + return {"ok": True} + + @router.delete("/config/accounts/{account_id}") + async def delete_caldav_account(account_id: str, request: Request): + """Remove a CalDAV account by id.""" + owner = _require_user(request) + accounts = _get_caldav_accounts(owner) + new_accounts = [a for a in accounts if a.get("id") != account_id] + if len(new_accounts) == len(accounts): + raise HTTPException(404, "Account not found") + _save_caldav_accounts(owner, new_accounts) return {"ok": True} @router.post("/test") async def test_connection(request: Request): - """Actually probe the configured CalDAV server with a PROPFIND - request (the same handshake every CalDAV client uses). Accepts - an optional {url, username, password} body so the user can test - a configuration BEFORE saving it; falls back to the stored - creds otherwise. Returns {ok, error?} with a useful message on - failure (status code, auth issue, network error).""" + """Probe a CalDAV server with a PROPFIND. Accepts an optional body: + {url, username, password} to test before saving, or {account_id} to + test an already-saved account. Falls back to the first saved account + when nothing is provided.""" owner = _require_user(request) try: body = await request.json() @@ -452,14 +891,31 @@ async def test_connection(request: Request): user = (body.get("username") or "").strip() pw = body.get("password") or "" if not (url and user and pw): - # Fall back to saved settings for this user. - from routes.prefs_routes import _load_for_user - cfg = (_load_for_user(owner) or {}).get("caldav", {}) or {} - url = url or (cfg.get("url") or "") - user = user or (cfg.get("username") or "") - pw = pw or (cfg.get("password") or "") + # Look up a saved account: by id if supplied, else first account. + accounts = _get_caldav_accounts(owner) + acc = None + if body.get("account_id"): + acc = next((a for a in accounts if a.get("id") == body["account_id"]), None) + if acc is None and accounts: + acc = accounts[0] + if acc: + url = url or (acc.get("url") or "") + user = user or (acc.get("username") or "") + if not pw: + pw = acc.get("password") or "" + if pw: + try: + from src.secret_storage import decrypt + pw = decrypt(pw) + except Exception: + pass if not (url and user and pw): return {"ok": False, "error": "Missing URL, username, or password"} + from src.caldav_sync import validate_caldav_url + try: + url = validate_caldav_url(url) + except ValueError as e: + return {"ok": False, "error": str(e)} import httpx propfind_body = ( '\n' @@ -467,13 +923,42 @@ async def test_connection(request: Request): '' ) try: - async with httpx.AsyncClient(timeout=8.0, follow_redirects=True) as cx: + # Build an SSL context that trusts the operator's custom CA bundle + # (SSL_CERT_FILE / REQUESTS_CA_BUNDLE) so self-signed CalDAV servers + # pass the pre-flight the same way they pass the real sync. + # trust_env=False is kept to block proxy/auth env leakage; the CA + # bundle is loaded explicitly instead. + import ssl as _ssl + _ssl_ctx = _ssl.create_default_context() + # Disable VERIFY_X509_STRICT so certs without a keyUsage extension + # (common in self-signed setups) are accepted, matching the + # requests/urllib3 behavior used by the CalDAV sync path. + _ssl_ctx.verify_flags &= ~_ssl.VERIFY_X509_STRICT + _ca_bundle = _os.environ.get("SSL_CERT_FILE") or _os.environ.get("REQUESTS_CA_BUNDLE") + if _ca_bundle: + if _os.path.isfile(_ca_bundle): + _ssl_ctx.load_verify_locations(_ca_bundle) + else: + logger.warning("CalDAV test: CA bundle %s not found, using system CAs", _ca_bundle) + async with httpx.AsyncClient(timeout=8.0, follow_redirects=False, trust_env=False, verify=_ssl_ctx) as cx: r = await cx.request( "PROPFIND", url, auth=(user, pw), headers={"Depth": "0", "Content-Type": "application/xml"}, content=propfind_body, ) + # If the server demands Digest (Baïkal default, SabreDAV-based + # servers, Radicale with htdigest), the Basic attempt above + # 401s. Retry once with httpx.DigestAuth so this test matches + # what the real sync does via caldav.DAVClient in + # src/caldav_sync.py (which negotiates the scheme). + if r.status_code == 401 and "digest" in r.headers.get("www-authenticate", "").lower(): + r = await cx.request( + "PROPFIND", url, + auth=httpx.DigestAuth(user, pw), + headers={"Depth": "0", "Content-Type": "application/xml"}, + content=propfind_body, + ) # 207 = Multi-Status — standard CalDAV success. 200 also # acceptable. Anything else (401/403/404/5xx) means trouble. if r.status_code in (200, 207): @@ -484,6 +969,8 @@ async def test_connection(request: Request): return {"ok": False, "error": "Forbidden — user can't access that URL"} if r.status_code == 404: return {"ok": False, "error": "Not found — check the URL path"} + if 300 <= r.status_code < 400: + return {"ok": False, "error": "Redirects are not followed for CalDAV safety; use the final URL"} return {"ok": False, "error": f"HTTP {r.status_code}"} except httpx.ConnectError as e: return {"ok": False, "error": f"Connection refused: {e}"[:200]} @@ -493,13 +980,34 @@ async def test_connection(request: Request): return {"ok": False, "error": str(e)[:200]} @router.post("/sync") - async def sync_caldav_endpoint(request: Request): - """Pull events from the configured CalDAV server into local DB. + async def sync_caldav_endpoint(request: Request, direction: str = "pull"): + """Sync events with the configured CalDAV server. Returns counts + any per-calendar errors. Called by the frontend on calendar open and by the periodic scheduler loop.""" owner = _require_user(request) - from src.caldav_sync import sync_caldav - return await sync_caldav(owner) + from src.caldav_sync import sync_caldav_direction + return await sync_caldav_direction(owner, direction) + + + @router.delete("/calendars/{cal_id}") + async def delete_calendar(request: Request, cal_id: str): + owner = _require_user(request) + db = SessionLocal() + try: + cal = _get_or_404_calendar(db, cal_id, owner) + db.query(CalendarEvent).filter(CalendarEvent.calendar_id == cal_id).delete() + db.delete(cal) + db.commit() + return {"ok": True} + except HTTPException: + raise + except Exception as e: + db.rollback() + logger.error("Failed to delete calendar %s: %s", cal_id, e) + raise HTTPException(500, "Failed to delete calendar") + finally: + db.close() + @router.get("/calendars") async def list_calendars(request: Request): @@ -509,7 +1017,7 @@ async def list_calendars(request: Request): _ensure_default_calendar(db, owner) cals = db.query(CalendarCal).filter(CalendarCal.owner == owner).all() return {"calendars": [ - {"name": c.name, "href": c.id, "color": c.color} + {"name": c.name, "href": c.id, "color": c.color, "source": c.source} for c in cals ]} except HTTPException: @@ -535,11 +1043,29 @@ async def list_events(request: Request, start: str, end: str, calendar: str = "" db = SessionLocal() try: # Scope events to calendars owned by the caller. + # Non-recurring events must overlap the query window; recurring + # events (with RRULE) whose base dtstart is before the window end + # are fetched so their actual occurrences can be expanded + # server-side and appear in every year they repeat, not just the + # DTSTART year. q = db.query(CalendarEvent).join(CalendarCal).filter( - CalendarEvent.dtstart < end_dt, - CalendarEvent.dtend > start_dt, CalendarEvent.status != "cancelled", CalendarCal.owner == owner, + or_( + # Non-recurring: event times must overlap the query window + and_( + or_(CalendarEvent.rrule == "", CalendarEvent.rrule.is_(None)), + CalendarEvent.dtstart < end_dt, + CalendarEvent.dtend > start_dt, + ), + # Recurring: dtstart before window end — RRULE expansion + # generates the actual occurrences within the window + and_( + CalendarEvent.rrule.isnot(None), + CalendarEvent.rrule != "", + CalendarEvent.dtstart < end_dt, + ), + ), ) if calendar: q = q.filter( @@ -547,7 +1073,19 @@ async def list_events(request: Request, start: str, end: str, calendar: str = "" (CalendarCal.name == calendar) ) events = q.order_by(CalendarEvent.dtstart).all() - return {"events": [_event_to_dict(e) for e in events]} + + # Expand recurring events into individual occurrences. + expanded = [] + for e in events: + expanded.extend(_expand_rrule(e, start_dt, end_dt)) + + # Sort by occurrence start time for consistent frontend ordering. + truncated = any(e.get("truncated") for e in expanded) + expanded.sort(key=lambda d: d["dtstart"]) + response: dict = {"events": expanded} + if truncated: + response["truncated"] = True + return response except HTTPException: raise except Exception as e: @@ -559,6 +1097,7 @@ async def list_events(request: Request, start: str, end: str, calendar: str = "" @router.post("/events") async def create_event(request: Request, data: EventCreate): owner = _require_user(request) + _reserve_calendar_uploads(request, data.color, data.description, data.location) db = SessionLocal() try: cal = None @@ -601,9 +1140,12 @@ async def create_event(request: Request, data: EventCreate): is_utc=_is_utc and not data.all_day, rrule=data.rrule or "", color=data.color or None, + caldav_sync_pending="create" if cal.source == "caldav" else None, ) db.add(ev) db.commit() + if cal.source == "caldav": + await _push_caldav_event_after_commit(owner, uid, "create") return {"ok": True, "uid": uid} except HTTPException: raise @@ -617,9 +1159,14 @@ async def create_event(request: Request, data: EventCreate): @router.put("/events/{uid}") async def update_event(request: Request, uid: str, data: EventUpdate): owner = _require_user(request) + _reserve_calendar_uploads(request, data.color, data.description, data.location) + try: + base_uid = _resolve_base_uid(uid) + except ValueError as e: + raise HTTPException(400, str(e)) db = SessionLocal() try: - ev = _get_or_404_event(db, uid, owner) + ev = _get_or_404_event(db, base_uid, owner) if data.summary is not None: ev.summary = data.summary if data.description is not None: @@ -645,7 +1192,12 @@ async def update_event(request: Request, uid: str, data: EventUpdate): ev.rrule = data.rrule if data.color is not None: ev.color = data.color if data.color else None + is_caldav = ev.calendar and ev.calendar.source == "caldav" + if is_caldav: + ev.caldav_sync_pending = "update" db.commit() + if is_caldav: + await _push_caldav_event_after_commit(owner, base_uid, "update") return {"ok": True} except HTTPException: raise @@ -657,13 +1209,37 @@ async def update_event(request: Request, uid: str, data: EventUpdate): db.close() @router.delete("/events/{uid}") - async def delete_event(request: Request, uid: str): + async def delete_event(request: Request, uid: str, scope: str = "series"): owner = _require_user(request) + try: + base_uid = _resolve_base_uid(uid) + except ValueError as e: + raise HTTPException(400, str(e)) db = SessionLocal() try: - ev = _get_or_404_event(db, uid, owner) + ev = _get_or_404_event(db, base_uid, owner) + is_occurrence_delete = scope in {"occurrence", "instance"} and "::" in uid and bool(ev.rrule) + is_caldav = ev.calendar and ev.calendar.source == "caldav" + if is_occurrence_delete: + key = _occurrence_exdate_key(uid, ev) + if not key: + raise HTTPException(400, "Invalid recurring occurrence uid") + exdates = _recurrence_exdates(ev) + if key not in exdates: + exdates.append(key) + ev.recurrence_exdates = json.dumps(sorted(exdates)) + if is_caldav: + ev.caldav_sync_pending = "update" + db.commit() + if is_caldav: + await _push_caldav_event_after_commit(owner, base_uid, "update") + return {"ok": True, "scope": "occurrence", "exdate": key} + if is_caldav: + _record_caldav_delete_tombstone(db, ev, owner) db.delete(ev) db.commit() + if is_caldav: + await _push_caldav_event_after_commit(owner, base_uid, "delete") return {"ok": True} except HTTPException: raise @@ -677,6 +1253,7 @@ async def delete_event(request: Request, uid: str): @router.post("/calendars") async def create_calendar(request: Request, name: str = "Imported", color: str = "#5b8abf"): owner = _require_user(request) + _reserve_calendar_uploads(request, color) db = SessionLocal() try: cal = CalendarCal( @@ -699,6 +1276,7 @@ async def create_calendar(request: Request, name: str = "Imported", color: str = @router.put("/calendars/{cal_id}") async def update_calendar(request: Request, cal_id: str, name: str = None, color: str = None): owner = _require_user(request) + _reserve_calendar_uploads(request, color) db = SessionLocal() try: cal = _get_or_404_calendar(db, cal_id, owner) @@ -717,27 +1295,10 @@ async def update_calendar(request: Request, cal_id: str, name: str = None, color finally: db.close() - @router.delete("/calendars/{cal_id}") - async def delete_calendar(request: Request, cal_id: str): - owner = _require_user(request) - db = SessionLocal() - try: - cal = _get_or_404_calendar(db, cal_id, owner) - db.query(CalendarEvent).filter(CalendarEvent.calendar_id == cal_id).delete() - db.delete(cal) - db.commit() - return {"ok": True} - except HTTPException: - raise - except Exception as e: - db.rollback() - return {"error": str(e)} - finally: - db.close() - # 10 MB hard cap on ICS upload. Loading the whole file into memory is - # unavoidable with python-icalendar, so an unbounded upload would OOM. - _ICS_MAX_BYTES = 10 * 1024 * 1024 + # Hard cap on ICS upload (ICS_MAX_BYTES, default 10 MB). Loading the whole + # file into memory is unavoidable with python-icalendar, so an unbounded + # upload would OOM. @router.post("/import") async def import_ics(request: Request, file: UploadFile = File(...), calendar_name: str = ""): @@ -747,9 +1308,7 @@ async def import_ics(request: Request, file: UploadFile = File(...), calendar_na owner = _require_user(request) db = SessionLocal() try: - content = await file.read() - if len(content) > _ICS_MAX_BYTES: - raise HTTPException(413, f"ICS file too large (max {_ICS_MAX_BYTES // (1024*1024)} MB)") + content = await read_upload_limited(file, ICS_MAX_BYTES, "ICS file") try: cal_data = iCal.from_ical(content) except Exception as e: @@ -775,7 +1334,7 @@ async def import_ics(request: Request, file: UploadFile = File(...), calendar_na db.commit() db.refresh(target_cal) - imported = skipped = 0 + imported = skipped = repaired = 0 for comp in cal_data.walk(): if comp.name != "VEVENT": continue @@ -795,7 +1354,12 @@ async def import_ics(request: Request, file: UploadFile = File(...), calendar_na source_uid = str(comp.get("uid", "")) or None if source_uid: src_dtstart = dtstart.dt - naive_src = src_dtstart.replace(tzinfo=None) if hasattr(src_dtstart, 'tzinfo') and src_dtstart.tzinfo else src_dtstart + # Normalize to the SAME naive form import_ics stores, so a + # re-import of a tz-aware event matches the existing row. + # The old code stripped tzinfo WITHOUT converting to UTC + # (wall clock), while storage converts to UTC first, so + # every re-import of a TZID event created a duplicate. + naive_src = _ics_naive_dtstart(src_dtstart) existing = ( db.query(CalendarEvent) .filter( @@ -806,6 +1370,18 @@ async def import_ics(request: Request, file: UploadFile = File(...), calendar_na .first() ) if existing: + # An import predating the clamp below may have stored + # this same event with a non-positive duration, which + # the list_events overlap filter hides. Re-importing + # lands here and would skip without touching that row, + # so the event would stay invisible. Backfill the clamp + # onto the stored row before skipping it. + fixed_end = _ensure_positive_duration( + existing.dtstart, existing.dtend, bool(existing.all_day) + ) + if fixed_end != existing.dtend: + existing.dtend = fixed_end + repaired += 1 skipped += 1 continue @@ -839,6 +1415,8 @@ async def import_ics(request: Request, file: UploadFile = File(...), calendar_na else: end_dt = start_dt + timedelta(hours=1) + end_dt = _ensure_positive_duration(start_dt, end_dt, all_day) + ev = CalendarEvent( uid=uid_val, calendar_id=target_cal.id, @@ -859,6 +1437,7 @@ async def import_ics(request: Request, file: UploadFile = File(...), calendar_na "ok": True, "imported": imported, "skipped": skipped, + "repaired": repaired, "calendar": cal_display, "calendar_id": target_cal.id, } @@ -889,34 +1468,37 @@ async def export_ics(request: Request, cal_id: str): "BEGIN:VCALENDAR", "VERSION:2.0", "PRODID:-//Odysseus//Calendar//EN", - f"X-WR-CALNAME:{cal.name}", + f"X-WR-CALNAME:{_ics_escape(cal.name)}", ] for ev in events: lines.append("BEGIN:VEVENT") lines.append(f"UID:{ev.uid}") - lines.append(f"SUMMARY:{ev.summary or ''}") + lines.append(f"SUMMARY:{_ics_escape(ev.summary or '')}") if ev.all_day: lines.append(f"DTSTART;VALUE=DATE:{ev.dtstart.strftime('%Y%m%d')}") lines.append(f"DTEND;VALUE=DATE:{ev.dtend.strftime('%Y%m%d')}") else: - lines.append(f"DTSTART:{ev.dtstart.strftime('%Y%m%dT%H%M%S')}") - lines.append(f"DTEND:{ev.dtend.strftime('%Y%m%dT%H%M%S')}") + _dt_suffix = "Z" if getattr(ev, "is_utc", False) else "" + lines.append(f"DTSTART:{ev.dtstart.strftime('%Y%m%dT%H%M%S')}{_dt_suffix}") + lines.append(f"DTEND:{ev.dtend.strftime('%Y%m%dT%H%M%S')}{_dt_suffix}") if ev.description: - escaped_desc = ev.description.replace(chr(10), "\\n") - lines.append(f"DESCRIPTION:{escaped_desc}") + lines.append(f"DESCRIPTION:{_ics_escape(ev.description)}") if ev.location: - lines.append(f"LOCATION:{ev.location}") + lines.append(f"LOCATION:{_ics_escape(ev.location)}") if ev.rrule: lines.append(f"RRULE:{ev.rrule}") lines.append("END:VEVENT") lines.append("END:VCALENDAR") ics_data = "\r\n".join(lines) - safe_name = cal.name.replace(" ", "_").replace("/", "_") + download_name = _safe_ics_filename(cal.name) return Response( content=ics_data, media_type="text/calendar", - headers={"Content-Disposition": f'attachment; filename="{safe_name}.ics"'}, + headers={ + "Content-Disposition": f'attachment; filename="{download_name}"', + "X-Content-Type-Options": "nosniff", + }, ) except HTTPException: raise @@ -938,7 +1520,7 @@ async def quick_parse(request: Request): "tomorrow", "next Tuesday", "in 30 minutes" resolve correctly. Uses the "utility" endpoint (small / fast model) to keep latency low. """ - _require_user(request) + owner = _require_user(request) from src.endpoint_resolver import resolve_endpoint from src.llm_core import llm_call_async from src.text_helpers import strip_think @@ -949,23 +1531,36 @@ async def quick_parse(request: Request): text = (body.get("text") or "").strip() if not text: raise HTTPException(400, "text is required") + from src.user_time import ( + clear_user_time_context, + current_datetime_prompt, + now_user_local, + set_user_tz_name, + set_user_tz_offset, + ) + + clear_user_time_context() tz_hint = (body.get("tz") or "").strip() + if body.get("tz_offset") is not None: + set_user_tz_offset(body.get("tz_offset")) + if tz_hint: + set_user_tz_name(tz_hint) - url, model, headers = resolve_endpoint("utility") + url, model, headers = resolve_endpoint("utility", owner=owner or None) if not url: - url, model, headers = resolve_endpoint("default") + url, model, headers = resolve_endpoint("default", owner=owner or None) if not url or not model: return {"ok": False, "error": "No LLM endpoint configured"} - now = datetime.now() + now = now_user_local() now_iso = now.strftime("%Y-%m-%dT%H:%M:%S") # The model gets only the schema it needs to fill out; we re-validate # everything client-side too. system_prompt = ( - "You are a calendar event parser. Read the user's one-line " + current_datetime_prompt() + + "You are a calendar event parser. Read the user's one-line " "description and emit STRICT JSON describing the event. " - f"Today is {now.strftime('%A, %Y-%m-%d')} ({now_iso}). " - + (f"User timezone: {tz_hint}. " if tz_hint else "") + f"The current user-local timestamp is {now_iso}. " + "Resolve relative dates (\"tomorrow\", \"friday\", \"next monday\", " "\"in 30 minutes\") against today. Default duration is 60 minutes " "when no end time is given. If the text mentions a date with no " diff --git a/routes/chat_helpers.py b/routes/chat_helpers.py index ce2e0cfd0..63c8abc8f 100644 --- a/routes/chat_helpers.py +++ b/routes/chat_helpers.py @@ -3,6 +3,7 @@ import asyncio import json import logging +import os import re from dataclasses import dataclass, field from typing import Any, Optional @@ -11,15 +12,59 @@ from core.database import SessionLocal from core.database import Session as DBSession, ModelEndpoint from src.llm_core import normalize_model_id +from src.endpoint_resolver import normalize_base from src.context_compactor import maybe_compact, trim_for_context -from src.auth_helpers import get_current_user +from src.model_context import estimate_tokens +from src.auth_helpers import effective_user from src.prompt_security import untrusted_context_message +from src.attachment_refs import attachment_ref from routes.prefs_routes import _load_for_user as load_prefs_for_user from fastapi import HTTPException logger = logging.getLogger(__name__) +_CASUAL_OPENING_RE = re.compile( + r"^\s*(?:h+i+|hey+|hello+|yo+|sup+|what'?s up|wass?up|hiya|howdy|" + r"lol|lmao|haha+|hehe+|thanks?|thank you|ty|idk|dunno|meh|bruh|bro)\b(?P.*)$", + re.IGNORECASE, +) +_CASUAL_BLOCKLIST_RE = re.compile( + r"\b(?:cookbook|serve|serving|launch|start|vllm|sglang|llama\.?cpp|ollama|" + r"download|model|email|document|doc|note|calendar|task|search|web|research|" + r"file|folder|repo|git|settings?|endpoint|api|token|mcp)\b", + re.IGNORECASE, +) + + +def _is_casual_low_signal(text: str) -> bool: + """Short greetings/slang should not pull memory, skills, RAG, or docs.""" + s = str(text or "").strip() + m = _CASUAL_OPENING_RE.match(s) + if not m: + return False + tail = m.group("tail") or "" + if _CASUAL_BLOCKLIST_RE.search(tail): + return False + tail_words = re.findall(r"[A-Za-z0-9_'-]+", tail) + return len(tail_words) <= 2 + + +# Strong references to in-flight fire-and-forget tasks scheduled from this +# module. asyncio only keeps weak references to tasks created via +# create_task, so without this the GC can collect a task mid-execution and +# the background work (extraction, auto-naming) silently never runs. +# Mirrors WebhookManager._spawn_tracked from src/webhook_manager.py. +_BG_TASKS: set[asyncio.Task] = set() + + +def _spawn_bg(coro) -> asyncio.Task: + """Schedule a background task and hold a strong reference until it finishes.""" + task = asyncio.create_task(coro) + _BG_TASKS.add(task) + task.add_done_callback(_BG_TASKS.discard) + return task + # ── Data containers ────────────────────────────────────────────────────── # @@ -56,11 +101,19 @@ class ChatContext: uprefs: dict preset: PresetInfo preprocessed: PreprocessedMessage + context_trimmed: bool = False + context_messages_before_trim: int = 0 + context_messages_after_trim: int = 0 + context_tokens_before_trim: int = 0 + context_tokens_after_trim: int = 0 # Documents auto-created server-side during preprocess (e.g. when an # attached fillable PDF gets rendered into a markdown editor doc). # The chat route emits a doc_update SSE event for each before streaming # begins, so the editor pane switches to the new doc immediately. auto_opened_docs: list = field(default_factory=list) + # Uploads attached to this user turn, resolved and owner-checked for the + # agent's private context. This is not emitted to the browser. + uploaded_files: list = field(default_factory=list) # ── Helpers ────────────────────────────────────────────────────────────── # @@ -73,10 +126,10 @@ def _enforce_chat_privileges(request, sess) -> None: allowlist, or HTTPException(429) if the user has hit their daily message cap. No-op for unauthenticated callers or when auth_manager is absent (single-user mode). Admins receive ADMIN_PRIVILEGES from get_privileges, - which means empty allowed_models / zero cap → no-op for them. + which means unrestricted allowed_models / zero cap -> no-op for them. """ try: - user = get_current_user(request) + user = effective_user(request) except Exception: user = None if not user: @@ -86,8 +139,18 @@ def _enforce_chat_privileges(request, sess) -> None: return privs = auth_manager.get_privileges(user) or {} - allowed = privs.get("allowed_models") or [] - if allowed and sess.model and sess.model not in allowed: + + # Explicit "block everything" sentinel takes precedence over the + # allowlist — it's the only way to distinguish "user clicked [None]" + # (block all) from "user clicked [All]" (no restriction), since both + # otherwise produce an empty `allowed_models` list. + if privs.get("block_all_models"): + raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.") + + allowed_raw = privs.get("allowed_models") + allowed = allowed_raw if isinstance(allowed_raw, list) else [] + restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed) + if restricted and sess.model and sess.model not in allowed: raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.") cap = int(privs.get("max_messages_per_day") or 0) @@ -119,7 +182,7 @@ def needs_auto_name(name: str) -> bool: if name.startswith("Chat:") or name == "Chat": return True # Default frontend name: "modelname HH:MM:SS AM/PM" - if re.match(r'^.+ \d{1,2}:\d{2}:\d{2}\s*(AM|PM)$', name): + if re.match(r"^.+ \d{1,2}:\d{2}:\d{2}(\s*(AM|PM))?$", name, re.IGNORECASE): return True return False @@ -146,9 +209,13 @@ async def auto_name_session(session_manager, sess): if not first_msg: return + owner = getattr(sess, "owner", None) t_url, t_model, t_headers = resolve_task_endpoint( - sess.endpoint_url, sess.model, sess.headers, + sess.endpoint_url, sess.model, sess.headers, owner=owner ) + if not t_model: + logger.debug("[auto-name] No model provided, skipping") + return # max_tokens big enough that reasoning models (Minimax M2, # DeepSeek R1, QwQ, etc.) have headroom for @@ -188,14 +255,26 @@ def try_fallback_endpoint(sess, session_id: str) -> dict | None: Returns {"model": ..., "endpoint_url": ..., "endpoint_name": ...} or None. """ import requests as _req - from src.endpoint_resolver import build_chat_url, build_headers, normalize_base + from src.endpoint_resolver import ( + build_chat_url, + build_headers, + build_models_url, + normalize_base, + resolve_endpoint_runtime, + ) + from src.chatgpt_subscription import is_chatgpt_subscription_base current_url = sess.endpoint_url or "" + owner = getattr(sess, "owner", None) db = SessionLocal() try: - endpoints = db.query(ModelEndpoint).filter( + q = db.query(ModelEndpoint).filter( ModelEndpoint.is_enabled == True - ).all() + ) + if owner: + from src.auth_helpers import owner_filter + q = owner_filter(q, ModelEndpoint, owner) + endpoints = q.all() finally: db.close() @@ -204,22 +283,33 @@ def try_fallback_endpoint(sess, session_id: str) -> dict | None: # Skip current endpoint if current_url and base in current_url: continue - # Quick ping - ping_url = base + "/models" - headers = {} - if ep.api_key: - headers["Authorization"] = f"Bearer {ep.api_key}" try: - r = _req.get(ping_url, headers=headers, timeout=5) - r.raise_for_status() - data = r.json() - models = [m.get("id") for m in (data.get("data") or []) if m.get("id")] + base, api_key = resolve_endpoint_runtime(ep, owner=owner) + except Exception: + continue + ping_url = build_models_url(base) + headers = build_headers(api_key, base) + try: + if ping_url: + r = _req.get(ping_url, headers=headers, timeout=5) + r.raise_for_status() + data = r.json() + models = [m.get("id") for m in (data.get("data") or []) if m.get("id")] + if not models: + models = [ + m.get("name") or m.get("model") + for m in (data.get("models") or []) + if m.get("name") or m.get("model") + ] + else: + models = json.loads(ep.cached_models or "[]") if not models: continue # Found a working endpoint — update session new_model = models[0] chat_url = build_chat_url(base) - new_headers = build_headers(ep.api_key, base) + new_headers = build_headers(api_key, base) + persisted_headers = {} if is_chatgpt_subscription_base(base) else new_headers sess.model = new_model sess.endpoint_url = chat_url @@ -231,7 +321,7 @@ def try_fallback_endpoint(sess, session_id: str) -> dict | None: _db.query(DBSession).filter(DBSession.id == session_id).update({ "model": new_model, "endpoint_url": chat_url, - "headers": json.dumps(new_headers), + "headers": persisted_headers, }) _db.commit() finally: @@ -265,11 +355,16 @@ def extract_preset(chat_handler, preset_id) -> PresetInfo: async def preprocess( chat_handler, message, att_ids, sess, auto_opened_docs: Optional[list] = None, + allow_tool_preprocessing: bool = True, ) -> PreprocessedMessage: """Run chat_handler.preprocess_message and wrap the result.""" enhanced, user_content, text_ctx, yt_transcripts, att_meta = ( await chat_handler.preprocess_message( - message, att_ids, sess, auto_opened_docs=auto_opened_docs + message, + att_ids, + sess, + auto_opened_docs=auto_opened_docs, + allow_tool_preprocessing=allow_tool_preprocessing, ) ) return PreprocessedMessage( @@ -281,6 +376,62 @@ async def preprocess( ) +def build_uploaded_file_manifest(att_ids: list, upload_handler, owner: Optional[str]) -> list[dict]: + """Resolve current-turn upload IDs into a small tool-facing manifest. + + The chat UI already sends attachment ids, and preprocessing inlines as much + text as fits. Agent mode still needs a discoverable bridge for files whose + content was truncated/omitted or when the model chooses file tools. Only + owner-authorized uploads are included, and paths must remain inside the + configured upload directory. + """ + if not att_ids or not upload_handler or not hasattr(upload_handler, "resolve_upload"): + return [] + + def _read_file_can_open(path: str) -> bool: + try: + from src.tool_execution import _resolve_tool_path + + return _resolve_tool_path(path) == os.path.realpath(path) + except Exception: + return False + + manifest: list[dict] = [] + for att_id in att_ids: + try: + info = upload_handler.resolve_upload(str(att_id), owner=owner) + except Exception: + logger.debug("Failed to resolve upload %r for agent manifest", att_id, exc_info=True) + continue + if not isinstance(info, dict): + continue + + path = info.get("path") + if path: + try: + inside = True + if hasattr(upload_handler, "_inside_upload_dir"): + inside = bool(upload_handler._inside_upload_dir(path)) + elif hasattr(upload_handler, "inside_base_dir"): + inside = bool(upload_handler.inside_base_dir(path)) + if not inside or not os.path.exists(path) or not _read_file_can_open(path): + path = None + except Exception: + path = None + + ref = attachment_ref({**info, "id": info.get("id") or str(att_id)}) + ref.update({ + "id": ref["attachment_id"], + "uri": f"odysseus://attachment/{ref['attachment_id']}", + "read_policy": "owner_checked_upload", + # Transitional compatibility: existing built-in tools can still use + # this path, but only after owner, upload-root, and tool-root checks. + "path": path, + }) + manifest.append(ref) + return manifest + + def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, incognito: bool = False): """Add user message to session history and update session name. In incognito mode, still add to in-memory history (for conversation context) @@ -294,42 +445,188 @@ def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, inco def fire_message_event(request, webhook_manager, session_id: str, sess, message: str, compare_mode: bool = False): """Fire webhook and event_bus events for a new user message.""" if webhook_manager and not compare_mode: - asyncio.create_task(webhook_manager.fire("chat.message", { + webhook_manager.fire_and_forget("chat.message", { "session_id": session_id, "model": sess.model, "message": message[:2000], - })) + }) from src.event_bus import fire_event - user = get_current_user(request) + user = effective_user(request) fire_event("message_sent", user) -def resolve_session_auth(sess, session_id: str): - """Ensure session has auth headers — resolve from endpoint DB if missing.""" - has_auth = sess.headers and isinstance(sess.headers, dict) and any( - k.lower() in ('authorization', 'x-api-key') for k in sess.headers +def _session_url_matches_endpoint(session_url: str, endpoint_base: str) -> bool: + if not session_url or not endpoint_base: + return False + try: + from src.endpoint_resolver import build_chat_url, normalize_base + + sess_url = session_url.rstrip("/") + base = normalize_base(endpoint_base).rstrip("/") + return sess_url in { + base, + base + "/chat/completions", + build_chat_url(base).rstrip("/"), + } + except Exception: + return False + + +def _has_auth_keys(headers) -> bool: + """True if a headers dict carries an Authorization/x-api-key entry.""" + return isinstance(headers, dict) and any( + k.lower() in ('authorization', 'x-api-key') for k in headers ) - if has_auth: + + +def resolve_session_auth(sess, session_id: str, owner: Optional[str] = None): + """Ensure session has auth headers — resolve from endpoint DB if missing.""" + try: + from src.chatgpt_subscription import is_chatgpt_subscription_base + is_chatgpt_subscription = is_chatgpt_subscription_base(getattr(sess, "endpoint_url", "") or "") + except Exception: + is_chatgpt_subscription = False + has_auth = _has_auth_keys(sess.headers) + if has_auth and not is_chatgpt_subscription: return try: - from src.endpoint_resolver import build_headers + from src.endpoint_resolver import build_headers, resolve_endpoint_runtime db = SessionLocal() try: - domain = sess.endpoint_url.split("//")[1].split("/")[0] if "//" in sess.endpoint_url else "" - if domain: - ep = db.query(ModelEndpoint).filter(ModelEndpoint.base_url.contains(domain)).first() - if ep and ep.api_key: - sess.headers = build_headers(ep.api_key, ep.base_url) - db.query(DBSession).filter(DBSession.id == session_id).update( - {"headers": json.dumps(sess.headers)} - ) - db.commit() - logger.info(f"Resolved and persisted auth headers for session {session_id} from endpoint {ep.name}") + target_url = getattr(sess, "endpoint_url", "") or "" + if not target_url: + return + q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) + if owner: + # Missing headers usually means "recover from the saved endpoint". + # Scope that lookup to the session owner, otherwise two users + # with similar endpoint URLs can borrow each other's API key. + from src.auth_helpers import owner_filter + q = owner_filter(q, ModelEndpoint, owner) + for ep in q.all(): + if not _session_url_matches_endpoint(target_url, ep.base_url or ""): + continue + try: + base, api_key = resolve_endpoint_runtime(ep, owner=owner) + except Exception as e: + logger.warning("Failed to resolve provider auth for session %s: %s", session_id, e) + return + if not api_key: + # No usable key (e.g. ChatGPT Subscription needs re-auth). + return + sess.headers = build_headers(api_key, base) + if is_chatgpt_subscription: + # The bearer is short-lived and re-resolved per request, so it + # stays request-local and is never written to the plaintext + # sessions.headers column. Proactively strip any bearer an + # older code path may have persisted so it does not linger. + stale_q = db.query(DBSession).filter(DBSession.id == session_id) + if owner: + stale_q = stale_q.filter(DBSession.owner == owner) + stored = stale_q.first() + if stored is not None and _has_auth_keys(stored.headers): + stale_q.update({"headers": {}}) + db.commit() + logger.info(f"Cleared persisted ChatGPT Subscription bearer from session {session_id}") + logger.debug(f"Resolved request-local ChatGPT Subscription auth for session {session_id}") + return + update_q = db.query(DBSession).filter(DBSession.id == session_id) + if owner: + update_q = update_q.filter(DBSession.owner == owner) + update_q.update({"headers": sess.headers}) + db.commit() + logger.info(f"Resolved and persisted auth headers for session {session_id} from endpoint {ep.name}") + return finally: db.close() except Exception as e: logger.warning(f"Failed to resolve session headers: {e}") +def _match_cached_model_id(requested: str, models) -> Optional[str]: + if not requested or not models: + return None + model_ids = [str(m) for m in models if m] + if requested in model_ids: + return requested + + req_base = os.path.basename(requested.rstrip("/")) + for model_id in model_ids: + if os.path.basename(model_id.rstrip("/")) == req_base: + return model_id + return None + + +def _normalize_model_id_from_cache(sess) -> Optional[str]: + """Use stored endpoint model IDs before falling back to a live /models probe.""" + endpoint_url = getattr(sess, "endpoint_url", "") or "" + requested = getattr(sess, "model", "") or "" + if not endpoint_url or not requested: + return None + + try: + session_base = normalize_base(endpoint_url) + except Exception: + session_base = endpoint_url.rstrip("/") + if not session_base: + return None + + db = SessionLocal() + try: + q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) + owner = getattr(sess, "owner", None) + if owner: + from src.auth_helpers import owner_filter + q = owner_filter(q, ModelEndpoint, owner) + endpoints = q.all() + for ep in endpoints: + try: + if normalize_base(getattr(ep, "base_url", "") or "") != session_base: + continue + except Exception: + continue + + raw_models = getattr(ep, "cached_models", None) + if not raw_models: + continue + try: + models = json.loads(raw_models) if isinstance(raw_models, str) else raw_models + except Exception: + continue + + matched = _match_cached_model_id(requested, models) + if matched: + return matched + except Exception as e: + logger.debug("Cached model normalization skipped: %s", e) + finally: + db.close() + + return None + + +def _session_is_research_spinoff(sess) -> bool: + """True if this session was created via research "Discuss" spin-off. + + Detected by the primer system message the spin-off endpoint seeds into + history (metadata ``research_spinoff_from``). Such sessions are grounded + on the seeded report, so global memory + personal-doc RAG injection is + suppressed for them (the report is the sole knowledge base). Handles both + ChatMessage objects and plain dicts. + """ + for m in getattr(sess, "history", []) or []: + role = getattr(m, "role", None) + if role is None and isinstance(m, dict): + role = m.get("role") + if role != "system": + continue + md = getattr(m, "metadata", None) + if md is None and isinstance(m, dict): + md = m.get("metadata") + if (md or {}).get("research_spinoff_from"): + return True + return False + + async def build_chat_context( sess, request, @@ -350,6 +647,7 @@ async def build_chat_context( webhook_manager=None, use_enhanced_message: bool = False, agent_mode: bool = False, + allow_tool_preprocessing: bool = True, ) -> ChatContext: """Build the full context (preface + messages) for an LLM call. @@ -367,6 +665,7 @@ async def build_chat_context( preprocessed = await preprocess( chat_handler, message, att_ids or [], sess, auto_opened_docs=auto_opened_docs, + allow_tool_preprocessing=allow_tool_preprocessing, ) # Add user message to history @@ -376,27 +675,48 @@ async def build_chat_context( if not incognito: fire_message_event(request, webhook_manager, session_id, sess, message, compare_mode) - # Resolve user prefs - user = get_current_user(request) + # Resolve owner-scoped prefs/context. Browser requests keep the cookie user; + # bearer-token chat requests use the token owner instead of the "api" sentinel. + user = effective_user(request) uprefs = load_prefs_for_user(user) + uploaded_files = build_uploaded_file_manifest( + att_ids or [], + getattr(chat_handler, "upload_handler", None), + getattr(sess, "owner", None), + ) + casual_low_signal = _is_casual_low_signal(message) # Memory enabled? mem_enabled = not incognito and not no_memory and uprefs.get("memory_enabled", True) # Skills injection respects its own enable toggle (mirrors memory_enabled). # When off, the "Available skills" index is not added to the prompt. skills_enabled = not incognito and uprefs.get("skills_enabled", True) + if not allow_tool_preprocessing: + mem_enabled = False + skills_enabled = False + if casual_low_signal: + mem_enabled = False + skills_enabled = False logger.debug( "Memory enabled=%s for user=%s (incognito=%s, no_memory=%s, pref=%s)", mem_enabled, user, incognito, no_memory, uprefs.get("memory_enabled", "NOT_SET"), ) + # Research-spinoff ("Discuss") sessions are grounded on the seeded report: + # the primer system message IS the knowledge base. Injecting global memory + # or personal-doc RAG on every turn pulls in keyword-matched but off-topic + # facts ("wrong data") and competes with the report, so suppress both here. + is_research_spinoff = _session_is_research_spinoff(sess) + if is_research_spinoff: + mem_enabled = False + # Use RAG? use_rag_val = (str(use_rag).lower() != "false") if use_rag is not None else True - if incognito: + if incognito or not allow_tool_preprocessing or is_research_spinoff or casual_low_signal: use_rag_val = False # If pre-fetched search context was provided (compare mode), skip live web search - skip_web = bool(search_context) + skip_web = bool(search_context) or not allow_tool_preprocessing or casual_low_signal # Build context preface # The stream path uses enhanced_message (with CoT/preprocessing applied), @@ -415,7 +735,7 @@ async def build_chat_context( incognito=incognito, use_skills=skills_enabled, ) - if use_rag is not None: + if use_rag is not None or is_research_spinoff or casual_low_signal: _preface_kwargs["use_rag"] = use_rag_val preface, rag_sources, web_sources = chat_processor.build_context_preface(**_preface_kwargs) @@ -423,26 +743,56 @@ async def build_chat_context( used_memories = getattr(chat_processor, '_last_used_memories', []) # Inject pre-fetched search context (compare mode) - if search_context: + if search_context and allow_tool_preprocessing and not casual_low_signal: preface.append(untrusted_context_message("prefetched search context", search_context)) # YouTube transcripts for transcript in preprocessed.youtube_transcripts: preface.append(untrusted_context_message("youtube transcript", transcript)) - # Normalize model ID - norm = normalize_model_id(sess.endpoint_url, sess.model) + # Normalize model ID. Prefer cached endpoint models so group chat does not + # re-hit slow local /models endpoints on every participant turn. + norm = _normalize_model_id_from_cache(sess) or normalize_model_id( + sess.endpoint_url, + sess.model, + owner=getattr(sess, "owner", None), + ) if norm: sess.model = norm # Build messages messages = preface + sess.get_context_messages() + # Current date/time — injected as a standalone *user*-role context message + # placed immediately before the latest user turn, NOT folded into the + # system prompt. Its text changes every minute, and local OpenAI-compatible + # backends (llama.cpp / LM Studio) key their KV-cache prefix off the + # system message byte-for-byte; mixing ever-changing timestamp text into + # it would invalidate the cached prefix on every request (issue #2927). + # Placing it at the tail also keeps it out of the stable + # preface+history prefix, so that prefix stays byte-identical turn over + # turn (modulo the genuinely new history entries) and the cache survives. + if not agent_mode: + try: + from src.user_time import current_datetime_context_message + _dt_msg = current_datetime_context_message() + if messages and messages[-1].get("role") == "user": + messages.insert(len(messages) - 1, _dt_msg) + else: + messages.append(_dt_msg) + except Exception: + logger.debug("Failed to add current date/time context", exc_info=True) + # Auto-compact messages, context_length, was_compacted = await maybe_compact( - sess, sess.endpoint_url, sess.model, messages, sess.headers, + sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user, ) + _before_trim_messages = len(messages) + _before_trim_tokens = estimate_tokens(messages) messages = trim_for_context(messages, context_length) + _after_trim_messages = len(messages) + _after_trim_tokens = estimate_tokens(messages) + _context_trimmed = _after_trim_messages < _before_trim_messages or _after_trim_tokens < _before_trim_tokens return ChatContext( preface=preface, @@ -456,7 +806,13 @@ async def build_chat_context( uprefs=uprefs, preset=preset, preprocessed=preprocessed, + context_trimmed=_context_trimmed, + context_messages_before_trim=_before_trim_messages, + context_messages_after_trim=_after_trim_messages, + context_tokens_before_trim=_before_trim_tokens, + context_tokens_after_trim=_after_trim_tokens, auto_opened_docs=auto_opened_docs, + uploaded_files=uploaded_files, ) @@ -490,6 +846,8 @@ def _normalize_thinking(text: str) -> str: import re if not text: return text + from src.text_helpers import normalize_thinking_markup + text = normalize_thinking_markup(text) reasoning_prefix_re = re.compile( r'^\s*(?:thinking(?:\s+process)?\s*:|the user |i need |i should |i will |they are |the question |i can )', re.IGNORECASE, @@ -600,6 +958,10 @@ def _extract_thinking_meta(text: str) -> dict | None: import re if not text: return None + from src.text_helpers import normalize_thinking_markup + original_text = text + text = normalize_thinking_markup(text) + normalized_changed = text != original_text # Check for tags (native or injected) time_match = re.search(r' dict | None: if thinking and reply: return {"thinking": thinking, "reply": reply, "time": think_time} + if normalized_changed and text.strip() and text.strip() != original_text.strip(): + return {"thinking": "", "reply": text.strip(), "time": think_time} + return None @@ -638,7 +1003,8 @@ def clean_thinking_for_save(content: str, metadata: dict | None = None) -> tuple md = dict(metadata) if metadata else {} info = _extract_thinking_meta(content) if info: - md["thinking"] = info["thinking"] + if info.get("thinking"): + md["thinking"] = info["thinking"] if info.get("time"): md["thinking_time"] = info["time"] return info["reply"], md @@ -663,7 +1029,19 @@ def save_assistant_response( ): """Add assistant response to session history. In incognito mode, keeps in-memory context but skips DB persistence.""" md = dict(last_metrics) if last_metrics else {} - md["model"] = sess.model + def _model_value(value) -> str: + if value is None: + return "" + if not isinstance(value, str): + value = str(value) + return value.strip() + + requested_model = _model_value(md.get("requested_model") or md.get("selected_model") or getattr(sess, "model", "")) + actual_model = _model_value(md.get("model") or md.get("actual_model") or requested_model) + if requested_model: + md["requested_model"] = requested_model + if actual_model: + md["model"] = actual_model if character_name: md["character_name"] = character_name if web_sources: @@ -682,8 +1060,10 @@ def save_assistant_response( # Extract thinking into metadata (don't pollute message content with tags) _think_info = _extract_thinking_meta(full_response) if _think_info: - md["thinking"] = _think_info["thinking"] - md["thinking_time"] = _think_info.get("time") + if _think_info.get("thinking"): + md["thinking"] = _think_info["thinking"] + if _think_info.get("time"): + md["thinking_time"] = _think_info.get("time") _content = _think_info["reply"] else: _content = full_response @@ -710,6 +1090,54 @@ def save_assistant_response( return None +def _is_session_stream_active(session_id: str) -> bool: + """Best-effort check for "is a chat completion currently streaming for + this session?" — used to keep background extraction from overlapping a + main completion and competing for the local backend's processing slots + (issue #2927). Lazily imports the route module's live registry to avoid + a circular import (chat_routes imports this module at load time).""" + try: + from routes import chat_routes as _cr + return session_id in getattr(_cr, "_active_streams", {}) + except Exception: + return False + + +async def _run_extraction_jobs_sequentially(session_id: str, jobs: list, max_wait_s: float = 120.0): + """Run queued background-extraction coroutines one at a time, only once + no chat completion is actively streaming for this session. + + As diagnosed in issue #2927, firing memory/skill extraction concurrently + with the main chat completion (or with each other) makes them compete for + the local backend's limited processing slots, evicting the main + conversation's cached KV-cache checkpoint and forcing a full prompt + re-evaluation on the next turn. Waiting for the stream to go idle and then + running the jobs strictly in sequence keeps at most one "side" request in + flight against the backend at any time, and never alongside the user's + own conversation. + """ + # Wait for the triggering turn's own stream to finish winding down (it + # almost always already has by the time this task gets scheduled — this + # is a small safety margin, not the primary mechanism). + waited = 0.0 + poll = 0.25 + while _is_session_stream_active(session_id) and waited < max_wait_s: + await asyncio.sleep(poll) + waited += poll + + for name, job in jobs: + # Re-check before each job: a fast follow-up message from the user + # may have started a new stream for this session while we waited. + waited = 0.0 + while _is_session_stream_active(session_id) and waited < max_wait_s: + await asyncio.sleep(poll) + waited += poll + try: + await job + except Exception: + logger.warning("[bg-extract] %s extraction job failed for session %s", name, session_id, exc_info=True) + + def run_post_response_tasks( sess, session_manager, @@ -730,21 +1158,37 @@ def run_post_response_tasks( skills_manager=None, owner: str = None, extract_skills: bool = True, + allow_background_extraction: bool = True, ): - """Fire background tasks after a completed response: memory extraction, webhooks, auto-name, skill extraction.""" + """Fire background tasks after a completed response: memory extraction, webhooks, auto-name, skill extraction. + + Memory/skill extraction are queued to run *sequentially*, after the main + completion stream for this session has fully wound down — never + concurrently with it or with each other. As diagnosed in issue #2927, + firing these "side" LLM calls in parallel with the main chat completion + makes them compete for the local backend's limited processing slots + (llama.cpp defaults to 4), evicting the main conversation's cached + checkpoint and forcing a full prompt re-evaluation on the next turn. By + the time this function runs the main response is already saved, but the + extraction calls themselves are still async — queuing them through + ``_queue_background_extraction`` keeps them from overlapping the *next* + turn's request too. + """ + _extraction_jobs: list = [] + # Memory extraction — only every 4th message pair to avoid excess LLM calls _msg_count = len(sess.history) if hasattr(sess, 'history') else 0 _should_extract = (_msg_count >= 4) and (_msg_count % 4 == 0) - if not incognito and not compare_mode and _should_extract and uprefs.get("auto_memory", True): + if allow_background_extraction and not incognito and not compare_mode and _should_extract and uprefs.get("auto_memory", True): from services.memory.memory_extractor import extract_and_store from src.task_endpoint import resolve_task_endpoint t_url, t_model, t_headers = resolve_task_endpoint( - sess.endpoint_url, sess.model, sess.headers, + sess.endpoint_url, sess.model, sess.headers, owner=owner, ) - asyncio.create_task(extract_and_store( + _extraction_jobs.append(("memory", extract_and_store( sess, memory_manager, memory_vector, t_url, t_model, t_headers, - )) + ))) # Skill extraction from complex agent runs. Only when the user actually # chose agent mode — not a chat we auto-escalated for a notes/calendar @@ -762,6 +1206,7 @@ def run_post_response_tasks( ) if ( extract_skills + and allow_background_extraction and auto_skills_enabled and not incognito and not compare_mode @@ -776,15 +1221,18 @@ def run_post_response_tasks( from services.memory.skill_extractor import maybe_extract_skill from src.task_endpoint import resolve_task_endpoint s_url, s_model, s_headers = resolve_task_endpoint( - sess.endpoint_url, sess.model, sess.headers, + sess.endpoint_url, sess.model, sess.headers, owner=owner, ) logger.debug("[skill-extract] dispatching extractor (model=%s)", s_model) - asyncio.create_task(maybe_extract_skill( + _extraction_jobs.append(("skill", maybe_extract_skill( sess, skills_manager, s_url, s_model, s_headers, agent_rounds, agent_tool_calls, owner=owner, - )) + ))) + + if _extraction_jobs: + _spawn_bg(_run_extraction_jobs_sequentially(session_id, _extraction_jobs)) # Token accumulation if last_metrics: @@ -792,11 +1240,11 @@ def run_post_response_tasks( # Webhook if webhook_manager and not compare_mode: - asyncio.create_task(webhook_manager.fire("chat.completed", { + webhook_manager.fire_and_forget("chat.completed", { "session_id": session_id, "model": sess.model, "user_message": message, "response": full_response[:2000], - })) + }) # Auto-name if needs_auto_name(sess.name): - asyncio.create_task(auto_name_session(session_manager, sess)) + _spawn_bg(auto_name_session(session_manager, sess)) diff --git a/routes/chat_routes.py b/routes/chat_routes.py index bb72ea7a9..b8d9934b4 100644 --- a/routes/chat_routes.py +++ b/routes/chat_routes.py @@ -2,10 +2,12 @@ import asyncio import json +import os +import re import time import logging from datetime import datetime -from typing import Dict, Any, AsyncGenerator, List +from typing import Dict, Any, AsyncGenerator, List, Optional from fastapi import APIRouter, Request, HTTPException, Form, Query from fastapi.responses import StreamingResponse @@ -19,14 +21,18 @@ from src.model_context import estimate_tokens from src.chat_helpers import coerce_message_and_session from src.endpoint_resolver import normalize_base as _normalize_base, build_chat_url +from src.session_search import search_session_messages from src.prompt_security import untrusted_context_message from core.exceptions import SessionNotFoundError -from src.auth_helpers import get_current_user +from src.auth_helpers import effective_user, get_current_user from routes.session_routes import _verify_session_owner -from core.database import SessionLocal +from routes.document_helpers import _owner_session_filter +from core.database import SessionLocal, get_session_mode, set_session_mode from core.database import Session as DBSession, ChatMessage as DBChatMessage from core.database import Document as DBDocument, ModelEndpoint +from core.log_safety import redact_url from routes.research_routes import _resolve_research_endpoint +from routes.model_routes import _visible_models from routes.chat_helpers import ( resolve_session_auth, build_chat_context, @@ -35,11 +41,19 @@ clean_thinking_for_save, _enforce_chat_privileges, ) +from src.action_intents import ToolIntent, classify_tool_intent as _classify_tool_intent +from src.tool_policy import ( + WEB_TOOL_NAMES, + build_effective_tool_policy, + is_web_search_explicitly_denied, + web_search_enabled_for_turn, +) logger = logging.getLogger(__name__) # Track active streams for partial-save safety net _active_streams: Dict[str, dict] = {} +_IMAGE_MODEL_PREFIXES = ("gpt-image", "dall-e", "chatgpt-image") def _stream_set(session_id: str, **fields) -> None: @@ -55,38 +69,103 @@ def _stream_set(session_id: str, **fields) -> None: rec.update(fields) -import re as _re -# Phrases that clearly signal the user wants to create a todo / reminder / -# calendar event. When any of these hit in plain chat mode we silently -# escalate to the agent loop so manage_notes / manage_calendar are in scope. -_TOOL_INTENT_PATTERNS = [ - _re.compile(r"\bremind\s+me\b", _re.I), - _re.compile(r"\badd\s+(a\s+|an\s+)?(todo|task|reminder)\b", _re.I), - _re.compile(r"\b(create|schedule|book)\s+(a\s+|an\s+)?(event|meeting|appointment|reminder|call)\b", _re.I), - _re.compile(r"\bput\s+.+\bon\s+(my\s+)?calendar\b", _re.I), - _re.compile(r"\b(todo|reminder)\s*:", _re.I), - _re.compile(r"\bmake\s+(a\s+|an\s+)?(note|todo|reminder)\b", _re.I), - # Email intent — "write/send/email/message [someone]", "write hi to X" - _re.compile(r"\b(write|send)\s+.{1,30}\bto\s+\w+", _re.I), - _re.compile(r"\b(send|write|reply)\s+(an?\s+)?(email|message|mail)\b", _re.I), - _re.compile(r"\b(email|message)\s+\w+\b", _re.I), - _re.compile(r"\bcheck\s+(my\s+)?(email|inbox|mail)\b", _re.I), - _re.compile(r"\bunread\s+(email|mail)s?\b", _re.I), - # Shell / remote-host intent — covers the deepseek "can you ssh into X" - # case. We escalate to agent so `bash` is available; the model can still - # decide it doesn't need to actually run anything. - _re.compile(r"\bssh\s+(in)?to\b", _re.I), - _re.compile(r"\bssh\s+\w+", _re.I), - _re.compile(r"\b(run|execute)\s+.{1,40}\bon\s+\w+", _re.I), - _re.compile(r"\b(can|could|please|would)\s+you\s+(run|execute|exec)\b", _re.I), - _re.compile(r"\b(deploy|build|install|restart|reboot|kill|tail|grep|cat|ls|cd|cp|mv|rm)\b\s+\S+", _re.I), - _re.compile(r"\b(check|see)\s+(if|whether|what)\s+.{1,40}\b(running|process|service|port|file|exists?)\b", _re.I), -] - -def _message_needs_tools(text: str) -> bool: - if not text: +def _message_plain_text(content: Any) -> str: + if isinstance(content, list): + parts: List[str] = [] + for block in content: + if isinstance(block, dict): + text = block.get("text") + if isinstance(text, str): + parts.append(text) + elif isinstance(block, str): + parts.append(block) + return " ".join(parts) + return str(content or "") + + +def _last_user_plain_text(messages: List[Dict[str, Any]]) -> str: + for msg in reversed(messages or []): + if msg.get("role") == "user": + return _message_plain_text(msg.get("content")) + return "" + + +def _ensure_current_request_is_latest_user(messages: List[Dict[str, Any]], current_message: str) -> List[Dict[str, Any]]: + """Defensively keep detached streams grounded on the request that created them.""" + current = str(current_message or "").strip() + if not current: + return messages + latest = _last_user_plain_text(messages).strip() + if latest == current or current in latest or latest in current: + return messages + logger.warning( + "[chat_stream] latest user context mismatch; appending current request for model call. latest=%r current=%r", + latest[:120], + current[:120], + ) + repaired = list(messages or []) + repaired.append({"role": "user", "content": current}) + return repaired + + +_WEB_FOLLOWUP_RE = re.compile( + r"^\s*(?:(?:can|could|would|will)\s+you\s+)?" + r"(?:check|try\s+again|look(?:\s+now|\s+it\s+up)?|search(?:\s+now|\s+online|\s+it)?|" + r"do\s+it|again)\??\s*$", + re.I, +) +_RECENT_WEB_CONTEXT_RE = re.compile( + r"\b(?:weather|forecast|rain|raining|hourly|news|headlines|rate|exchange|currency|" + r"price|current|latest|search|look\s+up|online)\b", + re.I, +) + + +def _recent_session_text(sess, limit: int = 8, max_chars: int = 2000) -> str: + history = getattr(sess, "history", None) or getattr(sess, "_history", None) or [] + chunks: List[str] = [] + for msg in history[-limit:]: + content = getattr(msg, "content", None) + if content is None and isinstance(msg, dict): + content = msg.get("content") + text = _message_plain_text(content).strip() + if text: + chunks.append(text) + return " ".join(chunks)[-max_chars:] + + +def _is_contextual_web_followup(message: str, sess) -> bool: + """Treat short retry/check replies as web lookups when recent context was web.""" + if not message or not _WEB_FOLLOWUP_RE.search(message): return False - return any(p.search(text) for p in _TOOL_INTENT_PATTERNS) + return bool(_RECENT_WEB_CONTEXT_RE.search(_recent_session_text(sess))) + + +def _resolve_request_workspace(request, raw_value) -> tuple: + """Resolve the posted workspace for this request: (workspace, rejected). + + Privilege is checked BEFORE the path ever touches the filesystem. Only + admin/single-user callers can use the workspace-backed file/shell tools, + so only they get vet_workspace() and the workspace_rejected signal. For + any other caller the submitted value is dropped uniformly, with no vetting + and no event: otherwise the presence/absence of workspace_rejected would + let a non-admin chat caller probe which host paths exist. + + vet_workspace rejects non-directories, sensitive roots (.ssh, .gnupg, + ...), and filesystem roots; on rejection there is no confinement and the + default tool-path allowlist applies. The rejected value is surfaced so the + stream can tell an admin client (which believes a workspace is active) + that it was dropped. + """ + requested = (raw_value or "").strip() + if not requested: + return "", "" + from src.tool_security import owner_is_admin_or_single_user + if not owner_is_admin_or_single_user(get_current_user(request)): + return "", "" + from src.tool_execution import vet_workspace + workspace = vet_workspace(requested) or "" + return workspace, (requested if not workspace else "") def _session_url_matches_endpoint(session_url: str, endpoint_base: str) -> bool: @@ -102,13 +181,17 @@ def _session_url_matches_endpoint(session_url: str, endpoint_base: str) -> bool: return sess in variants or sess.startswith(base + "/") -def _clear_orphaned_session_endpoint(sess) -> bool: +def _clear_orphaned_session_endpoint(sess, owner: str | None = None) -> bool: """Clear a session model if its endpoint was deleted from ModelEndpoint.""" if not getattr(sess, "endpoint_url", ""): return False db = SessionLocal() try: - endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all() + q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) + if owner: + from src.auth_helpers import owner_filter + q = owner_filter(q, ModelEndpoint, owner) + endpoints = q.all() for ep in endpoints: if _session_url_matches_endpoint(sess.endpoint_url or "", ep.base_url or ""): return False @@ -122,13 +205,207 @@ def _clear_orphaned_session_endpoint(sess) -> bool: sess.model = "" sess.headers = {} return True + except Exception as e: + logger.warning("Failed to clear orphaned session endpoint", exc_info=e) + db.rollback() + return False + finally: + db.close() + + +def _endpoint_cache_contains_model(endpoint, model: str) -> bool: + """Return True when a populated endpoint model cache includes ``model``. + + Empty/malformed caches are treated as unknown rather than a negative match + so older image endpoints without cached models still work. + """ + raw = getattr(endpoint, "cached_models", None) + if not raw: + return True + try: + models = json.loads(raw) if isinstance(raw, str) else raw + except Exception as e: + logger.warning("Failed to parse cached models list, treating as containing model", exc_info=e) + return True + if not isinstance(models, list) or not models: + return True + wanted = (model or "").strip() + return wanted in {str(item).strip() for item in models} + + +def _is_image_generation_session(sess, owner: str | None = None) -> bool: + """Whether this chat session should bypass text chat and generate images. + + Model-name prefixes are explicit image models. Endpoint type is only used + when the current session endpoint actually matches that image endpoint, and + when a populated endpoint model cache includes the selected model. This + prevents an image endpoint on the same host from misrouting ordinary text + models into the image-generation path. + """ + model = (getattr(sess, "model", "") or "").strip() + if any(model.lower().startswith(prefix) for prefix in _IMAGE_MODEL_PREFIXES): + return True + + endpoint_url = (getattr(sess, "endpoint_url", "") or "").strip() + if not endpoint_url: + return False + + db = SessionLocal() + try: + q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) + if owner: + from src.auth_helpers import owner_filter + q = owner_filter(q, ModelEndpoint, owner) + endpoints = q.all() + for endpoint in endpoints: + if (getattr(endpoint, "model_type", None) or "llm") != "image": + continue + if not _session_url_matches_endpoint(endpoint_url, getattr(endpoint, "base_url", "") or ""): + continue + if _endpoint_cache_contains_model(endpoint, model): + return True except Exception: + return False + finally: + db.close() + return False + + +def _recover_empty_session_model(sess, session_id: str, owner: str | None = None) -> bool: + """Re-populate sess.model from the matching endpoint's cached models. + + Covers the window between endpoint setup and the first chat send: the + picker showed a model in the dropdown but the session record never got + written (Issue #587 — UI uses the cached endpoint list, not s.model). + For ChatGPT Subscription, also repairs stale OpenAI API model names such as + ``gpt-5`` that are not accepted by the Codex-backed ChatGPT account route. + """ + current_model = (getattr(sess, "model", "") or "").strip() + endpoint_url = (getattr(sess, "endpoint_url", "") or "").strip() + is_chatgpt_subscription = False + if current_model: + try: + from src.chatgpt_subscription import is_chatgpt_subscription_base + is_chatgpt_subscription = is_chatgpt_subscription_base(endpoint_url) + if not is_chatgpt_subscription: + return False + except Exception: + return False + db = SessionLocal() + try: + # Prefer the endpoint whose base URL matches the session — we know the + # user already pointed this session at that endpoint, so its first + # cached model is the most defensible default. + ep = None + if getattr(sess, "endpoint_url", ""): + q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True) + if owner: + from src.auth_helpers import owner_filter + q = owner_filter(q, ModelEndpoint, owner) + endpoints = q.all() + for cand in endpoints: + if _session_url_matches_endpoint(sess.endpoint_url or "", cand.base_url or ""): + ep = cand + break + if not ep: + return False + if not is_chatgpt_subscription: + try: + from src.chatgpt_subscription import is_chatgpt_subscription_base + is_chatgpt_subscription = is_chatgpt_subscription_base(getattr(ep, "base_url", "") or endpoint_url) + except Exception: + is_chatgpt_subscription = False + try: + cached = json.loads(ep.cached_models) if isinstance(ep.cached_models, str) else (ep.cached_models or []) + except Exception as e: + logger.warning("Failed to parse cached_models for endpoint %r", getattr(ep, "id", "?"), exc_info=e) + cached = [] + if not cached: + visible = [] + else: + try: + visible = _visible_models(cached, getattr(ep, "hidden_models", None)) + except Exception: + visible = cached + if current_model and current_model in {str(item).strip() for item in visible}: + return False + if is_chatgpt_subscription: + live_models = [] + if getattr(ep, "provider_auth_id", None): + try: + from src.chatgpt_subscription import fetch_available_models + from src.endpoint_resolver import resolve_endpoint_runtime + _base, api_key = resolve_endpoint_runtime(ep, owner=owner) + if api_key: + live_models = fetch_available_models(api_key) + if live_models: + ep.cached_models = json.dumps(live_models) + db.commit() + except Exception: + live_models = [] + # ChatGPT Subscription recovery must use the live Codex catalog. + # Cached rows are only trusted above to avoid revalidating a model + # that is already present in the visible picker list. + cached = live_models + if not cached: + return False + try: + visible = _visible_models(cached, getattr(ep, "hidden_models", None)) + except Exception: + visible = cached + if current_model and current_model in {str(item).strip() for item in visible}: + return False + if not visible: + return False + model = visible[0] + if not isinstance(model, str) or not model.strip(): + return False + model = model.strip() + # Persist so the next request, websocket reconnect, or page reload + # picks up the same model (we'd otherwise re-pick on every send + # and silently switch on the user if the cached order shifts). + db_session_q = db.query(DBSession).filter(DBSession.id == session_id) + if owner: + db_session_q = db_session_q.filter(DBSession.owner == owner) + db_session = db_session_q.first() + if db_session: + db_session.model = model + db_session.updated_at = datetime.utcnow() + db.commit() + sess.model = model + logger.info( + "Recovered session model for %s — picked %r from endpoint %s", + session_id, model, ep.id, + ) + return True + except Exception as e: db.rollback() + logger.warning("Failed to recover empty session model for %s: %s", session_id, e) return False finally: db.close() +def _set_user_time_from_request(request: Request) -> None: + """Copy browser timezone headers into the per-request context. + + This is intentionally ephemeral: it is used only while building prompts + and running tools for this request. It is not persisted or logged. + """ + try: + tz_offset = request.headers.get("x-tz-offset") + tz_name = request.headers.get("x-tz-name") + from src.user_time import clear_user_time_context, set_user_tz_name, set_user_tz_offset + + clear_user_time_context() + if tz_offset is not None: + set_user_tz_offset(tz_offset) + if tz_name: + set_user_tz_name(tz_name) + except Exception: + pass + + def setup_chat_routes( session_manager, chat_handler, @@ -147,6 +424,8 @@ def setup_chat_routes( # ------------------------------------------------------------------ # @router.post("/api/chat", response_model=Dict[str, str]) async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str, str]: + _set_user_time_from_request(request) + message = chat_request.message session = chat_request.session att_ids = chat_request.attachments or [] @@ -163,15 +442,31 @@ async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str sess = session_manager.get_session(session) except KeyError: raise HTTPException(404, f"Session '{session}' not found") - if _clear_orphaned_session_endpoint(sess): + owner = effective_user(request) + if _clear_orphaned_session_endpoint(sess, owner=owner): raise HTTPException(400, "Selected model endpoint was removed. Pick another model in Settings.") + # Empty model + live endpoint = setup race (Issue #587). Repair from + # the endpoint's cached model list before privilege checks, which + # otherwise see "" and behave inconsistently with the allowlist. + _recover_empty_session_model(sess, session, owner=owner) + if not getattr(sess, "model", "").strip(): + raise HTTPException( + 400, + "No model selected for this chat. Open the model picker and choose one before sending.", + ) + # Same allowed_models + daily-cap gate as chat_stream (mirror so the # non-streaming path can't be used to bypass). _enforce_chat_privileges(request, sess) + tool_policy = build_effective_tool_policy(last_user_message=message) + allow_tool_preprocessing = not tool_policy.block_all_tool_calls + # Inline memory command - memory_response = await chat_handler.handle_memory_command(sess, message) + memory_response = None + if not tool_policy.blocks("manage_memory"): + memory_response = await chat_handler.handle_memory_command(sess, message) if memory_response: return {"response": memory_response} @@ -185,10 +480,15 @@ async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str use_web=use_web, time_filter=time_filter, webhook_manager=webhook_manager, + allow_tool_preprocessing=allow_tool_preprocessing, ) # Research injection - if use_research: + research_blocked_by_policy = ( + tool_policy.blocks("trigger_research") + or tool_policy.blocks("manage_research") + ) + if use_research and not research_blocked_by_policy: try: _r_ep, _r_model, _r_headers = _resolve_research_endpoint(sess) research_ctx = await research_handler.call_research_service( @@ -209,6 +509,7 @@ async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str temperature=ctx.preset.temperature, max_tokens=ctx.preset.max_tokens, prompt_type=preset_id, + session_id=session, ) _clean_reply, _clean_md = clean_thinking_for_save(reply, {"model": sess.model}) sess.add_message(ChatMessage("assistant", _clean_reply, metadata=_clean_md)) @@ -223,6 +524,7 @@ async def chat_endpoint(request: Request, chat_request: ChatRequest) -> Dict[str ctx.uprefs, memory_manager, memory_vector, webhook_manager, character_name=ctx.preset.character_name, owner=ctx.user, + allow_background_extraction=not tool_policy.block_all_tool_calls, ) return {"response": reply} @@ -244,16 +546,7 @@ async def chat_stream(request: Request) -> StreamingResponse: except Exception as e: raise HTTPException(400, f"Request parsing error: {e}") - # Stash the user's UTC offset (in minutes east of UTC) from the - # frontend so tools like manage_notes interpret natural-language - # times in the USER's tz, not the server's. See calendar_routes. - try: - _tz_hdr = request.headers.get("x-tz-offset") - if _tz_hdr is not None: - from routes.calendar_routes import set_user_tz_offset - set_user_tz_offset(_tz_hdr) - except Exception: - pass + _set_user_time_from_request(request) form_data = await request.form() message = form_data.get("message") @@ -263,17 +556,39 @@ async def chat_stream(request: Request) -> StreamingResponse: use_research = form_data.get("use_research") time_filter = form_data.get("time_filter") preset_id = form_data.get("preset_id") - allow_bash = form_data.get("allow_bash") - allow_web_search = form_data.get("allow_web_search") + # Issue #3229: API callers send JSON, not FormData. Read from the + # JSON body as fallback so callers who send {"allow_bash": true} + # actually get bash enabled. + allow_bash = form_data.get("allow_bash") or (body or {}).get("allow_bash") + allow_web_search = form_data.get("allow_web_search") or (body or {}).get("allow_web_search") use_rag = form_data.get("use_rag") search_context = form_data.get("search_context") # pre-fetched web search results (compare mode) compare_mode = str(form_data.get("compare_mode", "")).lower() == "true" incognito = str(form_data.get("incognito", "")).lower() == "true" + # Plan mode is not part of the merge-ready UI. Ignore stale clients or + # manual form posts that still send plan_mode=true. + plan_mode = False chat_mode = str(form_data.get("mode", "")).lower() # 'chat' or 'agent' + # Workspace: confine the agent's file/shell tools to this folder. + workspace, workspace_rejected = _resolve_request_workspace( + request, form_data.get("workspace") + ) + # Plan mode is a modifier on agent mode — it only makes sense with tools. + if plan_mode: + chat_mode = "agent" + # An approved plan being EXECUTED: the frontend sends the checklist back + # on each turn so we can pin it in context. This way a long plan on a + # weak model survives history truncation — the agent can always re-read + # the plan. Ignored while still proposing (plan_mode on). Capped so a + # huge plan can't blow the prompt. + approved_plan = "" + if not plan_mode: + approved_plan = (form_data.get("approved_plan") or "").strip()[:8192] # Did the USER explicitly pick agent mode? (vs. us auto-escalating # below). Skill extraction should only learn from real agent sessions, # not chats we quietly promoted for a notes/calendar intent. user_requested_agent = (chat_mode == "agent") + _search_enabled = web_search_enabled_for_turn(allow_web_search, use_web) # Intent auto-escalation: if the user is clearly asking the assistant # to create a todo, reminder, or calendar event, promote chat → agent # for this turn so the LLM has access to manage_notes / manage_calendar. @@ -282,13 +597,82 @@ async def chat_stream(request: Request) -> StreamingResponse: # its way through a plain chat request (and fail, especially with the # shell disabled). auto_escalated = False - if chat_mode == "chat" and isinstance(message, str) and _message_needs_tools(message): + _tool_intent = _classify_tool_intent(message) if isinstance(message, str) else None + if chat_mode == "chat" and _tool_intent and _tool_intent.needs_tools: + chat_mode = "agent" + auto_escalated = True + logger.info( + "chat→agent auto-escalation: category=%s reason=%s", + _tool_intent.category, + _tool_intent.reason, + ) + elif chat_mode == "chat" and _search_enabled: chat_mode = "agent" auto_escalated = True - logger.info("chat→agent auto-escalation: message matched tool-intent pattern") + logger.info("chat→agent auto-escalation: search enabled") active_doc_id = form_data.get("active_doc_id", "").strip() logger.info(f"[doc-inject] chat_mode={chat_mode}, active_doc_id={active_doc_id!r}") + # Active email reader — when the user has an email open in the UI, the + # frontend passes its uid/folder/account so "reply", "summarize this", + # etc. resolve to the real email instead of the agent inventing a + # fake markdown draft. + active_email_uid = form_data.get("active_email_uid", "").strip() + active_email_folder = form_data.get("active_email_folder", "INBOX").strip() or "INBOX" + active_email_account = form_data.get("active_email_account", "").strip() + active_email_ctx: Optional[Dict[str, str]] = None + # Always reset between requests so a stale active-email pointer from + # a previous turn (different reader closed, different account, etc.) + # can't leak in when the user has no email open this turn. + try: + from src.tool_implementations import clear_active_email + clear_active_email() + except Exception: + pass + if active_email_uid: + active_email_ctx = { + "uid": active_email_uid, + "folder": active_email_folder, + "account": active_email_account, + } + # Try to enrich with subject + from so the agent's system prompt + # block can quote them. Best-effort: a stale cache is fine, a + # missing email just means we pass uid/folder/account only. + try: + from routes.email_routes import _read_cache_get, _read_cache_key + _ck = _read_cache_key(active_email_account or None, active_email_folder, active_email_uid, owner=get_current_user(request)) + _cached_email = _read_cache_get(_ck) + if _cached_email and isinstance(_cached_email, dict): + active_email_ctx["subject"] = str(_cached_email.get("subject") or "") + active_email_ctx["from"] = str( + _cached_email.get("from_address") + or _cached_email.get("from") + or _cached_email.get("from_name") + or "" + ) + _body_preview = (_cached_email.get("body") or "")[:2000] + if _body_preview: + active_email_ctx["body_preview"] = _body_preview + except Exception as _e: + logger.debug(f"[email-inject] cache enrich skipped: {_e}") + # Stash so email tools can resolve "this email" without UID guessing. + try: + from src.tool_implementations import set_active_email + set_active_email( + uid=active_email_uid, + folder=active_email_folder, + account=active_email_account or None, + subject=active_email_ctx.get("subject"), + sender=active_email_ctx.get("from"), + ) + except Exception as _e: + logger.debug(f"[email-inject] set_active_email failed: {_e}") + logger.info( + "[email-inject] active_email uid=%s folder=%s account=%s subject=%r", + active_email_uid, active_email_folder, active_email_account or "(default)", + active_email_ctx.get("subject", ""), + ) + try: # Attachment-only sends: skip the message-required check when the # user has attached one or more files (the attachment IS the action). @@ -303,8 +687,35 @@ async def chat_stream(request: Request) -> StreamingResponse: # but BEFORE loading. Prevents cross-user session hijack. _verify_session_owner(request, session) sess = session_manager.get_session(session) - if _clear_orphaned_session_endpoint(sess): + owner = effective_user(request) + if _clear_orphaned_session_endpoint(sess, owner=owner): raise HTTPException(400, "Selected model endpoint was removed. Pick another model in Settings.") + # Issue #587: picker shows a model from the endpoint cache but + # s.model never made it onto the DB row (first-send race after + # endpoint setup, or a previous endpoint delete/recreate). Pull + # the first cached model off the matching endpoint so the + # upstream isn't called with model="" (which surfaces as a + # generic 401/503). + _recover_empty_session_model(sess, session, owner=owner) + if not getattr(sess, "model", "").strip(): + raise HTTPException( + 400, + "No model selected for this chat. Open the model picker and choose one before sending.", + ) + if ( + chat_mode == "chat" + and isinstance(message, str) + and (not _tool_intent or not _tool_intent.needs_tools) + and _is_contextual_web_followup(message, sess) + ): + _tool_intent = ToolIntent(True, "web", "contextual web lookup follow-up") + chat_mode = "agent" + auto_escalated = True + logger.info( + "chat→agent auto-escalation: category=%s reason=%s", + _tool_intent.category, + _tool_intent.reason, + ) except SessionNotFoundError as e: raise HTTPException(404, str(e)) except (ValueError, ValidationError): @@ -321,31 +732,14 @@ async def chat_stream(request: Request) -> StreamingResponse: _enforce_chat_privileges(request, sess) # Ensure session has auth headers - resolve_session_auth(sess, session) + resolve_session_auth(sess, session, owner=effective_user(request)) # Check for research_pending BEFORE mode persist overwrites it do_research = str(use_research).lower() == "true" if not do_research: - try: - _mode_db = SessionLocal() - _db_mode = _mode_db.query(DBSession.mode).filter(DBSession.id == session).scalar() - _mode_db.close() - if _db_mode == 'research_pending': - do_research = True - logger.info(f"Session {session} in research_pending — auto-triggering research") - except Exception: - pass - - # Persist session mode (research > agent > chat) - _effective_mode = 'research' if do_research else (chat_mode or 'chat') - if _effective_mode in ('agent', 'research', 'chat'): - try: - _mdb = SessionLocal() - _mdb.query(DBSession).filter(DBSession.id == session).update({"mode": _effective_mode}) - _mdb.commit() - _mdb.close() - except Exception as _me: - logger.warning("Failed to persist session mode: %s", _me) + if get_session_mode(session) == 'research_pending': + do_research = True + logger.info(f"Session {session} in research_pending — auto-triggering research") att_ids = [] if body and isinstance(body.get("attachments"), list): @@ -353,10 +747,14 @@ async def chat_stream(request: Request) -> StreamingResponse: elif attachments: try: att_ids = [str(x) for x in json.loads(attachments)] - except Exception: - pass + except Exception as e: + logger.warning("Failed to parse attachments JSON, ignoring attachments", exc_info=e) no_memory = str(form_data.get("no_memory", "")).lower() == "true" + pre_context_tool_policy = build_effective_tool_policy( + last_user_message=message, + ) + allow_tool_preprocessing = not pre_context_tool_policy.block_all_tool_calls # Build shared context (stream path uses enhanced_message for context preface) ctx = await build_chat_context( @@ -378,6 +776,7 @@ async def chat_stream(request: Request) -> StreamingResponse: # manage_skills (agent mode). In plain chat or incognito the # index would be useless / unwanted noise. agent_mode=(chat_mode == "agent"), + allow_tool_preprocessing=allow_tool_preprocessing, ) _research_flags = {"do": do_research} # Mutable container for generator scope @@ -388,18 +787,60 @@ async def chat_stream(request: Request) -> StreamingResponse: try: if active_doc_id: logger.info(f"[doc-inject] active_doc_id from frontend: {active_doc_id}") - active_doc = _doc_db.query(DBDocument).filter( - DBDocument.id == active_doc_id, - ).first() + # Scope to the caller's documents. The session and in-memory + # fallbacks below are already owner/session-bound; this + # explicit-id path looked up by id alone, so a user could + # inject another user's document by passing its id. + _doc_q = _doc_db.query(DBDocument).filter(DBDocument.id == active_doc_id) + active_doc = _owner_session_filter(_doc_q, ctx.user).first() if active_doc: - logger.info(f"[doc-inject] found by ID: title={active_doc.title!r}, lang={active_doc.language!r}, is_active={active_doc.is_active}, content_len={len(active_doc.current_content or '')}") + doc_session = active_doc.session_id + doc_owner = getattr(active_doc, "owner", None) + if doc_owner and ctx.user and doc_owner != ctx.user: + logger.warning( + "[doc-inject] ignoring active_doc_id %s owned by another user", + active_doc_id, + ) + active_doc = None + else: + # NOTE: previously dropped the doc when doc.session_id + # != current chat session — but that broke the common + # case of "open an email draft from one chat, ask a + # different chat to write into it". The frontend only + # sends active_doc_id for docs currently visible in + # the UI, and we already owner-checked above, so trust + # the explicit signal. We just log the mismatch and + # re-bind the doc to the current session so future + # turns find it via the session-fallback path too. + if doc_session and doc_session != session: + logger.info( + "[doc-inject] cross-session active_doc_id %s (was session %s, now %s) — accepting and rebinding", + active_doc_id, doc_session, session, + ) + try: + active_doc.session_id = session + _doc_db.commit() + except Exception as _e: + _doc_db.rollback() + logger.warning(f"[doc-inject] session rebind failed: {_e}") + logger.info(f"[doc-inject] found by ID: title={active_doc.title!r}, lang={active_doc.language!r}, is_active={active_doc.is_active}, content_len={len(active_doc.current_content or '')}") else: logger.warning(f"[doc-inject] NOT FOUND by ID {active_doc_id}") if not active_doc: - active_doc = _doc_db.query(DBDocument).filter( + _email_doc_q = _doc_db.query(DBDocument).filter( + DBDocument.session_id == session, + DBDocument.is_active == True, + DBDocument.language == "email", + ) + active_doc = _owner_session_filter(_email_doc_q, ctx.user).order_by(DBDocument.updated_at.desc()).first() + if active_doc: + logger.info(f"[doc-inject] found email draft by session fallback: title={active_doc.title!r}") + if not active_doc: + _session_doc_q = _doc_db.query(DBDocument).filter( DBDocument.session_id == session, DBDocument.is_active == True - ).order_by(DBDocument.updated_at.desc()).first() + ) + active_doc = _owner_session_filter(_session_doc_q, ctx.user).order_by(DBDocument.updated_at.desc()).first() if active_doc: logger.info(f"[doc-inject] found by session fallback: title={active_doc.title!r}") # Last resort: the document the agent itself just created/edited @@ -410,10 +851,11 @@ async def chat_stream(request: Request) -> StreamingResponse: # leak a doc that belongs to a DIFFERENT session. if not active_doc: try: - from src.tool_implementations import get_active_document + from src.agent_tools.document_tools import get_active_document _mem_id = get_active_document() if _mem_id: - cand = _doc_db.query(DBDocument).filter(DBDocument.id == _mem_id).first() + _mem_q = _doc_db.query(DBDocument).filter(DBDocument.id == _mem_id) + cand = _owner_session_filter(_mem_q, ctx.user).first() if cand and (not cand.session_id or cand.session_id == session): active_doc = cand logger.info(f"[doc-inject] found by in-memory active id: title={active_doc.title!r} (session_id={cand.session_id!r})") @@ -430,10 +872,35 @@ async def chat_stream(request: Request) -> StreamingResponse: # Build disabled-tools set from frontend toggles + user privileges disabled_tools = set() - if str(allow_bash).lower() != "true": + # Only disable bash when the caller *explicitly* set it to a falsy + # value. When unset (None), defer to per-user privilege checks below. + # Web search is per-turn opt-in: either the chat pre-search setting + # (`use_web=true`) or agent web toggle (`allow_web_search=true`) must + # explicitly enable it. + if allow_bash is not None and str(allow_bash).lower() != "true": disabled_tools.add("bash") - if str(allow_web_search).lower() != "true": - disabled_tools.add("web_search") + _explicit_web_intent = bool(_tool_intent and _tool_intent.category == "web") + if is_web_search_explicitly_denied(allow_web_search) or not _search_enabled: + disabled_tools.update(WEB_TOOL_NAMES) + if _explicit_web_intent: + # A direct lookup/search request should not drift into personal + # tools or shell fallbacks. It can only use web_search/web_fetch + # when the request's explicit web setting enabled them. + disabled_tools.update({ + "bash", "python", + "search_chats", "manage_skills", "manage_memory", + "read_file", "write_file", "edit_file", + "create_document", "edit_document", "update_document", + "send_email", "reply_to_email", + "manage_notes", "manage_calendar", "manage_tasks", + "api_call", "builtin_browser", + }) + if _search_enabled: + disabled_tools.difference_update(WEB_TOOL_NAMES) + else: + disabled_tools.update(WEB_TOOL_NAMES) + elif _search_enabled: + disabled_tools.difference_update(WEB_TOOL_NAMES) # Nobody/incognito mode: deny tools that would expose the user's # persistent memory, past chats, or other identity-linked data. @@ -444,6 +911,21 @@ async def chat_stream(request: Request) -> StreamingResponse: "manage_skills", # skill presets tied to user }) + # Active email reader open → strip the tools that let the agent drift + # away from the visible email or skip review. The only allowed compose + # path is ui_control open_email_reply, which opens the same draft editor + # as the Reply button with the generated body pre-filled. This prevents + # the model from falling back to direct SMTP when it botches a draft + # call, and prevents fake email-shaped documents. + if active_email_ctx and active_email_ctx.get("uid"): + disabled_tools.update({ + "create_document", + "send_email", + "reply_to_email", + "mcp__email__send_email", + "mcp__email__reply_to_email", + }) + # Enforce per-user privileges _privs = {} _user = ctx.user @@ -497,7 +979,33 @@ async def chat_stream(request: Request) -> StreamingResponse: disabled_tools.update(_compare_strip) # In chat mode compare, disable ALL agent tools (no bash, python, file ops) if chat_mode == 'chat': - disabled_tools.update({"bash", "python", "read_file", "write_file", "web_search", "search_chats", "manage_tasks"}) + disabled_tools.update({"bash", "python", "read_file", "write_file", "web_search", "web_fetch", "search_chats", "manage_tasks"}) + + # Plan mode: investigate read-only, propose a plan, don't mutate. Block + # every tool not on the read-only allowlist. (stream_agent_loop enforces + # this again + drops MCP, so this is belt-and-suspenders.) + if plan_mode: + from src.tool_security import plan_mode_disabled_tools + disabled_tools.update(plan_mode_disabled_tools()) + + tool_policy = build_effective_tool_policy( + disabled_tools=disabled_tools, + last_user_message=message, + ) + disabled_tools = tool_policy.all_disabled_names() + research_blocked_by_policy = bool( + tool_policy.blocks("trigger_research") + or tool_policy.blocks("manage_research") + ) + effective_do_research = bool( + do_research and _research_flags["do"] and not research_blocked_by_policy + ) + + # Persist session mode after policy/privilege gates so blocked research + # turns remain ordinary chat/agent streams and saved messages. + _effective_mode = 'research' if effective_do_research else (chat_mode or 'chat') + if _effective_mode in ('agent', 'research', 'chat'): + set_session_mode(session, _effective_mode) async def stream_with_save() -> AsyncGenerator[str, None]: # _effective_mode is read-only here; closure captures it from @@ -506,7 +1014,14 @@ async def stream_with_save() -> AsyncGenerator[str, None]: web_sources = ctx.web_sources # Register active stream for partial-save safety net - _active_streams[session] = {"status": "streaming", "partial": "", "query": message, "is_research": do_research, "mode": _effective_mode} + _active_streams[session] = {"status": "streaming", "partial": "", "query": message, "is_research": effective_do_research, "mode": _effective_mode} + + # The client sent a workspace the server refused to bind (deleted + # folder, file path, sensitive dir, filesystem root). Tell it up + # front so the UI can clear the pill instead of displaying a + # confinement that is not actually in effect. + if workspace_rejected: + yield f"data: {json.dumps({'type': 'workspace_rejected', 'data': {'path': workspace_rejected}})}\n\n" if ctx.preprocessed.attachment_meta: yield f"data: {json.dumps({'type': 'attachments', 'data': ctx.preprocessed.attachment_meta})}\n\n" @@ -530,10 +1045,10 @@ async def stream_with_save() -> AsyncGenerator[str, None]: yield f"data: {json.dumps({'type': 'memories_used', 'data': ctx.used_memories})}\n\n" # Run research as a background task (survives page refresh) - if do_research and _research_flags["do"]: + if effective_do_research: _r_ep, _r_model, _r_headers = _resolve_research_endpoint(sess) _auth_keys = list(_r_headers.keys()) if _r_headers else [] - logger.info(f"Research endpoint resolved: model={_r_model}, endpoint={_r_ep}, auth_keys={_auth_keys}, sess_headers_keys={list(sess.headers.keys()) if isinstance(sess.headers, dict) else type(sess.headers)}") + logger.info(f"Research endpoint resolved: model={_r_model}, endpoint={redact_url(_r_ep)}, auth_keys={_auth_keys}, sess_headers_keys={list(sess.headers.keys()) if isinstance(sess.headers, dict) else type(sess.headers)}") # Clarification round: only for very short/vague queries on first research message. # Skip in compare mode — each pane is a fresh session, so every one would @@ -547,13 +1062,7 @@ async def stream_with_save() -> AsyncGenerator[str, None]: logger.info(f"First research message — asking clarifying questions for: {message[:60]}") yield f'data: {json.dumps({"type": "model_info", "model": sess.model, "suffix": "Research"})}\n\n' # Set DB mode to research_pending so the NEXT message auto-triggers research - try: - _pdb = SessionLocal() - _pdb.query(DBSession).filter(DBSession.id == session).update({"mode": "research_pending"}) - _pdb.commit() - _pdb.close() - except Exception as _pe: - logger.warning(f"Failed to set research_pending: {_pe}") + set_session_mode(session, "research_pending") ctx.messages.insert(0, {"role": "system", "content": "The user wants to start deep web research. Before searching, ask 2-3 brief " "clarifying questions to understand exactly what they want to know. For example: " @@ -613,6 +1122,7 @@ def _on_research_done(_sid, _result, _sources, _findings): prior_findings=_prior_findings, prior_urls=_prior_urls, on_complete=_on_research_done, + owner=_user, ) _heartbeat_counter = 0 @@ -655,13 +1165,16 @@ def _on_research_done(_sid, _result, _sources, _findings): _active_streams.pop(session, None) return - messages = ctx.messages + messages = _ensure_current_request_is_latest_user(ctx.messages, message) # Auto-compact notification if ctx.was_compacted: yield f"data: {json.dumps({'type': 'compacted', 'context_length': ctx.context_length})}\n\n" + if ctx.context_trimmed and not ctx.was_compacted: + yield f"data: {json.dumps({'type': 'context_trimmed', 'data': {'context_length': ctx.context_length, 'messages_before': ctx.context_messages_before_trim, 'messages_after': ctx.context_messages_after_trim, 'tokens_before': ctx.context_tokens_before_trim, 'tokens_after': ctx.context_tokens_after_trim}})}\n\n" full_response = "" + thinking_response = "" last_metrics = None # Configured fallback chain for the default chat model. Tried in @@ -669,12 +1182,12 @@ def _on_research_done(_sid, _result, _sources, _findings): # output. Resolved once per request. try: from src.endpoint_resolver import resolve_chat_fallback_candidates - _fallback_candidates = resolve_chat_fallback_candidates() + _fallback_candidates = resolve_chat_fallback_candidates(owner=_user) except Exception: _fallback_candidates = [] # Send model name early so the frontend can show it during streaming - _model_suffix = "Research" if do_research else None + _model_suffix = "Research" if effective_do_research else None _model_info = {"type": "model_info", "model": sess.model} if _model_suffix: _model_info["suffix"] = _model_suffix @@ -682,29 +1195,14 @@ def _on_research_done(_sid, _result, _sources, _findings): _model_info["character_name"] = ctx.preset.character_name yield f'data: {json.dumps(_model_info)}\n\n' - # Detect image models and route directly to image generation - _IMAGE_MODEL_PREFIXES = ("gpt-image", "dall-e", "chatgpt-image") - _is_image_model = any(sess.model.lower().startswith(p) for p in _IMAGE_MODEL_PREFIXES) - - # Also check if the endpoint is registered as an image-type endpoint - if not _is_image_model: - try: - from src.endpoint_resolver import normalize_base as _nb - _ep_base = _nb(sess.endpoint_url) - _db = SessionLocal() - try: - _is_image_model = _db.query(ModelEndpoint).filter( - ModelEndpoint.model_type == "image", - ModelEndpoint.is_enabled == True, - ModelEndpoint.base_url.contains(_ep_base.split("://")[-1].split("/")[0]), - ).first() is not None - finally: - _db.close() - except Exception: - pass - - if _is_image_model: + if _is_image_generation_session(sess, owner=_user): from src.settings import get_setting + if tool_policy.blocks("generate_image"): + _blocked_msg = tool_policy.reason_for("generate_image") + yield f'data: {json.dumps({"delta": _blocked_msg})}\n\n' + yield "data: [DONE]\n\n" + _active_streams.pop(session, None) + return if not get_setting("image_gen_enabled", True): yield f'data: {json.dumps({"delta": "Image generation is disabled by the administrator."})}\n\n' yield "data: [DONE]\n\n" @@ -714,7 +1212,7 @@ def _on_research_done(_sid, _result, _sources, _findings): _user_msg = message or "" yield f'data: {json.dumps({"type": "tool_start", "tool": "generate_image", "command": _user_msg[:100]})}\n\n' yield ": heartbeat\n\n" - _img_result = await do_generate_image(f"{_user_msg}\n{sess.model}", session) + _img_result = await do_generate_image(f"{_user_msg}\n{sess.model}", session, owner=_user) _img_output = _img_result.get("results", _img_result.get("error", "")) _img_tool_data = {"type": "tool_output", "tool": "generate_image", "command": _user_msg[:100], "output": _img_output, "exit_code": 0 if "error" not in _img_result else 1} for _k in ("image_url", "image_id", "image_prompt", "image_model", "image_size", "image_quality"): @@ -738,6 +1236,9 @@ def _on_research_done(_sid, _result, _sources, _findings): return elif chat_mode == "chat": _chat_start = time.time() + _answered_by = None # set if the selected model failed and a fallback answered + _requested_model = sess.model + _actual_model = None # ── Chat mode: call stream_llm directly, NO tools, NO document access ── try: _chat_candidates = [(sess.endpoint_url, sess.model, sess.headers)] + _fallback_candidates @@ -753,21 +1254,57 @@ def _on_research_done(_sid, _result, _sources, _findings): max_tokens=ctx.preset.max_tokens, prompt_type=preset_id, tools=None, + session_id=session, ): if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"): try: data = json.loads(chunk[6:]) if "delta" in data: - full_response += data["delta"] - _stream_set(session, partial=full_response) + # Reasoning tokens arrive flagged thinking:true. + # Forward them so the client can show a thinking + # indicator, but don't fold them into the saved + # reply (mirrors the rewrite path below). + if data.get("thinking"): + thinking_response += data["delta"] + else: + full_response += data["delta"] + _stream_set(session, partial=full_response) yield chunk + elif data.get("type") == "fallback": + # Selected model failed; a fallback answered. + # Forward the notice and remember the real model. + _answered_by = data.get("answered_by") or _answered_by + _actual_model = _actual_model or _answered_by + data["selected_model"] = data.get("selected_model") or _requested_model + yield chunk + elif data.get("type") == "model_actual": + _actual_model = data.get("model") or _actual_model + data["requested_model"] = _requested_model + yield f'data: {json.dumps(data)}\n\n' elif data.get("type") == "usage": last_metrics = data.get("data", {}) - last_metrics["model"] = sess.model + _reported_model = last_metrics.get("model") + last_metrics["requested_model"] = _requested_model + last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model + if ctx.context_trimmed: + last_metrics["context_trimmed"] = True + last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim + last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim + last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim + last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim if ctx.context_length and last_metrics.get("input_tokens"): pct = min(round((last_metrics["input_tokens"] / ctx.context_length) * 100, 1), 100.0) last_metrics["context_percent"] = pct last_metrics["context_length"] = ctx.context_length + # The frontend reads `tokens_per_second`; the raw usage event + # carries the backend's true gen speed as `gen_tps` (llama.cpp + # timings). Map it through so this direct-chat path shows real + # t/s instead of "n/a" → falling back to a bare token count. + if last_metrics.get("gen_tps") and not last_metrics.get("tokens_per_second"): + last_metrics["tokens_per_second"] = last_metrics["gen_tps"] + last_metrics["tps_source"] = "backend" + # Wall-clock response time for the stats popup ("Time"). + last_metrics.setdefault("response_time", round(time.time() - _chat_start, 2)) yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n' except json.JSONDecodeError: yield chunk @@ -791,36 +1328,48 @@ def _on_research_done(_sid, _result, _sources, _findings): "tokens_per_second": _tps, "context_percent": _ctx_pct, "context_length": ctx.context_length, - "model": sess.model, + "model": _actual_model or _answered_by or _requested_model, + "requested_model": _requested_model, "usage_source": "estimated", } yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n' if full_response: + _metrics_to_save = dict(last_metrics or {}) + if thinking_response.strip() and not _metrics_to_save.get("thinking"): + _metrics_to_save["thinking"] = thinking_response.strip() _saved_id = save_assistant_response( - sess, session_manager, session, full_response, last_metrics, + sess, session_manager, session, full_response, _metrics_to_save, character_name=ctx.preset.character_name, web_sources=web_sources, rag_sources=ctx.rag_sources, research_sources=research_sources, used_memories=ctx.used_memories, - do_research=do_research, + do_research=effective_do_research, incognito=incognito, ) if _saved_id: yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n' run_post_response_tasks( sess, session_manager, session, message, full_response, - last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager, + _metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager, incognito=incognito, compare_mode=compare_mode, character_name=ctx.preset.character_name, - owner=_user, + owner=_user, + allow_background_extraction=not tool_policy.block_all_tool_calls, ) _stream_set(session, status="done") yield chunk except (asyncio.CancelledError, GeneratorExit): if full_response: logger.info("Client disconnected mid-stream (chat mode) for session %s, saving partial (%d chars)", session, len(full_response)) - _stopped_content, _stopped_md = clean_thinking_for_save(full_response, {"stopped": True, "model": sess.model}) + _stopped_content, _stopped_md = clean_thinking_for_save( + full_response, + { + "stopped": True, + "model": _actual_model or _answered_by or _requested_model, + "requested_model": _requested_model, + }, + ) sess.add_message(ChatMessage("assistant", _stopped_content, metadata=_stopped_md)) if not incognito: session_manager.save_sessions() @@ -831,9 +1380,31 @@ def _on_research_done(_sid, _result, _sources, _findings): # ── Agent mode: full agent loop with tools ── _agent_rounds = 0 _agent_tool_calls = 0 + _answered_by = None # set if the selected model failed and a fallback answered + _requested_model = sess.model + _actual_model = None try: from src.settings import get_setting - _tool_budget = int(get_setting("agent_max_tool_calls", 0)) + from src.agent_tools import MAX_AGENT_ROUNDS as _DEFAULT_ROUNDS + # Per-message tool budget from settings; guard defensively in + # case settings.json was hand-edited to a non-numeric value + # (the HTTP admin endpoint validates, but direct edits bypass + # it). 0 = unlimited, matching auth_routes set_settings(). + try: + _tool_budget = int(get_setting("agent_max_tool_calls", 0)) + except (TypeError, ValueError): + _tool_budget = 0 + # Per-message round cap from settings; clamp defensively in + # case settings.json was hand-edited to a bad value. + try: + _max_rounds = int(get_setting("agent_max_rounds", _DEFAULT_ROUNDS) or _DEFAULT_ROUNDS) + except (TypeError, ValueError): + _max_rounds = _DEFAULT_ROUNDS + _max_rounds = max(1, min(_max_rounds, 200)) + + _forced_tools = None + if _search_enabled: + _forced_tools = set(WEB_TOOL_NAMES) async for chunk in stream_agent_loop( sess.endpoint_url, @@ -844,19 +1415,33 @@ def _on_research_done(_sid, _result, _sources, _findings): max_tokens=ctx.preset.max_tokens, prompt_type=preset_id, max_tool_calls=_tool_budget, + max_rounds=_max_rounds, context_length=ctx.context_length, active_document=active_doc, + active_email=active_email_ctx, session_id=session, disabled_tools=disabled_tools if disabled_tools else None, + tool_policy=tool_policy, owner=_user, fallbacks=_fallback_candidates, + plan_mode=plan_mode, + approved_plan=approved_plan or None, + workspace=workspace or None, + forced_tools=_forced_tools, + uploaded_files=ctx.uploaded_files, ): if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"): try: data = json.loads(chunk[6:]) if "delta" in data: - full_response += data["delta"] - _stream_set(session, partial=full_response) + # Reasoning tokens arrive flagged thinking:true. + # Forward them for the live indicator, but keep + # them out of the saved reply (same as chat mode). + if data.get("thinking"): + thinking_response += data["delta"] + else: + full_response += data["delta"] + _stream_set(session, partial=full_response) yield chunk elif data.get("type") == "web_sources": web_sources = data.get("data", []) @@ -865,24 +1450,55 @@ def _on_research_done(_sid, _result, _sources, _findings): "tool_start", "tool_output", "agent_step", "doc_stream_open", "doc_stream_delta", "doc_update", "doc_suggestions", "ui_control", + "rounds_exhausted", "budget_exceeded", + "loop_breaker_triggered", + "intent_nudge_exhausted", + "ask_user", + "plan_update", ): if data.get("type") == "agent_step": _agent_rounds = max(_agent_rounds, data.get("round", 1)) elif data.get("type") == "tool_start": _agent_tool_calls += 1 yield chunk + elif data.get("type") == "fallback": + # Selected model failed; a fallback answered. + # Forward the notice and remember the real + # model so metrics reflect it, not the masked + # selected model. + _answered_by = data.get("answered_by") or _answered_by + _actual_model = _actual_model or _answered_by + data["selected_model"] = data.get("selected_model") or _requested_model + yield chunk + elif data.get("type") == "model_actual": + _actual_model = data.get("model") or _actual_model + data["requested_model"] = _requested_model + yield f'data: {json.dumps(data)}\n\n' elif data.get("type") == "metrics": last_metrics = data.get("data", {}) - last_metrics["model"] = sess.model + _reported_model = last_metrics.get("model") + last_metrics["requested_model"] = last_metrics.get("requested_model") or _requested_model + last_metrics["model"] = _reported_model or _actual_model or _answered_by or _requested_model + if ctx.context_trimmed: + last_metrics["context_trimmed"] = True + last_metrics["context_messages_before_trim"] = ctx.context_messages_before_trim + last_metrics["context_messages_after_trim"] = ctx.context_messages_after_trim + last_metrics["context_tokens_before_trim"] = ctx.context_tokens_before_trim + last_metrics["context_tokens_after_trim"] = ctx.context_tokens_after_trim yield f'data: {json.dumps({"type": "metrics", "data": last_metrics})}\n\n' except json.JSONDecodeError: yield chunk elif chunk.startswith("event: "): yield chunk elif chunk == "data: [DONE]\n\n": - if full_response: + _has_tool_events = bool((last_metrics or {}).get("tool_events")) + if full_response or _has_tool_events: + _response_to_save = full_response or "Done." + _metrics_to_save = dict(last_metrics or {}) + if thinking_response.strip() and not _metrics_to_save.get("thinking"): + _metrics_to_save["thinking"] = thinking_response.strip() _saved_id = save_assistant_response( - sess, session_manager, session, full_response, last_metrics, + sess, session_manager, session, _response_to_save, _metrics_to_save, character_name=ctx.preset.character_name, web_sources=web_sources, rag_sources=ctx.rag_sources, @@ -892,8 +1508,8 @@ def _on_research_done(_sid, _result, _sources, _findings): if _saved_id: yield f'data: {json.dumps({"type": "message_saved", "id": _saved_id})}\n\n' run_post_response_tasks( - sess, session_manager, session, message, full_response, - last_metrics, ctx.uprefs, memory_manager, memory_vector, webhook_manager, + sess, session_manager, session, message, _response_to_save, + _metrics_to_save, ctx.uprefs, memory_manager, memory_vector, webhook_manager, incognito=incognito, compare_mode=compare_mode, character_name=ctx.preset.character_name, agent_rounds=_agent_rounds, @@ -901,6 +1517,7 @@ def _on_research_done(_sid, _result, _sources, _findings): skills_manager=skills_manager, owner=_user, extract_skills=user_requested_agent, + allow_background_extraction=not tool_policy.block_all_tool_calls, ) _stream_set(session, status="done") yield chunk @@ -914,7 +1531,14 @@ def _on_research_done(_sid, _result, _sources, _findings): try: if full_response: logger.info("Client disconnected mid-stream for session %s, saving partial response (%d chars)", session, len(full_response)) - _stopped_content2, _stopped_md2 = clean_thinking_for_save(full_response, {"stopped": True, "model": sess.model}) + _stopped_content2, _stopped_md2 = clean_thinking_for_save( + full_response, + { + "stopped": True, + "model": _actual_model or _answered_by or _requested_model, + "requested_model": _requested_model, + }, + ) sess.add_message(ChatMessage("assistant", _stopped_content2, metadata=_stopped_md2)) if not incognito: session_manager.save_sessions() @@ -933,11 +1557,29 @@ async def _safe_stream() -> AsyncGenerator[str, None]: finally: _active_streams.pop(session, None) - # Run the stream as a DETACHED background task so it survives the client - # closing the tab / navigating away (true terminal-agent behavior). The - # SSE response just subscribes (replay buffered output + live); dropping - # the SSE only removes a subscriber — the run keeps going and saves the - # assistant message on completion regardless. Reconnect via /api/chat/resume. + # Compare panes are short-lived, single-shot generations whose sessions + # exist only to drive that one pane — there's nothing to "resume" and + # the user expects the pane's Stop button (which aborts the fetch, + # closing this SSE) to promptly cancel the upstream LLM call. Detaching + # them would keep burning upstream tokens/compute after the pane is + # stopped or the comparison is abandoned, and would surface a stale + # "still streaming" /resume target for a session nobody will revisit. + # + # So: stream them directly (no agent_runs wrapping). Starlette cancels + # the underlying async generator (raising CancelledError/GeneratorExit + # inside it) as soon as it notices the client disconnected — which the + # mode-specific except blocks above already handle by saving the + # partial response exactly once. This stops the upstream call promptly + # without waiting on the next streamed chunk. + # + # Normal chat/agent streams keep the DETACHED behavior below: they + # survive the client closing the tab / navigating away. The SSE response just subscribes (replay + # buffered output + live); dropping the SSE only removes a subscriber — + # the run keeps going and saves the assistant message on completion + # regardless. Reconnect via /api/chat/resume. + if compare_mode: + return StreamingResponse(_safe_stream(), media_type="text/event-stream") + agent_runs.start(session, _safe_stream()) return StreamingResponse(agent_runs.subscribe(session), media_type="text/event-stream") @@ -970,11 +1612,15 @@ async def chat_stream_status(request: Request, session_id: str) -> Dict[str, Any _verify_session_owner(request, session_id) # A detached run can still be going even if _active_streams was popped; # report it as active so the client knows to reconnect via /resume. - if session_id not in _active_streams: + # Read once via .get() to avoid a KeyError race between the membership + # check and the indexed read if a sibling stream's finally pops the + # entry in between (same pattern _stream_set already uses). + rec = _active_streams.get(session_id) + if rec is None: if agent_runs.is_active(session_id): return {"status": "streaming", "detached": True} raise HTTPException(404, "No active stream for this session") - return _active_streams[session_id] + return rec # ------------------------------------------------------------------ # # POST /api/inject_context @@ -1003,46 +1649,17 @@ async def search_messages( if not q or not q.strip(): return [] - _user = get_current_user(request) - query_term = q.strip() - db = SessionLocal() - try: - base_q = ( - db.query(DBChatMessage, DBSession.name) - .join(DBSession, DBChatMessage.session_id == DBSession.id) - .filter( - DBSession.archived == False, - DBChatMessage.content.ilike(f"%{query_term}%"), - DBChatMessage.role.in_(["user", "assistant"]), - ) + _user = effective_user(request) + return [ + result.to_dict() + for result in search_session_messages( + q, + limit=limit, + owner=_user, + restrict_owner=_user is not None, + include_legacy_owner=False, ) - if _user: - base_q = base_q.filter(DBSession.owner == _user) - rows = base_q.order_by(DBChatMessage.timestamp.desc()).limit(limit).all() - - results = [] - for msg, session_name in rows: - content = msg.content or "" - lower_content = content.lower() - idx = lower_content.find(query_term.lower()) - if idx == -1: - snippet = content[:120] - else: - start = max(0, idx - 50) - end = min(len(content), idx + len(query_term) + 50) - snippet = ("..." if start > 0 else "") + content[start:end] + ("..." if end < len(content) else "") - - results.append({ - "session_id": msg.session_id, - "session_name": session_name or "Untitled", - "role": msg.role, - "content_snippet": snippet, - "timestamp": msg.timestamp.isoformat() if msg.timestamp else None, - }) - - return results - finally: - db.close() + ] # ------------------------------------------------------------------ # # POST /api/rewrite — lightweight rewrite of last AI message (no tools) @@ -1138,7 +1755,7 @@ async def stream_rewrite() -> AsyncGenerator[str, None]: db_msg = ( db.query(DBChatMessage) .filter(DBChatMessage.session_id == session_id, DBChatMessage.role == 'assistant') - .order_by(DBChatMessage.created_at.desc()) + .order_by(DBChatMessage.timestamp.desc()) .first() ) if db_msg: diff --git a/routes/chatgpt_subscription_routes.py b/routes/chatgpt_subscription_routes.py new file mode 100644 index 000000000..9c695b371 --- /dev/null +++ b/routes/chatgpt_subscription_routes.py @@ -0,0 +1,170 @@ +"""ChatGPT Subscription device-flow setup routes.""" + +import json +import logging +import uuid +from typing import Dict, Optional + +from fastapi import HTTPException, Request + +from core.database import ModelEndpoint, ProviderAuthSession, SessionLocal, utcnow_naive +from routes.device_flow import ( + DeviceFlowPoll, + DeviceFlowStart, + PendingDeviceFlowStore, + create_device_flow_router, +) +from src.auth_helpers import get_current_user +from src import chatgpt_subscription + +logger = logging.getLogger(__name__) + +_DEVICE_FLOW_STORE = PendingDeviceFlowStore() + + +def _provision_endpoint(tokens: Dict, owner: Optional[str]) -> Dict: + access_token = tokens.get("access_token") + refresh_token = tokens.get("refresh_token") + if not access_token or not refresh_token: + raise ValueError("ChatGPT token response was missing access_token or refresh_token") + + base = chatgpt_subscription.DEFAULT_CHATGPT_SUBSCRIPTION_BASE_URL + models = chatgpt_subscription.fetch_available_models(access_token) + if not models: + raise ValueError("ChatGPT Subscription connected, but no usable Codex models were discovered for this account.") + db = SessionLocal() + try: + auth = ( + db.query(ProviderAuthSession) + .filter( + ProviderAuthSession.provider == chatgpt_subscription.CHATGPT_SUBSCRIPTION_PROVIDER, + ProviderAuthSession.owner == owner, + ) + .first() + ) + if auth is None: + auth = ProviderAuthSession( + id=str(uuid.uuid4())[:8], + provider=chatgpt_subscription.CHATGPT_SUBSCRIPTION_PROVIDER, + owner=owner, + label="ChatGPT Subscription", + base_url=base, + auth_mode="chatgpt", + ) + db.add(auth) + auth.base_url = base + auth.access_token = access_token + auth.refresh_token = refresh_token + auth.last_refresh = utcnow_naive() + auth.auth_mode = "chatgpt" + + ep = ( + db.query(ModelEndpoint) + .filter( + ModelEndpoint.base_url == base, + ModelEndpoint.provider_auth_id == auth.id, + ModelEndpoint.owner == owner, + ) + .first() + ) + if ep is None: + ep = ModelEndpoint( + id=str(uuid.uuid4())[:8], + name="ChatGPT Subscription", + base_url=base, + model_type="llm", + endpoint_kind="api", + owner=owner, + ) + db.add(ep) + ep.name = "ChatGPT Subscription" + ep.base_url = base + ep.api_key = None + ep.provider_auth_id = auth.id + ep.is_enabled = True + ep.supports_tools = False + ep.model_type = "llm" + ep.endpoint_kind = "api" + ep.model_refresh_mode = "manual" + ep.cached_models = json.dumps(models) + db.commit() + result = { + "id": ep.id, + "name": ep.name, + "base_url": ep.base_url, + "models": models, + } + finally: + db.close() + + try: + from routes.model_routes import _invalidate_models_cache + + _invalidate_models_cache() + except Exception: + pass + return result + + +def _start_device_flow(request: Request, _form) -> DeviceFlowStart: + try: + data = chatgpt_subscription.request_device_code() + except Exception as exc: + raise chatgpt_subscription.to_http_exception(exc) + + device_auth_id = data.get("device_auth_id") + user_code = data.get("user_code") + if not device_auth_id or not user_code: + raise HTTPException(502, "ChatGPT did not return a complete device code") + verification_uri = data.get("verification_uri") or f"{chatgpt_subscription.CHATGPT_OAUTH_ISSUER}/codex/device" + return DeviceFlowStart( + pending={ + "device_auth_id": device_auth_id, + "user_code": user_code, + "owner": get_current_user(request) or None, + }, + response={ + "user_code": user_code, + "verification_uri": verification_uri, + }, + interval=int(data.get("interval") or 5), + expires_in=int(data.get("expires_in") or 900), + ) + + +def _poll_device_flow(_request: Request, pending: Dict) -> DeviceFlowPoll: + try: + data = chatgpt_subscription.poll_device_auth(pending["device_auth_id"], pending["user_code"]) + except Exception as exc: + logger.debug("ChatGPT device poll failed: %s", exc) + return DeviceFlowPoll.pending(str(exc)) + + authorization_code = data.get("authorization_code") + code_verifier = data.get("code_verifier") + if authorization_code and code_verifier: + try: + tokens = chatgpt_subscription.exchange_authorization_code(authorization_code, code_verifier) + result = _provision_endpoint(tokens, pending["owner"]) + except Exception as exc: + logger.exception("ChatGPT Subscription endpoint provisioning failed") + raise chatgpt_subscription.to_http_exception(exc) + return DeviceFlowPoll.authorized(result) + + err = data.get("error") or data.get("status") + if err in ("authorization_pending", "pending", None): + return DeviceFlowPoll.pending() + if err == "slow_down": + return DeviceFlowPoll.slow_down(int(data.get("interval") or 0) or None) + if err in ("expired_token", "access_denied", "denied"): + return DeviceFlowPoll.failed(err) + return DeviceFlowPoll.pending(err or "unknown") + + +def setup_chatgpt_subscription_routes(): + return create_device_flow_router( + prefix="/api/chatgpt-subscription", + tags=["chatgpt-subscription"], + store=_DEVICE_FLOW_STORE, + start_flow=_start_device_flow, + poll_flow=_poll_device_flow, + ) diff --git a/routes/cleanup/__init__.py b/routes/cleanup/__init__.py new file mode 100644 index 000000000..891d27e96 --- /dev/null +++ b/routes/cleanup/__init__.py @@ -0,0 +1,5 @@ +"""Cleanup route domain package (slice 2g, #4082/#4071). + +Contains cleanup_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/cleanup_routes.py re-exports from here. +""" diff --git a/routes/cleanup/cleanup_routes.py b/routes/cleanup/cleanup_routes.py new file mode 100644 index 000000000..ce1b63be0 --- /dev/null +++ b/routes/cleanup/cleanup_routes.py @@ -0,0 +1,60 @@ +# routes/cleanup_routes.py +"""Routes for cleanup operations.""" +import logging +from fastapi import APIRouter, HTTPException, Request +from src.cleanup_service import get_cleanup_preview, cleanup_sessions +from src.auth_helpers import get_current_user + +logger = logging.getLogger(__name__) + +def setup_cleanup_routes(session_manager): + """ + Setup cleanup-related routes. + + Args: + session_manager: SessionManager instance + + Returns: + APIRouter instance with cleanup routes + """ + router = APIRouter(prefix="/api/cleanup") + + @router.get("/preview") + async def cleanup_preview(request: Request): + """ + Preview what would be cleaned up without making any changes. + + Returns: + JSON response with lists of sessions that would be archived/deleted and estimated space savings + """ + user = get_current_user(request) + try: + preview = await get_cleanup_preview(owner=user) + return preview + except Exception as e: + logger.error(f"Cleanup preview failed: {e}") + raise HTTPException(500, "Cleanup preview generation failed") + + @router.post("") + async def cleanup_endpoint(request: Request): + """ + Perform cleanup operations: + 1. Archive inactive sessions (not accessed for 7 days) + 2. Delete old sessions (archived, not important, not accessed for 14+ days, with fewer than 10 messages) + + Returns: + JSON response with counts of deleted and archived sessions, and space freed + """ + user = get_current_user(request) + try: + archived_count, deleted_count, space_freed_mb = await cleanup_sessions(session_manager, owner=user) + return { + "archived_count": archived_count, + "deleted_count": deleted_count, + "space_freed_mb": round(space_freed_mb, 2) + } + except Exception as e: + logger.error(f"Cleanup failed: {e}") + raise HTTPException(500, "Cleanup operation failed") + + return router diff --git a/routes/cleanup_routes.py b/routes/cleanup_routes.py index ce1b63be0..1639ca85d 100644 --- a/routes/cleanup_routes.py +++ b/routes/cleanup_routes.py @@ -1,60 +1,17 @@ -# routes/cleanup_routes.py -"""Routes for cleanup operations.""" -import logging -from fastapi import APIRouter, HTTPException, Request -from src.cleanup_service import get_cleanup_preview, cleanup_sessions -from src.auth_helpers import get_current_user +"""Backward-compat shim — canonical location is routes/cleanup/cleanup_routes.py. -logger = logging.getLogger(__name__) +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.cleanup_routes``, ``from routes.cleanup_routes import X``, +``importlib.import_module("routes.cleanup_routes")``, and the string-targeted +``monkeypatch.setattr("routes.cleanup_routes.get_cleanup_preview", ...)`` / +``"routes.cleanup_routes.get_current_user"`` / ``"routes.cleanup_routes. +cleanup_sessions"`` pattern used by test_cleanup_owner_scope.py all operate +on the *same* object the application actually uses. Keeps existing import +paths working after slice 2g (#4082/#4071). +""" -def setup_cleanup_routes(session_manager): - """ - Setup cleanup-related routes. +import sys as _sys - Args: - session_manager: SessionManager instance +from routes.cleanup import cleanup_routes as _canonical # noqa: F401 - Returns: - APIRouter instance with cleanup routes - """ - router = APIRouter(prefix="/api/cleanup") - - @router.get("/preview") - async def cleanup_preview(request: Request): - """ - Preview what would be cleaned up without making any changes. - - Returns: - JSON response with lists of sessions that would be archived/deleted and estimated space savings - """ - user = get_current_user(request) - try: - preview = await get_cleanup_preview(owner=user) - return preview - except Exception as e: - logger.error(f"Cleanup preview failed: {e}") - raise HTTPException(500, "Cleanup preview generation failed") - - @router.post("") - async def cleanup_endpoint(request: Request): - """ - Perform cleanup operations: - 1. Archive inactive sessions (not accessed for 7 days) - 2. Delete old sessions (archived, not important, not accessed for 14+ days, with fewer than 10 messages) - - Returns: - JSON response with counts of deleted and archived sessions, and space freed - """ - user = get_current_user(request) - try: - archived_count, deleted_count, space_freed_mb = await cleanup_sessions(session_manager, owner=user) - return { - "archived_count": archived_count, - "deleted_count": deleted_count, - "space_freed_mb": round(space_freed_mb, 2) - } - except Exception as e: - logger.error(f"Cleanup failed: {e}") - raise HTTPException(500, "Cleanup operation failed") - - return router +_sys.modules[__name__] = _canonical diff --git a/routes/codex_routes.py b/routes/codex_routes.py new file mode 100644 index 000000000..9fe36a822 --- /dev/null +++ b/routes/codex_routes.py @@ -0,0 +1,910 @@ +"""Codex integration routes. + +These are small HTTP surfaces intended for the Codex plugin/MCP bridge. They +reuse existing Odysseus helpers and enforce API-token scopes before touching +user data. +""" + +import asyncio +import json +import zipfile +from io import BytesIO +from pathlib import Path +from typing import Any + +from fastapi import APIRouter, BackgroundTasks, Body, HTTPException, Request +from fastapi.responses import StreamingResponse + +from core.middleware import require_admin +from src.auth_helpers import require_authenticated_request, require_user +from src.tool_implementations import do_manage_notes +from src.constants import COOKBOOK_STATE_FILE +from routes._validators import validate_remote_host, validate_ssh_port + + +COOKBOOK_READ_SCOPES = {"cookbook:read", "cookbook:launch"} +COOKBOOK_LAUNCH_SCOPES = {"cookbook:launch"} +TODO_READ_SCOPES = {"todos:read", "todos:write"} +TODO_WRITE_SCOPES = {"todos:write"} +EMAIL_READ_SCOPES = {"email:read", "email:draft", "email:send"} +EMAIL_DRAFT_SCOPES = {"email:draft", "email:send"} +EMAIL_SEND_SCOPES = {"email:send"} +MEMORY_READ_SCOPES = {"memory:read", "memory:write"} +MEMORY_WRITE_SCOPES = {"memory:write"} +CALENDAR_READ_SCOPES = {"calendar:read", "calendar:write"} +CALENDAR_WRITE_SCOPES = {"calendar:write"} +DOCS_READ_SCOPES = {"documents:read", "documents:write"} +DOCS_WRITE_SCOPES = {"documents:write"} +WRITE_ACTIONS = {"add", "create", "new", "save", "remind", "update", "delete", "toggle_item", "remove", "remove_item"} + + +def _ssh_prefix_for_task(task: dict) -> tuple[str, str]: + """Resolve a cookbook task's stored SSH target into ``(host, port_flag)``. + + ``host`` is ``""`` for a local task. ``remoteHost`` / ``sshPort`` come from + cookbook_state.json and get interpolated into an ``ssh`` command string, so + validate them the same way the cookbook routes do. A tampered entry with + shell metacharacters in ``remoteHost`` is rejected with 400 rather than + injected. + """ + raw_host = task.get("remoteHost") + raw_port = task.get("sshPort") + host_value = str(raw_host).strip() if raw_host is not None else None + port_value = str(raw_port).strip() if raw_port is not None else None + host = validate_remote_host(host_value or None) or "" + ssh_port = validate_ssh_port(port_value or None) or "" + port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else "" + return host, port_flag + + +async def _as_owner(request: Request, owner: str, fn, *args, **kwargs): + """Run an existing route handler with request.state.current_user temporarily + set to ``owner`` so its internal get_current_user/require_user calls see + the scope-gated owner (not the "api" pseudo-user the bearer middleware sets). + Restores the original value when done. Works for sync and async handlers.""" + orig = getattr(request.state, "current_user", None) + orig_api_token = getattr(request.state, "api_token", None) + request.state.current_user = owner + request.state.api_token = False + try: + result = fn(*args, **kwargs) + if asyncio.iscoroutine(result): + result = await result + return result + finally: + request.state.current_user = orig + if orig_api_token is None: + try: + delattr(request.state, "api_token") + except AttributeError: + pass + else: + request.state.api_token = orig_api_token + + +def _scope_owner(request: Request, allowed: set[str]) -> str: + """Return the data owner if the caller is allowed for this Codex action.""" + if getattr(request.state, "api_token", False): + scopes = set(getattr(request.state, "api_token_scopes", []) or []) + if not scopes.intersection(allowed): + required = " or ".join(sorted(allowed)) + raise HTTPException(403, f"API token missing required scope: {required}") + owner = getattr(request.state, "api_token_owner", None) + if not owner: + raise HTTPException(403, "API token has no owner") + return owner + return require_user(request) + + +def _scope_owner_all(request: Request, required: set[str]) -> str: + """Return owner only when an API token has every required scope.""" + if getattr(request.state, "api_token", False): + scopes = set(getattr(request.state, "api_token_scopes", []) or []) + missing = required - scopes + if missing: + raise HTTPException(403, f"API token missing required scope: {' and '.join(sorted(missing))}") + owner = getattr(request.state, "api_token_owner", None) + if not owner: + raise HTTPException(403, "API token has no owner") + return owner + return require_user(request) + + +def _require_cookbook_scope(request: Request, allowed: set[str]) -> str: + """Authorize a Codex cookbook route. + + For API-token callers, enforce the given scope set. + For cookie-session callers, additionally require admin privileges + because cookbook surfaces expose host topology, task logs, tmux + commands, and model-serving controls. + """ + owner = _scope_owner(request, allowed) + if not getattr(request.state, "api_token", False): + require_admin(request) + return owner + + +def _find_endpoint(router: APIRouter | None, method: str, path: str): + if router is None: + return None + for route in getattr(router, "routes", []): + if getattr(route, "path", "") == path and method in getattr(route, "methods", set()): + return route.endpoint + return None + + +def _clamp_pagination(offset: Any, limit: Any, *, default_limit: int = 50, max_limit: int = 50) -> tuple[int, int]: + try: + parsed_offset = int(0 if offset in (None, "") else offset) + except (TypeError, ValueError): + raise HTTPException(400, "Invalid offset") + try: + parsed_limit = int(default_limit if limit in (None, "") else limit) + except (TypeError, ValueError): + raise HTTPException(400, "Invalid limit") + return max(0, parsed_offset), max(1, min(parsed_limit, max_limit)) + + +def setup_codex_routes( + email_router: APIRouter | None = None, + memory_router: APIRouter | None = None, + calendar_router: APIRouter | None = None, + document_router: APIRouter | None = None, +) -> APIRouter: + router = APIRouter(prefix="/api/codex", tags=["codex"]) + email_list_endpoint = _find_endpoint(email_router, "GET", "/api/email/list") + email_read_endpoint = _find_endpoint(email_router, "GET", "/api/email/read/{uid}") + email_send_endpoint = _find_endpoint(email_router, "POST", "/api/email/send") + email_draft_endpoint = _find_endpoint(email_router, "POST", "/api/email/draft") + memory_list_endpoint = _find_endpoint(memory_router, "GET", "/api/memory") + memory_add_endpoint = _find_endpoint(memory_router, "POST", "/api/memory/add") + calendar_list_events = _find_endpoint(calendar_router, "GET", "/api/calendar/events") + calendar_create_event = _find_endpoint(calendar_router, "POST", "/api/calendar/events") + documents_library_endpoint = _find_endpoint(document_router, "GET", "/api/documents/library") + documents_get_endpoint = _find_endpoint(document_router, "GET", "/api/document/{doc_id}") + documents_create_endpoint = _find_endpoint(document_router, "POST", "/api/document") + + @router.get("/capabilities") + def capabilities(request: Request): + token_scopes = set(getattr(request.state, "api_token_scopes", []) or []) + has_token = bool(getattr(request.state, "api_token", False)) + def scoped(allowed): + return bool(token_scopes.intersection(allowed)) if has_token else True + return { + "integration": "codex", + "token_scopes": sorted(token_scopes), + "tools": { + "todos": { + "read": scoped(TODO_READ_SCOPES), + "write": scoped(TODO_WRITE_SCOPES), + "actions": ["list", "add", "update", "delete", "toggle_item"], + }, + "email": { + "read": scoped(EMAIL_READ_SCOPES), + "draft": scoped(EMAIL_DRAFT_SCOPES), + "send": scoped(EMAIL_SEND_SCOPES), + "actions": ["list", "read", "draft_document", "draft", "send"], + }, + "memory": { + "read": scoped(MEMORY_READ_SCOPES), + "write": scoped(MEMORY_WRITE_SCOPES), + "actions": ["list", "add", "delete"], + "available": memory_list_endpoint is not None, + }, + "calendar": { + "read": scoped(CALENDAR_READ_SCOPES), + "write": scoped(CALENDAR_WRITE_SCOPES), + "actions": ["list_events", "create_event", "delete_event"], + "available": calendar_list_events is not None, + }, + "documents": { + "read": scoped(DOCS_READ_SCOPES), + "write": scoped(DOCS_WRITE_SCOPES), + "actions": ["library", "read", "create", "delete"], + "available": documents_library_endpoint is not None, + }, + "cookbook": { + "read": scoped(COOKBOOK_READ_SCOPES), + "launch": scoped(COOKBOOK_LAUNCH_SCOPES), + "actions": ["tasks", "servers", "output", "serve", "stop"], + }, + }, + "safety": { + "email_send_requires_confirmation": True, + "destructive_actions_should_confirm": True, + }, + } + + @router.get("/plugin.zip") + def plugin_zip(request: Request): + require_authenticated_request(request) + root = Path(__file__).resolve().parent.parent / "integrations" / "codex" + if not root.exists(): + raise HTTPException(404, "Codex plugin bundle not found") + buf = BytesIO() + with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for path in sorted(root.rglob("*")): + if path.is_dir() or "__pycache__" in path.parts or path.suffix == ".pyc": + continue + zf.write(path, Path("odysseus") / path.relative_to(root)) + buf.seek(0) + headers = {"Content-Disposition": 'attachment; filename="odysseus-codex-plugin.zip"'} + return StreamingResponse(buf, media_type="application/zip", headers=headers) + + @router.get("/todos") + async def list_todos(request: Request, archived: bool = False, label: str | None = None): + owner = _scope_owner(request, TODO_READ_SCOPES) + args: dict[str, Any] = {"action": "list", "archived": archived} + if label: + args["label"] = label + return await do_manage_notes(json.dumps(args), owner=owner) + + @router.post("/todos") + async def manage_todos(request: Request, body: dict[str, Any] = Body(default_factory=dict)): + action = str(body.get("action") or "add").replace("-", "_").strip().lower() + allowed = TODO_WRITE_SCOPES if action in WRITE_ACTIONS else TODO_READ_SCOPES + owner = _scope_owner(request, allowed) + args = dict(body) + args["action"] = action + return await do_manage_notes(json.dumps(args), owner=owner) + + @router.get("/emails") + async def list_emails( + request: Request, + folder: str = "INBOX", + limit: int = 10, + offset: int = 0, + filter: str = "all", + from_addr: str | None = None, + account_id: str | None = None, + has_attachments: int = 0, + ): + owner = _scope_owner(request, EMAIL_READ_SCOPES) + if email_list_endpoint is None: + raise HTTPException(503, "Email integration is not available") + limit = max(1, min(int(limit or 10), 50)) + offset = max(0, int(offset or 0)) + if account_id: + from routes.email_helpers import _assert_owns_account + + _assert_owns_account(account_id, owner) + return await email_list_endpoint( + folder=folder, + limit=limit, + offset=offset, + filter=filter, + from_addr=from_addr, + account_id=account_id, + has_attachments=has_attachments, + cache_bust=None, + owner=owner, + ) + + @router.get("/emails/{uid}") + async def read_email( + request: Request, + uid: str, + folder: str = "INBOX", + account_id: str | None = None, + mark_seen: bool = False, + ): + owner = _scope_owner(request, EMAIL_READ_SCOPES) + if email_read_endpoint is None: + raise HTTPException(503, "Email integration is not available") + if account_id: + from routes.email_helpers import _assert_owns_account + + _assert_owns_account(account_id, owner) + return await email_read_endpoint( + uid=uid, + folder=folder, + account_id=account_id, + mark_seen=mark_seen, + owner=owner, + ) + + # ── Email draft + send ──────────────────────────────────────────────── + # Both handlers in routes/email_routes.py already accept `owner=` via + # FastAPI Depends, so we call them directly without patching state. + + def _email_draft_document_content(body: dict[str, Any]) -> str: + def clean(v: Any) -> str: + if isinstance(v, list): + return ", ".join(str(x).strip() for x in v if str(x).strip()) + return str(v or "").strip() + + to = clean(body.get("to")) + cc = clean(body.get("cc")) + bcc = clean(body.get("bcc")) + subject = clean(body.get("subject")) + in_reply_to = clean(body.get("in_reply_to")) + references = clean(body.get("references")) + body_text = str(body.get("body") or body.get("body_html") or "").strip() + lines = [ + f"To: {to}", + ] + if cc: + lines.append(f"Cc: {cc}") + if bcc: + lines.append(f"Bcc: {bcc}") + lines.append(f"Subject: {subject}") + if in_reply_to: + lines.append(f"In-Reply-To: {in_reply_to}") + if references: + lines.append(f"References: {references}") + lines.extend(["---", body_text]) + return "\n".join(lines).rstrip() + "\n" + + @router.post("/emails/draft-document") + async def codex_email_draft_document(request: Request, body: dict[str, Any] = Body(default_factory=dict)): + owner = _scope_owner(request, EMAIL_DRAFT_SCOPES) + docs_owner = _scope_owner_all(request, DOCS_WRITE_SCOPES) + if docs_owner != owner: + raise HTTPException(403, "API token owner mismatch") + if documents_create_endpoint is None: + raise HTTPException(503, "Documents integration is not available") + from routes.document_routes import DocumentCreate + + subject = str(body.get("subject") or "Email draft").strip() or "Email draft" + title = str(body.get("title") or subject).strip() or "Email draft" + req = DocumentCreate( + session_id=body.get("session_id"), + title=title, + language="email", + content=_email_draft_document_content(body), + ) + result = await _as_owner(request, owner, documents_create_endpoint, request, req) + if isinstance(result, dict): + result = dict(result) + result["draft_type"] = "document" + result["send_required_confirmation"] = True + return result + + @router.post("/emails/draft") + async def codex_email_draft(request: Request, body: dict[str, Any] = Body(default_factory=dict)): + owner = _scope_owner(request, EMAIL_DRAFT_SCOPES) + if email_draft_endpoint is None: + raise HTTPException(503, "Email integration is not available") + from routes.email_routes import SendEmailRequest + + try: + req = SendEmailRequest(**body) + except Exception as exc: + raise HTTPException(400, f"Invalid draft payload: {exc}") + return await email_draft_endpoint(req=req, owner=owner) + + @router.post("/emails/send") + async def codex_email_send(request: Request, body: dict[str, Any] = Body(default_factory=dict)): + owner = _scope_owner(request, EMAIL_SEND_SCOPES) + if email_send_endpoint is None: + raise HTTPException(503, "Email integration is not available") + from routes.email_routes import SendEmailRequest + + try: + req = SendEmailRequest(**body) + except Exception as exc: + raise HTTPException(400, f"Invalid send payload: {exc}") + return await email_send_endpoint(req=req, background_tasks=BackgroundTasks(), owner=owner) + + # ── Memory ──────────────────────────────────────────────────────────── + + @router.get("/memory") + async def codex_memory_list(request: Request): + owner = _scope_owner(request, MEMORY_READ_SCOPES) + if memory_list_endpoint is None: + raise HTTPException(503, "Memory integration is not available") + return await _as_owner(request, owner, memory_list_endpoint, request) + + @router.post("/memory") + async def codex_memory_add(request: Request, body: dict[str, Any] = Body(default_factory=dict)): + owner = _scope_owner(request, MEMORY_WRITE_SCOPES) + if memory_add_endpoint is None: + raise HTTPException(503, "Memory integration is not available") + from src.request_models import MemoryAddRequest + + try: + memory_data = MemoryAddRequest( + text=str(body.get("text") or "").strip(), + category=body.get("category", "fact"), + source=body.get("source", "user"), + session_id=body.get("session_id"), + ) + except Exception as exc: + raise HTTPException(400, f"Invalid memory payload: {exc}") + if not memory_data.text: + raise HTTPException(400, "Empty memory text") + return await _as_owner(request, owner, memory_add_endpoint, request, memory_data) + + # ── Calendar ────────────────────────────────────────────────────────── + + @router.get("/calendar/events") + async def codex_calendar_list(request: Request, start: str, end: str, calendar: str = ""): + owner = _scope_owner(request, CALENDAR_READ_SCOPES) + if calendar_list_events is None: + raise HTTPException(503, "Calendar integration is not available") + return await _as_owner(request, owner, calendar_list_events, request, start, end, calendar) + + @router.post("/calendar/events") + async def codex_calendar_create(request: Request, body: dict[str, Any] = Body(default_factory=dict)): + owner = _scope_owner(request, CALENDAR_WRITE_SCOPES) + if calendar_create_event is None: + raise HTTPException(503, "Calendar integration is not available") + from routes.calendar_routes import EventCreate + + try: + data = EventCreate(**body) + except Exception as exc: + raise HTTPException(400, f"Invalid event payload: {exc}") + return await _as_owner(request, owner, calendar_create_event, request, data) + + # ── Documents ───────────────────────────────────────────────────────── + + @router.get("/documents") + async def codex_documents_library( + request: Request, + search: str | None = None, + language: str | None = None, + sort: str = "recent", + offset: int = 0, + limit: int = 50, + archived: bool = False, + ): + owner = _scope_owner(request, DOCS_READ_SCOPES) + if documents_library_endpoint is None: + raise HTTPException(503, "Documents integration is not available") + offset, limit = _clamp_pagination(offset, limit) + result = await _as_owner( + request, owner, documents_library_endpoint, + request, search, language, sort, offset, limit, archived, + ) + if isinstance(result, dict): + docs = result.get("documents") + total = result.get("total") + if isinstance(docs, list) and isinstance(total, int): + next_offset = offset + len(docs) + result["next_offset"] = next_offset if next_offset < total else None + return result + + @router.get("/documents/{doc_id}") + async def codex_documents_get(request: Request, doc_id: str): + owner = _scope_owner(request, DOCS_READ_SCOPES) + if documents_get_endpoint is None: + raise HTTPException(503, "Documents integration is not available") + return await _as_owner(request, owner, documents_get_endpoint, request, doc_id) + + # ── DELETE endpoints so agents can clean up after themselves ────────── + + memory_delete_endpoint = _find_endpoint(memory_router, "DELETE", "/api/memory/{memory_id}") + calendar_delete_event = _find_endpoint(calendar_router, "DELETE", "/api/calendar/events/{uid}") + documents_delete_endpoint = _find_endpoint(document_router, "DELETE", "/api/document/{doc_id}") + + @router.delete("/memory/{memory_id}") + async def codex_memory_delete(request: Request, memory_id: str): + owner = _scope_owner(request, MEMORY_WRITE_SCOPES) + if memory_delete_endpoint is None: + raise HTTPException(503, "Memory delete not available") + return await _as_owner(request, owner, memory_delete_endpoint, request, memory_id) + + @router.delete("/calendar/events/{uid}") + async def codex_calendar_delete(request: Request, uid: str): + owner = _scope_owner(request, CALENDAR_WRITE_SCOPES) + if calendar_delete_event is None: + raise HTTPException(503, "Calendar delete not available") + return await _as_owner(request, owner, calendar_delete_event, request, uid) + + @router.delete("/documents/{doc_id}") + async def codex_documents_delete(request: Request, doc_id: str): + owner = _scope_owner(request, DOCS_WRITE_SCOPES) + if documents_delete_endpoint is None: + raise HTTPException(503, "Documents delete not available") + return await _as_owner(request, owner, documents_delete_endpoint, request, doc_id) + + @router.post("/documents") + async def codex_documents_create(request: Request, body: dict[str, Any] = Body(default_factory=dict)): + owner = _scope_owner(request, DOCS_WRITE_SCOPES) + if documents_create_endpoint is None: + raise HTTPException(503, "Documents integration is not available") + from routes.document_routes import DocumentCreate + + try: + req = DocumentCreate(**body) + except Exception as exc: + raise HTTPException(400, f"Invalid document payload: {exc}") + return await _as_owner(request, owner, documents_create_endpoint, request, req) + + # ── Cookbook surface ── + # Lets the agent run the same launch / monitor / kill loop the user + # would do by hand in the Cookbook UI: read the current task list + + # tmux output, launch a serve task, stop one. Two scopes: + # cookbook:read — list tasks + tail output + list servers + # cookbook:launch — also start/stop serves (host shell exec) + # `cookbook:launch` is genuinely powerful: /api/model/serve runs SSH'd + # commands on the user's hosts. The existing _validate_serve_cmd + # allowlist (vllm/python3/sglang/llama-server/etc., no shell metachars) + # keeps the agent inside the same sandbox the UI uses. + + async def _run_shell(cmd: str, timeout: float = 15.0) -> dict: + """Run a shell command, return {exit_code, stdout, stderr}.""" + import asyncio as _asyncio + try: + proc = await _asyncio.create_subprocess_shell( + cmd, + stdout=_asyncio.subprocess.PIPE, + stderr=_asyncio.subprocess.PIPE, + ) + try: + stdout_b, stderr_b = await _asyncio.wait_for(proc.communicate(), timeout=timeout) + except _asyncio.TimeoutError: + proc.kill() + return {"exit_code": -1, "stdout": "", "stderr": "timed out"} + return { + "exit_code": proc.returncode, + "stdout": stdout_b.decode(errors="replace"), + "stderr": stderr_b.decode(errors="replace"), + } + except Exception as exc: + return {"exit_code": -1, "stdout": "", "stderr": str(exc)} + + def _read_cookbook_state() -> dict: + from pathlib import Path as _Path + import json as _json + p = _Path(COOKBOOK_STATE_FILE) + if not p.exists(): + return {} + try: + return _json.loads(p.read_text(encoding="utf-8")) + except Exception: + return {} + + def _redact_task(t: dict) -> dict: + """Strip secrets before returning to the agent.""" + clean = {k: v for k, v in t.items() if k not in ("hf_token", "_secrets")} + if isinstance(clean.get("payload"), dict): + pl = clean["payload"] + clean["payload"] = {k: v for k, v in pl.items() + if k not in ("hf_token", "_secrets")} + return clean + + @router.get("/cookbook/tasks") + async def codex_cookbook_tasks(request: Request): + _require_cookbook_scope(request, COOKBOOK_READ_SCOPES) + state = _read_cookbook_state() + tasks = state.get("tasks") or [] + return {"tasks": [_redact_task(t) for t in tasks]} + + @router.get("/cookbook/servers") + async def codex_cookbook_servers(request: Request): + _require_cookbook_scope(request, COOKBOOK_READ_SCOPES) + state = _read_cookbook_state() + servers = state.get("env", {}).get("servers") or [] + # Strip ssh creds / passwords; keep only what's needed to pick a host. + cleaned = [] + for s in servers: + cleaned.append({ + "name": s.get("name"), + "host": s.get("host"), + "port": s.get("port"), + "env": s.get("env"), + "envPath": s.get("envPath"), + "platform": s.get("platform"), + "modelDirs": s.get("modelDirs"), + }) + return {"servers": cleaned} + + @router.get("/cookbook/output/{session_id}") + async def codex_cookbook_output(request: Request, session_id: str, tail: int = 400): + _require_cookbook_scope(request, COOKBOOK_READ_SCOPES) + # Defensive: session_id must be the tmux-style id we issue + # (`serve-XXXX` / `cookbook-XXXX` / `queue-XXXX`); anything else + # would let the agent run arbitrary `tmux capture-pane` targets. + import re as _re + if not _re.fullmatch(r"[a-zA-Z0-9_-]+", session_id): + raise HTTPException(400, "Invalid session id") + tail = max(20, min(int(tail or 400), 4000)) + # Resolve the task's host (if any) from cookbook state so we can + # ssh to the right box, exactly as the UI does in _reconnectTask. + state = _read_cookbook_state() + tasks = state.get("tasks") or [] + task = next((t for t in tasks if t.get("sessionId") == session_id), None) + if task is None: + raise HTTPException(404, "task not found") + host, port_flag = _ssh_prefix_for_task(task) + # Prefer the persisted log file over the tmux pane. The pane gets + # overwritten by the post-crash neofetch banner + bash prompt the + # moment vllm exits; the log file is the raw stdout/stderr and + # survives unchanged. Falls back to pane for older tasks predating + # the tee-to-log runner change. + log_path = f"/tmp/odysseus-tmux/{session_id}.log" + inner = ( + f"if [ -s {log_path} ]; then tail -n {tail} {log_path}; " + f"else tmux capture-pane -t {session_id} -p -S -{tail}; fi" + ) + if host: + import shlex + cmd = f"ssh {port_flag}{host} {shlex.quote(inner)}" + else: + cmd = inner + result = await _run_shell(cmd, timeout=15) + return { + "session_id": session_id, + "host": host or "local", + "exit_code": result.get("exit_code"), + "output": result.get("stdout", ""), + "task": _redact_task(task), + } + + @router.post("/cookbook/serve") + async def codex_cookbook_serve(request: Request, body: dict[str, Any] = Body(default_factory=dict)): + _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES) + # Wraps /api/model/serve with the SAME validation the UI uses. + # _validate_serve_cmd (called inside model_serve) rejects shell + # metachars and requires the leading binary to be in the + # cookbook allowlist (vllm / python3 / sglang / llama-server / ...). + from routes.cookbook_helpers import ServeRequest + # Accept friendly aliases agents naturally reach for. Without these, + # passing `host` silently maps to nothing and the serve runs LOCAL + # instead of on the intended remote — exactly the bug an agent + # would never debug on its own. + norm = dict(body or {}) + if "host" in norm and "remote_host" not in norm: + norm["remote_host"] = norm.pop("host") + if "model" in norm and "repo_id" not in norm: + norm["repo_id"] = norm.pop("model") + if "ssh_port" not in norm and "port" in norm and (str(norm.get("port") or "").isdigit() and int(norm["port"]) >= 1000): + # Heuristic: if `port` looks like an SSH port (≥1000) and there's + # no explicit ssh_port, treat it as such. UI ports (8000, 8001, + # 30000) belong inside the cmd string, not here. + pass # leave as-is — user's `port` here is ambiguous; skip remap. + try: + req = ServeRequest(**norm) + except Exception as exc: + raise HTTPException(400, f"Invalid serve payload: {exc}") + serve_endpoint = _find_endpoint(None, "POST", "/api/model/serve") + # Fall back to importing from the cookbook router registered on app. + if serve_endpoint is None: + from fastapi import FastAPI + app: FastAPI = request.app + for route in app.routes: + if getattr(route, "path", None) == "/api/model/serve" and "POST" in getattr(route, "methods", set()): + serve_endpoint = route.endpoint + break + if serve_endpoint is None: + raise HTTPException(503, "model serve endpoint unavailable") + return await serve_endpoint(request, req) + + @router.post("/cookbook/stop/{session_id}") + async def codex_cookbook_stop(request: Request, session_id: str): + _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES) + import re as _re + if not _re.fullmatch(r"[a-zA-Z0-9_-]+", session_id): + raise HTTPException(400, "Invalid session id") + state = _read_cookbook_state() + tasks = state.get("tasks") or [] + task = next((t for t in tasks if t.get("sessionId") == session_id), None) + host, port_flag = _ssh_prefix_for_task(task or {}) + if host: + cmd = f"ssh {port_flag}{host} \"tmux kill-session -t {session_id}\"" + else: + cmd = f"tmux kill-session -t {session_id}" + result = await _run_shell(cmd, timeout=10) + return {"session_id": session_id, "exit_code": result.get("exit_code"), "host": host or "local"} + + @router.get("/cookbook/cached") + async def codex_cookbook_cached(request: Request, host: str | None = None): + """List cached models on a configured server (or local if host is omitted). + Mirrors `list_cached_models` from the chat agent so external agents have + the same inventory view before deciding what to serve/download.""" + _require_cookbook_scope(request, COOKBOOK_READ_SCOPES) + # Hit /api/model/cached internally, with the same modelDirs the chat + # agent's list_cached_models would resolve from cookbook state. + state = _read_cookbook_state() + env = state.get("env") if isinstance(state, dict) else {} + servers = (env.get("servers") if isinstance(env, dict) else None) or [] + HF_DEFAULTS = {"~/.cache/huggingface/hub", "~/.cache/huggingface"} + def _dirs_for(srv: dict) -> str: + mds = srv.get("modelDirs") if isinstance(srv, dict) else None + if isinstance(mds, list): + extras = [d for d in mds if isinstance(d, str) and d.strip() and d.strip() not in HF_DEFAULTS] + return ",".join(extras) + if isinstance(mds, str) and mds.strip() not in HF_DEFAULTS: + return mds + return "" + # Resolve friendly host name → real host (matches list_cached_models flow). + resolved_host = host or "" + srv: dict[str, Any] = {} + if host: + srv = next( + (s for s in servers if isinstance(s, dict) + and (s.get("name") == host or s.get("host") == host)), + {}, + ) + if srv and srv.get("host"): + resolved_host = srv["host"] + else: + srv = next((s for s in servers if isinstance(s, dict) and not (s.get("host") or "").strip()), {}) + params: dict[str, str] = {} + if resolved_host: + params["host"] = resolved_host + md = _dirs_for(srv) + if md: + params["model_dir"] = md + if srv.get("port"): + params["ssh_port"] = str(srv["port"]) + if srv.get("platform"): + params["platform"] = srv["platform"] + cached_endpoint = _find_endpoint(None, "GET", "/api/model/cached") + if cached_endpoint is None: + from fastapi import FastAPI + app: FastAPI = request.app + for route in app.routes: + if getattr(route, "path", None) == "/api/model/cached" and "GET" in getattr(route, "methods", set()): + cached_endpoint = route.endpoint + break + if cached_endpoint is None: + raise HTTPException(503, "model cached endpoint unavailable") + # The endpoint reads host/model_dir/ssh_port/platform as kwargs. + return await cached_endpoint( + request, + host=params.get("host") or None, + model_dir=params.get("model_dir") or None, + ssh_port=params.get("ssh_port") or None, + platform=params.get("platform") or None, + ) + + @router.get("/cookbook/presets") + async def codex_cookbook_presets(request: Request): + """List saved serve presets (model + host + port + launch cmd). + Counterpart to `list_serve_presets`. Use BEFORE composing a `serve` + body — the user's saved preset usually has the working cmd already.""" + _require_cookbook_scope(request, COOKBOOK_READ_SCOPES) + state = _read_cookbook_state() + presets = state.get("presets") or [] + out = [] + for p in presets: + if not isinstance(p, dict): + continue + out.append({ + "name": p.get("name"), + "model": p.get("model") or p.get("modelId"), + "host": p.get("host") or p.get("remoteHost"), + "port": p.get("port"), + "cmd": p.get("cmd"), + }) + return {"presets": out, "default_host": (state.get("env") or {}).get("defaultServer", "")} + + @router.post("/cookbook/preset/{name}") + async def codex_cookbook_serve_preset(request: Request, name: str): + """Launch a saved preset by name. Reuses the working cmd + host the + user already saved, avoiding the cmd-allowlist trial-and-error loop.""" + _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES) + import re as _re + if not _re.fullmatch(r"[A-Za-z0-9 _.:@\-]+", name): + raise HTTPException(400, "Invalid preset name") + state = _read_cookbook_state() + presets = state.get("presets") or [] + lname = name.lower().strip() + chosen = next( + (p for p in presets if isinstance(p, dict) and (p.get("name") or "").lower() == lname), + None, + ) + if chosen is None: + chosen = next( + (p for p in presets if isinstance(p, dict) and lname in (p.get("name") or "").lower()), + None, + ) + if chosen is None: + raise HTTPException(404, f"No preset matching {name!r}") + repo_id = chosen.get("model") or chosen.get("modelId") or "" + cmd = (chosen.get("cmd") or "").strip() + host = chosen.get("host") or chosen.get("remoteHost") or "" + if not repo_id or not cmd or cmd.startswith("(adopted"): + raise HTTPException(400, f"Preset {chosen.get('name')!r} has no launchable cmd " + "(adopted from external launch). Use POST /cookbook/serve " + "with the actual cmd instead.") + # Reuse the serve handler we already validated. + from routes.cookbook_helpers import ServeRequest + body = {"repo_id": repo_id, "cmd": cmd} + if host: + body["remote_host"] = host + try: + req = ServeRequest(**body) + except Exception as exc: + raise HTTPException(400, f"Preset payload invalid: {exc}") + serve_endpoint = _find_endpoint(None, "POST", "/api/model/serve") + if serve_endpoint is None: + from fastapi import FastAPI + app: FastAPI = request.app + for route in app.routes: + if getattr(route, "path", None) == "/api/model/serve" and "POST" in getattr(route, "methods", set()): + serve_endpoint = route.endpoint + break + if serve_endpoint is None: + raise HTTPException(503, "model serve endpoint unavailable") + return await serve_endpoint(request, req) + + @router.post("/cookbook/adopt") + async def codex_cookbook_adopt(request: Request, body: dict[str, Any] = Body(default_factory=dict)): + """Adopt an existing tmux session (one started via raw ssh+tmux) into + cookbook tracking. Needed when serve_model rejects a cmd and the + agent falls back to direct ssh — without adoption the session is + invisible to the UI. Body: {tmux_session, model, host?, port?}.""" + _require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES) + norm = dict(body or {}) + sess = (norm.get("tmux_session") or norm.get("session_id") or "").strip() + model = (norm.get("model") or norm.get("repo_id") or "").strip() + host = validate_remote_host((norm.get("host") or norm.get("remote_host") or "").strip() or None) or "" + port = norm.get("port") or 8000 + import re as _re + if not sess or not _re.fullmatch(r"[a-zA-Z0-9_-]+", sess): + raise HTTPException(400, "tmux_session required, [a-zA-Z0-9_-]+ only") + if not model: + raise HTTPException(400, "model required") + # Verify the tmux session exists on the target host before adopting. + import shlex + if host: + check = f"ssh {shlex.quote(host)} 'tmux has-session -t {shlex.quote(sess)}'" + else: + check = f"tmux has-session -t {shlex.quote(sess)}" + chk = await _run_shell(check, timeout=8) + if chk.get("exit_code") not in (0, None): + raise HTTPException(404, f"tmux session {sess!r} not found on {host or 'local'}") + # Write into cookbook_state.json. + import time as _t, json as _json + from core.atomic_io import atomic_write_json + from pathlib import Path as _Path + cookbook_state_path = _Path(COOKBOOK_STATE_FILE) + try: + state = _json.loads(cookbook_state_path.read_text(encoding="utf-8")) + except Exception: + state = {} + tasks = state.setdefault("tasks", []) + if any(isinstance(t, dict) and t.get("sessionId") == sess for t in tasks): + return {"ok": True, "already_tracked": True, "session_id": sess} + tasks.append({ + "id": sess, "sessionId": sess, + "name": model.split("/")[-1] if "/" in model else model, + "type": "serve", "status": "running", + "output": f"Adopted externally-launched session {sess!r} on {host or 'local'}.", + "ts": int(_t.time() * 1000), + "payload": {"repo_id": model, "remote_host": host, "_cmd": "(adopted — launched outside cookbook)", "port": int(port)}, + "remoteHost": host, "sshPort": "", "platform": "linux", + "_serveReady": False, "_endpointAdded": False, "_adoptedExternally": True, + }) + try: + atomic_write_json(cookbook_state_path, state) + except Exception as exc: + raise HTTPException(500, f"state write failed: {exc}") + return {"ok": True, "session_id": sess, "host": host or "local"} + + return router + + +def setup_claude_routes() -> APIRouter: + """Serve the Claude Code skill bundle. + + Claude Code uses the same scope-gated `/api/codex/*` endpoints at runtime; + this router only exists to deliver the skill zip via `/api/claude/plugin.zip` + so the user-facing setup commands stay in the Claude namespace. + """ + router = APIRouter(prefix="/api/claude", tags=["claude"]) + + @router.get("/plugin.zip") + def plugin_zip(request: Request): + require_authenticated_request(request) + # Only ship the skills/ subtree so extracting at ~/.claude/ doesn't dump + # README.md or other bundle metadata into the user's claude config dir. + skills_root = Path(__file__).resolve().parent.parent / "integrations" / "claude" / "skills" + if not skills_root.exists(): + raise HTTPException(404, "Claude skill bundle not found") + bundle_root = skills_root.parent + buf = BytesIO() + with zipfile.ZipFile(buf, "w", compression=zipfile.ZIP_DEFLATED) as zf: + for path in sorted(skills_root.rglob("*")): + if path.is_dir() or "__pycache__" in path.parts or path.suffix == ".pyc": + continue + zf.write(path, path.relative_to(bundle_root)) + buf.seek(0) + headers = {"Content-Disposition": 'attachment; filename="odysseus-claude-skill.zip"'} + return StreamingResponse(buf, media_type="application/zip", headers=headers) + + return router diff --git a/routes/compare/__init__.py b/routes/compare/__init__.py new file mode 100644 index 000000000..03bfef9f5 --- /dev/null +++ b/routes/compare/__init__.py @@ -0,0 +1,5 @@ +"""Compare route domain package (slice 2i, #4082/#4071). + +Contains compare_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/compare_routes.py re-exports from here. +""" diff --git a/routes/compare/compare_routes.py b/routes/compare/compare_routes.py new file mode 100644 index 000000000..ad42f1a89 --- /dev/null +++ b/routes/compare/compare_routes.py @@ -0,0 +1,365 @@ +# routes/compare_routes.py +"""Model A/B comparison routes.""" +import json +import uuid +import random +from datetime import datetime +from fastapi import APIRouter, Form, HTTPException, Request +from typing import List +from pydantic import BaseModel +import logging + +from core.database import Comparison, SessionLocal +from core.session_manager import SessionManager +from src.auth_helpers import get_current_user +from routes.session_routes import _reject_raw_endpoint_url_for_non_admin + +logger = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/compare", tags=["compare"]) + + +def _owned_endpoint_by_url(db, base_url, owner): + """ModelEndpoint whose base_url == `base_url` and is VISIBLE to `owner` + (their own rows + legacy null-owner "shared" rows); None otherwise. + + Owner-scoped on purpose. ModelEndpoint is per-user (core/database.py: non-null + owner = private, "the model picker only shows the endpoint to that user") and + holds a decrypted `api_key`. start_comparison copies the matched row's api_key + into the caller-owned [CMP] session's headers, which then drives that session's + /api/chat_stream calls — so an UNSCOPED base_url match would let a user mint a + comparison bound to ANOTHER user's private endpoint and spend that owner's + api_key / reach whatever base_url they configured. Mirrors + session_routes._owned_endpoint. A null/empty owner is a no-op (single-user / + legacy mode). + """ + from core.database import ModelEndpoint + from src.auth_helpers import owner_filter + q = db.query(ModelEndpoint).filter(ModelEndpoint.base_url == base_url) + return owner_filter(q, ModelEndpoint, owner).first() + + +def _owned_endpoint_by_id(db, endpoint_id, owner): + """ModelEndpoint whose id == `endpoint_id` and is VISIBLE to `owner` (their + own rows + legacy null-owner "shared" rows); None otherwise. + + Preferred over _owned_endpoint_by_url for credential resolution: two visible + endpoints can share the same base_url but hold DIFFERENT api_keys (e.g. two + accounts on the same provider). A base_url-only match returns whichever row + sorts first, so it can copy the WRONG owner-scoped key into the [CMP] session. + An id pins the exact registered endpoint, so /api/compare/start prefers it and + only falls back to URL matching for legacy / admin raw-URL callers. Owner + scoping is identical to _owned_endpoint_by_url (a null/empty owner is a no-op). + """ + from core.database import ModelEndpoint + from src.auth_helpers import owner_filter + q = db.query(ModelEndpoint).filter(ModelEndpoint.id == endpoint_id) + return owner_filter(q, ModelEndpoint, owner).first() + + +class RecordVoteRequest(BaseModel): + prompt: str + models: List[str] + winner: str # model name or "tie" + is_blind: bool = True + + +def setup_compare_routes(session_manager: SessionManager): + """Setup comparison routes.""" + + @router.post("/start") + def start_comparison( + request: Request, + prompt: str = Form(...), + model_a: str = Form(...), + model_b: str = Form(...), + endpoint_a: str = Form(""), + endpoint_b: str = Form(""), + endpoint_a_id: str = Form(""), + endpoint_b_id: str = Form(""), + is_blind: str = Form("true"), + ): + """Create two ephemeral sessions and a comparison record. + + Returns the comparison ID and the two session IDs so the client + can fire two independent SSE streams to /api/chat_stream. + """ + user = getattr(request.state, 'current_user', None) + comp_id = str(uuid.uuid4()) + sid_a = str(uuid.uuid4()) + sid_b = str(uuid.uuid4()) + + # Blind mapping: randomly assign left/right + blind = str(is_blind).lower() == "true" + if blind: + mapping = {"left": "a", "right": "b"} + if random.random() > 0.5: + mapping = {"left": "b", "right": "a"} + else: + mapping = {"left": "a", "right": "b"} + + # Map session IDs to left/right based on blind mapping + session_left = sid_a if mapping["left"] == "a" else sid_b + session_right = sid_a if mapping["right"] == "a" else sid_b + + # In blind mode, name the helper sessions by their neutral slot + # ("Model A" / "Model B") instead of the real model. Otherwise the + # session name leaks the model in the sidebar and GET /api/sessions, + # de-anonymizing the comparison before the user votes (issue #1285). + slot_name = {session_left: "Model A", session_right: "Model B"} + + # SECURITY: resolve and validate BOTH endpoints before creating any + # session. Compare copies a registered endpoint's Authorization header + # into the [CMP] session, so validating one endpoint while creating its + # session, then rejecting the other, would leave a partial compare + # session behind with that header attached. Doing all the owner-scope + # resolution + raw-URL rejection up front means a 403 on either endpoint + # aborts the whole request with nothing created and no header copied. + from src.endpoint_resolver import build_chat_url, build_headers, normalize_base + resolved = [] + db = SessionLocal() + try: + for sid, model, endpoint, endpoint_id in [ + (sid_a, model_a, endpoint_a, endpoint_a_id), + (sid_b, model_b, endpoint_b, endpoint_b_id), + ]: + # Prefer an explicit endpoint id: it pins the EXACT registered + # endpoint (and its api_key), even when two endpoints visible to + # the caller share a base_url with different keys — a URL-only + # match would copy whichever row sorts first, i.e. possibly the + # wrong key. Fall back to URL resolution only for legacy / admin + # raw-URL callers that don't send an id. + eid = endpoint_id.strip() if isinstance(endpoint_id, str) else "" + if eid: + ep = _owned_endpoint_by_id(db, eid, user) + if ep is None: + # An id the caller can't see (wrong owner / deleted) must + # NOT silently fall back to a same-URL row with a different + # key — that's exactly the mix-up ids exist to prevent. + raise HTTPException(404, "Model endpoint not found") + # The id already resolved the endpoint; ignore any raw URL the + # caller also sent and dial the stored config instead. + endpoint = ep.base_url + elif not endpoint: + raise HTTPException( + 422, "endpoint_a/endpoint_b or endpoint_a_id/endpoint_b_id is required" + ) + else: + # Resolve the supplied URL to a ModelEndpoint the caller owns + # (their own rows + legacy null-owner shared rows), scoped so a + # comparison can't borrow another user's private endpoint key. + base = normalize_base(endpoint) + ep = _owned_endpoint_by_url(db, base, user) + # Reject *unregistered* raw URLs for signed-in non-admins; a + # matched registered endpoint supplies an id so the caller can + # still compare endpoints they own. Blanket-rejecting here (the + # earlier `endpoint_id=None` call) locked non-admins out of + # compare entirely, since compare resolves endpoints by URL with + # no endpoint_id. Mirrors the gallery inpaint/harmonize checks. + # Raised here (phase 1), before any session exists. + _reject_raw_endpoint_url_for_non_admin( + request, user, str(ep.id) if ep is not None else None, endpoint + ) + # Bind the [CMP] session to the RESOLVED endpoint, not the raw + # caller-supplied string. When the URL matches a registered + # endpoint visible to the caller, use that row's own normalized + # base URL (the same value owner scoping + endpoint validation + # already vetted) so the session dials exactly where the stored + # config points. The raw `endpoint` only survives for callers + # allowed to pass one — admins / single-user mode, where + # `_reject_raw_endpoint_url_for_non_admin` is a no-op and `ep` + # is None. Mirrors the registered-endpoint path in session_routes. + session_endpoint_url = ( + build_chat_url(normalize_base(ep.base_url)) if ep is not None else endpoint + ) + # Headers come only from a matched endpoint's key; None when + # `ep` is None (raw admin URL or no match), so a comparison can + # never inherit another user's key/headers. + headers = build_headers(ep.api_key, ep.base_url) if (ep and ep.api_key) else None + resolved.append((sid, model, session_endpoint_url, headers)) + finally: + db.close() + + # Both endpoints validated — only now create the ephemeral [CMP] + # sessions and copy any resolved headers. + for sid, model, session_endpoint_url, headers in resolved: + name = f"[CMP] {slot_name[sid]}" if blind else f"[CMP] {model.split('/')[-1]}" + session_manager.create_session( + session_id=sid, + name=name, + endpoint_url=session_endpoint_url, + model=model, + rag=False, + owner=user, + ) + if headers: + s = session_manager.sessions.get(sid) + if s: + s.headers = headers + + # Store comparison record + db = SessionLocal() + try: + comp = Comparison( + id=comp_id, + prompt=prompt, + model_a=model_a, + model_b=model_b, + # Record the URL the session actually dials. For URL callers this + # is their raw input; for id-only callers (empty endpoint_a/_b) + # fall back to the resolved endpoint URL so the column stays + # meaningful and non-null. resolved is in [a, b] order. + endpoint_a=endpoint_a or resolved[0][2], + endpoint_b=endpoint_b or resolved[1][2], + is_blind=blind, + blind_mapping=json.dumps(mapping), + owner=user, + ) + db.add(comp) + db.commit() + finally: + db.close() + + # In blind mode, withhold the model identities AND the left/right + # mapping from the response. The client already knows model_a/model_b + # (it sent them), so returning either would defeat blind mode. They are + # revealed by POST /api/compare/{id}/vote once the user has voted (#1285). + return { + "id": comp_id, + "session_left": session_left, + "session_right": session_right, + "model_left": None if blind else (model_a if mapping["left"] == "a" else model_b), + "model_right": None if blind else (model_a if mapping["right"] == "a" else model_b), + "is_blind": blind, + "mapping": None if blind else mapping, + } + + @router.post("/{comp_id}/vote") + def vote_comparison( + request: Request, + comp_id: str, + winner: str = Form(...), # "left", "right", or "tie" + ): + """Record the user's vote and reveal model names if blind.""" + user = get_current_user(request) + db = SessionLocal() + try: + comp = db.query(Comparison).filter(Comparison.id == comp_id).first() + if not comp: + raise HTTPException(404, "Comparison not found") + # SECURITY: strict ownership — null-owner Comparisons were + # accessible to every user. + if user and comp.owner != user: + raise HTTPException(404, "Comparison not found") + if comp.winner: + raise HTTPException(400, "Already voted") + + mapping = json.loads(comp.blind_mapping) if comp.blind_mapping else {"left": "a", "right": "b"} + + if winner == "tie": + comp.winner = "tie" + elif winner == "left": + comp.winner = mapping["left"] + elif winner == "right": + comp.winner = mapping["right"] + else: + raise HTTPException(400, "winner must be 'left', 'right', or 'tie'") + + comp.voted_at = datetime.utcnow() + db.commit() + + return { + "winner": comp.winner, + "model_a": comp.model_a, + "model_b": comp.model_b, + "revealed": { + "left": comp.model_a if mapping["left"] == "a" else comp.model_b, + "right": comp.model_a if mapping["right"] == "a" else comp.model_b, + }, + } + finally: + db.close() + + @router.post("/record") + def record_comparison(request: Request, body: RecordVoteRequest): + """Lightweight endpoint to record a comparison vote from the frontend.""" + user = get_current_user(request) + comp_id = str(uuid.uuid4()) + + model_a = body.models[0] if len(body.models) > 0 else "" + model_b = body.models[1] if len(body.models) > 1 else "" + + # For N>2 models, store the full list as JSON in blind_mapping + if len(body.models) > 2: + blind_mapping = json.dumps({"models": body.models}) + else: + blind_mapping = None + + db = SessionLocal() + try: + comp = Comparison( + id=comp_id, + prompt=body.prompt[:500], + model_a=model_a, + model_b=model_b, + endpoint_a="", + endpoint_b="", + winner=body.winner, + is_blind=body.is_blind, + blind_mapping=blind_mapping, + voted_at=datetime.utcnow(), + owner=user, + ) + db.add(comp) + db.commit() + finally: + db.close() + + return {"status": "ok", "id": comp_id} + + @router.get("/history") + def list_comparisons(request: Request): + """List past comparisons.""" + user = get_current_user(request) + db = SessionLocal() + try: + q = db.query(Comparison) + if user: + q = q.filter(Comparison.owner == user) + comps = q.order_by(Comparison.created_at.desc()).limit(50).all() + return [ + { + "id": c.id, + "prompt": c.prompt[:100], + "model_a": c.model_a, + "model_b": c.model_b, + "winner": c.winner, + "is_blind": c.is_blind, + "voted_at": c.voted_at.isoformat() if c.voted_at else None, + "created_at": c.created_at.isoformat() if c.created_at else None, + } + for c in comps + ] + finally: + db.close() + + @router.delete("/{comp_id}") + def delete_comparison(request: Request, comp_id: str): + """Delete a comparison and its ephemeral sessions.""" + user = get_current_user(request) + db = SessionLocal() + try: + comp = db.query(Comparison).filter(Comparison.id == comp_id).first() + if not comp: + raise HTTPException(404, "Comparison not found") + # SECURITY: strict ownership — null-owner Comparisons were + # accessible to every user. + if user and comp.owner != user: + raise HTTPException(404, "Comparison not found") + db.delete(comp) + db.commit() + return {"status": "deleted"} + finally: + db.close() + + return router diff --git a/routes/compare_routes.py b/routes/compare_routes.py index 18b21651a..d1d24c273 100644 --- a/routes/compare_routes.py +++ b/routes/compare_routes.py @@ -1,246 +1,18 @@ -# routes/compare_routes.py -"""Model A/B comparison routes.""" -import json -import uuid -import random -from datetime import datetime -from fastapi import APIRouter, Form, HTTPException, Request -from typing import List -from pydantic import BaseModel -import logging +"""Backward-compat shim — canonical location is routes/compare/compare_routes.py. -from core.database import Comparison, SessionLocal -from core.session_manager import SessionManager -from src.auth_helpers import get_current_user +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.compare_routes``, ``from routes.compare_routes import X``, +``importlib.import_module("routes.compare_routes")``, and the +``import ... as cr`` + ``monkeypatch.setattr(cr, "SessionLocal", ...)`` / +``"_owned_endpoint_by_url"`` / ``"_owned_endpoint_by_id"`` pattern used by +test_endpoint_owner_scope_followup.py all operate on the *same* object the +application actually uses. Keeps existing import paths working after +slice 2i (#4082/#4071). Source-introspection tests read the canonical file +by path. +""" -logger = logging.getLogger(__name__) +import sys as _sys -router = APIRouter(prefix="/api/compare", tags=["compare"]) +from routes.compare import compare_routes as _canonical # noqa: F401 - -class RecordVoteRequest(BaseModel): - prompt: str - models: List[str] - winner: str # model name or "tie" - is_blind: bool = True - - -def setup_compare_routes(session_manager: SessionManager): - """Setup comparison routes.""" - - @router.post("/start") - def start_comparison( - request: Request, - prompt: str = Form(...), - model_a: str = Form(...), - model_b: str = Form(...), - endpoint_a: str = Form(...), - endpoint_b: str = Form(...), - is_blind: str = Form("true"), - ): - """Create two ephemeral sessions and a comparison record. - - Returns the comparison ID and the two session IDs so the client - can fire two independent SSE streams to /api/chat_stream. - """ - comp_id = str(uuid.uuid4()) - sid_a = str(uuid.uuid4()) - sid_b = str(uuid.uuid4()) - - # Create ephemeral sessions (prefixed [CMP]) - for sid, model, endpoint in [(sid_a, model_a, endpoint_a), (sid_b, model_b, endpoint_b)]: - user = getattr(request.state, 'current_user', None) - session_manager.create_session( - session_id=sid, - name=f"[CMP] {model.split('/')[-1]}", - endpoint_url=endpoint, - model=model, - rag=False, - owner=user, - ) - # Copy API key from endpoint config - db = SessionLocal() - try: - from core.database import ModelEndpoint - # Find matching endpoint by URL - ep = db.query(ModelEndpoint).filter( - ModelEndpoint.base_url == endpoint.replace('/chat/completions', '') - ).first() - if ep and ep.api_key: - s = session_manager.sessions.get(sid) - if s: - s.headers = {"Authorization": f"Bearer {ep.api_key}"} - finally: - db.close() - - # Blind mapping: randomly assign left/right - blind = str(is_blind).lower() == "true" - if blind: - mapping = {"left": "a", "right": "b"} - if random.random() > 0.5: - mapping = {"left": "b", "right": "a"} - else: - mapping = {"left": "a", "right": "b"} - - # Store comparison record - db = SessionLocal() - try: - comp = Comparison( - id=comp_id, - prompt=prompt, - model_a=model_a, - model_b=model_b, - endpoint_a=endpoint_a, - endpoint_b=endpoint_b, - is_blind=blind, - blind_mapping=json.dumps(mapping), - owner=user, - ) - db.add(comp) - db.commit() - finally: - db.close() - - # Map session IDs to left/right based on blind mapping - session_left = sid_a if mapping["left"] == "a" else sid_b - session_right = sid_a if mapping["right"] == "a" else sid_b - - return { - "id": comp_id, - "session_left": session_left, - "session_right": session_right, - "model_left": model_a if mapping["left"] == "a" else model_b, - "model_right": model_a if mapping["right"] == "a" else model_b, - "is_blind": blind, - "mapping": mapping, - } - - @router.post("/{comp_id}/vote") - def vote_comparison( - request: Request, - comp_id: str, - winner: str = Form(...), # "left", "right", or "tie" - ): - """Record the user's vote and reveal model names if blind.""" - user = get_current_user(request) - db = SessionLocal() - try: - comp = db.query(Comparison).filter(Comparison.id == comp_id).first() - if not comp: - raise HTTPException(404, "Comparison not found") - # SECURITY: strict ownership — null-owner Comparisons were - # accessible to every user. - if user and comp.owner != user: - raise HTTPException(404, "Comparison not found") - if comp.winner: - raise HTTPException(400, "Already voted") - - mapping = json.loads(comp.blind_mapping) if comp.blind_mapping else {"left": "a", "right": "b"} - - if winner == "tie": - comp.winner = "tie" - elif winner == "left": - comp.winner = mapping["left"] - elif winner == "right": - comp.winner = mapping["right"] - else: - raise HTTPException(400, "winner must be 'left', 'right', or 'tie'") - - comp.voted_at = datetime.utcnow() - db.commit() - - return { - "winner": comp.winner, - "model_a": comp.model_a, - "model_b": comp.model_b, - "revealed": { - "left": comp.model_a if mapping["left"] == "a" else comp.model_b, - "right": comp.model_a if mapping["right"] == "a" else comp.model_b, - }, - } - finally: - db.close() - - @router.post("/record") - def record_comparison(request: Request, body: RecordVoteRequest): - """Lightweight endpoint to record a comparison vote from the frontend.""" - user = get_current_user(request) - comp_id = str(uuid.uuid4()) - - model_a = body.models[0] if len(body.models) > 0 else "" - model_b = body.models[1] if len(body.models) > 1 else "" - - # For N>2 models, store the full list as JSON in blind_mapping - if len(body.models) > 2: - blind_mapping = json.dumps({"models": body.models}) - else: - blind_mapping = None - - db = SessionLocal() - try: - comp = Comparison( - id=comp_id, - prompt=body.prompt[:500], - model_a=model_a, - model_b=model_b, - endpoint_a="", - endpoint_b="", - winner=body.winner, - is_blind=body.is_blind, - blind_mapping=blind_mapping, - voted_at=datetime.utcnow(), - owner=user, - ) - db.add(comp) - db.commit() - finally: - db.close() - - return {"status": "ok", "id": comp_id} - - @router.get("/history") - def list_comparisons(request: Request): - """List past comparisons.""" - user = get_current_user(request) - db = SessionLocal() - try: - q = db.query(Comparison) - if user: - q = q.filter(Comparison.owner == user) - comps = q.order_by(Comparison.created_at.desc()).limit(50).all() - return [ - { - "id": c.id, - "prompt": c.prompt[:100], - "model_a": c.model_a, - "model_b": c.model_b, - "winner": c.winner, - "is_blind": c.is_blind, - "voted_at": c.voted_at.isoformat() if c.voted_at else None, - "created_at": c.created_at.isoformat() if c.created_at else None, - } - for c in comps - ] - finally: - db.close() - - @router.delete("/{comp_id}") - def delete_comparison(request: Request, comp_id: str): - """Delete a comparison and its ephemeral sessions.""" - user = get_current_user(request) - db = SessionLocal() - try: - comp = db.query(Comparison).filter(Comparison.id == comp_id).first() - if not comp: - raise HTTPException(404, "Comparison not found") - # SECURITY: strict ownership — null-owner Comparisons were - # accessible to every user. - if user and comp.owner != user: - raise HTTPException(404, "Comparison not found") - db.delete(comp) - db.commit() - return {"status": "deleted"} - finally: - db.close() - - return router +_sys.modules[__name__] = _canonical diff --git a/routes/contacts/__init__.py b/routes/contacts/__init__.py new file mode 100644 index 000000000..382f8f848 --- /dev/null +++ b/routes/contacts/__init__.py @@ -0,0 +1,5 @@ +"""Contacts route domain package (slice 2e, #4082/#4071). + +Contains contacts_routes.py, migrated from the flat routes/ directory. +Backward-compat shim at routes/contacts_routes.py re-exports from here. +""" diff --git a/routes/contacts/contacts_routes.py b/routes/contacts/contacts_routes.py new file mode 100644 index 000000000..8a6dde8e3 --- /dev/null +++ b/routes/contacts/contacts_routes.py @@ -0,0 +1,916 @@ +""" +contacts_routes.py + +CardDAV contacts integration. Reads from local Radicale, supports +search and adding new contacts. +""" + +import re +import logging +import uuid +import json +import csv +import io +import os +import inspect +import httpx +from pathlib import Path +from datetime import datetime +from urllib.parse import urljoin, urlparse, urlunparse + +from core.log_safety import redact_url +from fastapi import APIRouter, Query, Depends, Response, HTTPException +from typing import List, Dict, Optional + +from core.middleware import require_admin +from src.url_safety import check_outbound_url + +logger = logging.getLogger(__name__) + +from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE +DATA_DIR = Path(_DATA_DIR) +SETTINGS_FILE = Path(_SETTINGS_FILE) +LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE) + + +def _load_settings(): + if SETTINGS_FILE.exists(): + return json.loads(SETTINGS_FILE.read_text(encoding="utf-8")) + return {} + + +def _save_settings(settings): + from core.atomic_io import atomic_write_json + atomic_write_json(str(SETTINGS_FILE), settings, indent=2) + + +def _get_carddav_config(): + import os + settings = _load_settings() + password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", "")) + if password and "carddav_password" in settings: + from src.secret_storage import decrypt + password = decrypt(password) + return { + "url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")), + "username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")), + "password": password, + } + + +def _carddav_configured(cfg: Optional[Dict] = None) -> bool: + cfg = cfg or _get_carddav_config() + return bool((cfg.get("url") or "").strip()) + + +def _validate_carddav_url(url: str) -> str: + cleaned = (url if isinstance(url, str) else "").strip().rstrip("/") + ok, reason = check_outbound_url( + cleaned, + block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + raise ValueError(f"Rejected CardDAV URL: {reason}") + return cleaned + + +def _carddav_base_url(cfg: Dict) -> str: + return _validate_carddav_url(cfg.get("url") or "") + + +def _normalize_contact(contact: Dict) -> Dict: + emails = [] + for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]): + e = str(e or "").strip() + if e and e not in emails: + emails.append(e) + phones = [] + for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]): + p = str(p or "").strip() + if p and p not in phones: + phones.append(p) + name = str(contact.get("name") or "").strip() + if not name and emails: + name = emails[0].split("@")[0] + address = str(contact.get("address") or "").strip() + return { + "uid": str(contact.get("uid") or uuid.uuid4()), + "name": name, + "emails": emails, + "phones": phones, + "address": address, + } + + +def _load_local_contacts() -> List[Dict]: + try: + if not LOCAL_CONTACTS_FILE.exists(): + return [] + data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8")) + rows = data.get("contacts", data) if isinstance(data, dict) else data + return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)] + except Exception as e: + logger.error(f"Failed to load local contacts: {e}") + return [] + + +def _save_local_contacts(contacts: List[Dict]) -> None: + from core.atomic_io import atomic_write_json + DATA_DIR.mkdir(parents=True, exist_ok=True) + atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2) + _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts] + _contact_cache["fetched_at"] = datetime.utcnow() + + +# ── vCard parsing ── + +def _vunesc(value: str) -> str: + """Reverse _vesc() — turn escaped vCard text back into the raw value. + Order matters: handle \\n/\\, /\\; first, backslash-unescape last.""" + if not value: + return value + out = [] + i = 0 + while i < len(value): + ch = value[i] + if ch == "\\" and i + 1 < len(value): + nxt = value[i + 1] + if nxt in ("n", "N"): + out.append("\n") + elif nxt in (",", ";", "\\"): + out.append(nxt) + else: + out.append(nxt) + i += 2 + else: + out.append(ch) + i += 1 + return "".join(out) + + +def _parse_vcards(text: str) -> List[Dict]: + """Parse a stream of vCards into dicts with name, email, phone.""" + # Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single + # space or tab is a continuation of the previous logical line. Real + # CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN / + # PHOTO lines, and splitting on raw newlines without unfolding dropped the + # continuation (e.g. "...@example\n .com" lost the ".com"), truncating the + # email/name. + text = re.sub(r"\r\n[ \t]", "", text or "") + text = re.sub(r"\n[ \t]", "", text) + contacts = [] + for block in re.split(r"BEGIN:VCARD", text): + if not block.strip(): + continue + contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""} + for line in block.split("\n"): + line = line.strip() + # Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...") + # that Apple Contacts / iCloud / many CardDAV servers emit by + # default — without this the property-name checks below miss those + # lines and silently drop the email / phone. The group token only + # precedes the property name, so it is safe to strip for matching + # and value extraction, and a no-op for non-grouped lines. + name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1) + if name_part.startswith("FN:") or name_part.startswith("FN;"): + contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else "" + elif name_part.startswith("EMAIL"): + # Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar + if ":" in name_part: + email_addr = _vunesc(name_part.split(":", 1)[1]) + if email_addr and email_addr not in contact["emails"]: + contact["emails"].append(email_addr) + elif name_part.startswith("TEL"): + if ":" in name_part: + phone = _vunesc(name_part.split(":", 1)[1]) + if phone and phone not in contact["phones"]: + contact["phones"].append(phone) + elif name_part.startswith("ADR"): + # vCard ADR is 7 semicolon-separated components: + # post-office-box;extended-address;street;locality;region;postal-code;country. + # Recover a human-readable string by joining non-empty + # components with ", ". + if ":" in name_part: + raw = name_part.split(":", 1)[1] + parts = [_vunesc(p).strip() for p in raw.split(";")] + contact["address"] = ", ".join(p for p in parts if p) + elif name_part.startswith("UID:"): + contact["uid"] = _vunesc(name_part[4:]) + if contact["name"] or contact["emails"]: + contacts.append(contact) + return contacts + + +def _vesc(value: str) -> str: + """Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma, + semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd' + or any value containing a newline produces a malformed vCard (broken + N/FN fields) or could inject arbitrary properties.""" + return ( + (value or "") + .replace("\\", "\\\\") + .replace("\n", "\\n") + .replace("\r", "") + .replace(",", "\\,") + .replace(";", "\\;") + ) + + +def _build_vcard(name: str, email: str, uid: Optional[str] = None, + emails: Optional[List[str]] = None, + phones: Optional[List[str]] = None, + address: Optional[str] = None) -> str: + """Build a vCard. Accepts either a single `email` (legacy callers) or + full `emails`/`phones` lists (edit path). The first email is marked + PREF=1. All values are RFC-6350-escaped.""" + if not uid: + uid = str(uuid.uuid4()) + # Normalize email lists — `email` arg is a convenience for single-email + # creation; `emails` (if given) is authoritative. + email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()] + phone_list = [p.strip() for p in (phones or []) if p and p.strip()] + # Try to split name into first/last + parts = name.strip().split() + if len(parts) >= 2: + first = parts[0] + last = " ".join(parts[1:]) + else: + first = name + last = "" + # N field is structured (5 components separated by ';') — escape each + # component individually so a comma in the name doesn't split it. + n_field = f"{_vesc(last)};{_vesc(first)};;;" + lines = [ + "BEGIN:VCARD", + "VERSION:4.0", + f"UID:{_vesc(uid)}", + f"FN:{_vesc(name)}", + f"N:{n_field}", + ] + for i, em in enumerate(email_list): + # First email is the preferred one. + lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}") + for ph in phone_list: + lines.append(f"TEL:{_vesc(ph)}") + # Address: stuff the whole human-readable string into the street + # component of ADR. vCard ADR has 7 semicolon-separated components: + # post-office-box;extended-address;street;locality;region;postal-code;country. + addr = (address or "").strip() + if addr: + lines.append(f"ADR:;;{_vesc(addr)};;;;") + lines.append("END:VCARD") + return "\r\n".join(lines) + "\r\n" + + +# ── In-memory cache ── + +_contact_cache = {"contacts": [], "fetched_at": None} + + +def _abs_url(href: str) -> str: + """Combine a multistatus (an absolute path like + /user/contacts/x.vcf) with the configured CardDAV server origin so we + get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only + for the configured origin; a cross-origin href is treated as a path on the + configured server so a malicious CardDAV response cannot redirect later + writes/deletes to cloud metadata or another host.""" + cfg = _get_carddav_config() + base = _carddav_base_url(cfg) + base_p = urlparse(base) + joined = urljoin(base.rstrip("/") + "/", href or "") + joined_p = urlparse(joined) + if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc): + joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, "")) + return _validate_carddav_url(joined) + + +# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request, +# alongside the resource href. Lets us map each contact's UID to the real +# server resource path (which is NOT always .vcf for contacts created +# by other clients). +_ADDRESSBOOK_QUERY = ( + '' + '' + '' + '' + '' +) + + +def _fetch_via_report(cfg, auth): + """Try a CardDAV REPORT addressbook-query — returns contacts WITH an + `href` field, or None if the server doesn't support it / errors.""" + from defusedxml import ElementTree as ET + try: + r = httpx.request( + "REPORT", cfg["url"], + content=_ADDRESSBOOK_QUERY.encode("utf-8"), + headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"}, + auth=auth, timeout=10, + ) + if r.status_code not in (207, 200): + return None + root = ET.fromstring(r.text) + ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"} + out = [] + for resp in root.findall("D:response", ns): + href_el = resp.find("D:href", ns) + data_el = resp.find(".//C:address-data", ns) + if href_el is None or data_el is None or not (data_el.text or "").strip(): + continue + parsed = _parse_vcards(data_el.text) + if not parsed: + continue + c = parsed[0] + c["href"] = href_el.text.strip() + out.append(c) + # If the REPORT parsed to ZERO contacts, don't trust it — some + # CardDAV servers treat an empty as "match nothing" and + # return a valid-but-empty 207. Return None so the caller falls + # back to the plain GET (which lists everything). A genuinely empty + # address book just costs one extra GET that also returns nothing. + if not out: + return None + return out + except Exception as e: + logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}") + return None + + +def _fetch_contacts(force=False): + """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON.""" + if not force and _contact_cache["fetched_at"]: + age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds() + if age < 60: + return _contact_cache["contacts"] + + cfg = _get_carddav_config() + if not _carddav_configured(cfg): + contacts = _load_local_contacts() + _contact_cache["contacts"] = contacts + _contact_cache["fetched_at"] = datetime.utcnow() + return contacts + + try: + cfg["url"] = _carddav_base_url(cfg) + auth = None + if cfg["username"]: + auth = (cfg["username"], cfg["password"]) + # Preferred path: REPORT gives us hrefs for reliable edit/delete. + contacts = _fetch_via_report(cfg, auth) + if contacts is None: + # Fallback: plain GET, concatenated vCards, no hrefs. + r = httpx.get(cfg["url"], auth=auth, timeout=10) + if r.status_code != 200: + logger.warning(f"CardDAV returned {r.status_code}") + return _contact_cache["contacts"] + contacts = _parse_vcards(r.text) + _contact_cache["contacts"] = contacts + _contact_cache["fetched_at"] = datetime.utcnow() + return contacts + except Exception as e: + logger.error(f"Failed to fetch contacts: {e}") + return _contact_cache["contacts"] + + +def _resolve_resource_url(uid: str) -> str: + """Map a contact UID to its real CardDAV resource URL. Uses the href + captured during fetch when available (handles contacts whose filename + != UID); falls back to the .vcf guess for app-created contacts or + when no href is known.""" + def _lookup(): + for c in _contact_cache.get("contacts", []): + if c.get("uid") == uid and c.get("href"): + return _abs_url(c["href"]) + return None + found = _lookup() + if found: + return found + # Not in cache (or no href) — refresh once and retry before guessing. + try: + _fetch_contacts(force=True) + except Exception: + pass + return _lookup() or _vcard_url(uid) + + +def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool: + """Add a new contact via CardDAV or local contacts.""" + email = (email or "").strip() + phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()] + cfg = _get_carddav_config() + if not _carddav_configured(cfg): + contacts = _load_local_contacts() + email_l = email.lower() + for c in contacts: + if email_l and email_l in [e.lower() for e in c.get("emails", [])]: + return True + if phone_list and any(p in (c.get("phones") or []) for p in phone_list): + return True + contacts.append(_normalize_contact({ + "name": name, + "emails": [email] if email else [], + "phones": phone_list, + "address": address, + })) + _save_local_contacts(contacts) + return True + + contact_uid = str(uuid.uuid4()) + vcard = _build_vcard(name, email, contact_uid, address=address, phones=phone_list) + try: + url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf" + auth = None + if cfg["username"]: + auth = (cfg["username"], cfg["password"]) + r = httpx.put( + url, + data=vcard.encode("utf-8"), + headers={"Content-Type": "text/vcard; charset=utf-8"}, + auth=auth, + timeout=10, + ) + if r.status_code in (200, 201, 204): + # Invalidate cache + _contact_cache["fetched_at"] = None + return True + logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}") + return False + except Exception as e: + logger.error(f"Failed to create contact: {e}") + return False + + +def _vcard_url(uid: str) -> str: + """The CardDAV resource URL for a given contact UID. The uid is URL- + encoded so a value containing '/', '..' or other path chars can't + escape the collection and target an arbitrary CardDAV resource.""" + from urllib.parse import quote + cfg = _get_carddav_config() + return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf" + + +def _import_vcards(text: str) -> Dict: + """Import a (possibly multi-card) .vcf blob. Each card is PUT to the + CardDAV server PRESERVING its full original content (ADR/ORG/photo/ + etc.) — we don't rebuild it, just ensure it has VERSION + UID and + normalize line endings. Returns {imported, failed, total}.""" + from urllib.parse import quote + cfg = _get_carddav_config() + if not cfg.get("url"): + parsed = _parse_vcards(text) + contacts = _load_local_contacts() + existing = { + e.lower() + for c in contacts + for e in (c.get("emails") or []) + if e + } + imported = 0 + for c in parsed: + emails = [e for e in (c.get("emails") or []) if e] + if emails and any(e.lower() in existing for e in emails): + continue + contacts.append(_normalize_contact(c)) + for e in emails: + existing.add(e.lower()) + imported += 1 + if imported: + _save_local_contacts(contacts) + return {"imported": imported, "failed": 0, "total": len(parsed)} + try: + base_url = _carddav_base_url(cfg) + except ValueError as e: + logger.warning("CardDAV import URL rejected: %s", e) + return {"imported": 0, "failed": 0, "total": 0, "error": str(e)} + auth = (cfg["username"], cfg["password"]) if cfg["username"] else None + # Split into individual cards. re.split drops the BEGIN line, so we + # re-add it. Normalize CRLF. + raw = (text or "").replace("\r\n", "\n").replace("\r", "\n") + blocks = [] + for chunk in raw.split("BEGIN:VCARD"): + chunk = chunk.strip() + if not chunk: + continue + # Trim anything after END:VCARD (defensive). + end = chunk.upper().find("END:VCARD") + body = chunk[: end + len("END:VCARD")] if end != -1 else chunk + blocks.append("BEGIN:VCARD\n" + body) + imported = 0 + failed = 0 + for block in blocks: + # Extract or assign a UID. + m = re.search(r"^UID:(.+)$", block, re.MULTILINE) + uid = (m.group(1).strip() if m else "") or str(uuid.uuid4()) + if not m: + # Inject a UID right after the VERSION line (or after BEGIN). + if re.search(r"^VERSION:", block, re.MULTILINE): + block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE) + else: + block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1) + elif not re.search(r"^VERSION:", block, re.MULTILINE): + block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1) + vcard = block.replace("\n", "\r\n") + "\r\n" + url = base_url + "/" + quote(uid, safe="") + ".vcf" + try: + r = httpx.put( + url, data=vcard.encode("utf-8"), + headers={"Content-Type": "text/vcard; charset=utf-8"}, + auth=auth, timeout=15, + ) + if r.status_code in (200, 201, 204): + imported += 1 + else: + failed += 1 + logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}") + except Exception as e: + failed += 1 + logger.error(f"Import PUT {uid} failed: {e}") + if imported: + _contact_cache["fetched_at"] = None + return {"imported": imported, "failed": failed, "total": len(blocks)} + + +def _import_csv_contacts(text: str) -> Dict: + """Import contacts from CSV. Supports common headers: + name/full_name/display_name, email/email_address/e-mail, phone/tel. + Falls back to first columns as name,email,phone when no headers exist.""" + raw = (text or "").strip() + if not raw: + return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"} + + try: + sample = raw[:2048] + dialect = csv.Sniffer().sniff(sample) + except Exception: + dialect = csv.excel + + stream = io.StringIO(raw) + try: + has_header = csv.Sniffer().has_header(raw[:2048]) + except Exception: + has_header = True + + rows = [] + if has_header: + reader = csv.DictReader(stream, dialect=dialect) + for row in reader: + lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()} + name = ( + lowered.get("name") or lowered.get("full name") or lowered.get("full_name") + or lowered.get("display name") or lowered.get("display_name") + or lowered.get("fn") or "" + ) + email = ( + lowered.get("email") or lowered.get("email address") + or lowered.get("email_address") or lowered.get("e-mail") + or lowered.get("mail") or "" + ) + phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or "" + rows.append((name, email, phone)) + else: + stream.seek(0) + reader = csv.reader(stream, dialect=dialect) + for row in reader: + cols = [(c or "").strip() for c in row] + if not any(cols): + continue + rows.append(( + cols[0] if len(cols) > 0 else "", + cols[1] if len(cols) > 1 else "", + cols[2] if len(cols) > 2 else "", + )) + + imported = 0 + failed = 0 + total = 0 + existing_emails = { + e.lower() + for c in _fetch_contacts() + for e in (c.get("emails") or []) + if e + } + for name, email, phone in rows: + email = (email or "").strip() + name = (name or "").strip() or (email.split("@")[0] if email else "") + if not email: + continue + total += 1 + if email.lower() in existing_emails: + continue + ok = _create_contact(name, email) + if ok: + imported += 1 + existing_emails.add(email.lower()) + # If the CSV had a phone number, rewrite the just-created row + # through the richer update path so phone lands in CardDAV too. + if phone: + try: + contacts = _fetch_contacts(force=True) + created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None) + if created and created.get("uid"): + _update_contact(created["uid"], name, [email], [phone]) + except Exception: + pass + else: + failed += 1 + + if imported: + _contact_cache["fetched_at"] = None + return {"imported": imported, "failed": failed, "total": total} + + +def _contacts_to_vcf(contacts: List[Dict]) -> str: + return "".join( + _build_vcard( + c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"), + "", + uid=c.get("uid") or str(uuid.uuid4()), + emails=c.get("emails") or [], + phones=c.get("phones") or [], + ) + for c in contacts + ) + + +def _contacts_to_csv(contacts: List[Dict]) -> str: + out = io.StringIO() + writer = csv.writer(out) + writer.writerow(["name", "email", "phone"]) + for c in contacts: + emails = c.get("emails") or [""] + phones = c.get("phones") or [""] + max_len = max(len(emails), len(phones), 1) + for i in range(max_len): + writer.writerow([ + c.get("name") or "", + emails[i] if i < len(emails) else "", + phones[i] if i < len(phones) else "", + ]) + return out.getvalue() + + +def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool: + """Rewrite an existing contact via CardDAV or local contacts.""" + cfg = _get_carddav_config() + if not _carddav_configured(cfg): + contacts = _load_local_contacts() + found = False + out = [] + for c in contacts: + if c.get("uid") == uid: + # Preserve existing address when caller passes "" (only + # updating name/emails/phones, not touching address). + addr = address if address else c.get("address", "") + out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr})) + found = True + else: + out.append(c) + if not found: + out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address})) + _save_local_contacts(out) + return True + + vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address) + # Use the real resource href (handles externally-created contacts whose + # filename != UID); falls back to the .vcf guess. + try: + url = _resolve_resource_url(uid) + auth = (cfg["username"], cfg["password"]) if cfg["username"] else None + r = httpx.put( + url, + data=vcard.encode("utf-8"), + headers={"Content-Type": "text/vcard; charset=utf-8"}, + auth=auth, + timeout=10, + ) + if r.status_code in (200, 201, 204): + _contact_cache["fetched_at"] = None + return True + logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}") + return False + except Exception as e: + logger.error(f"Failed to update contact: {e}") + return False + + +def _delete_contact(uid: str) -> bool: + """Delete a contact via CardDAV or local contacts.""" + cfg = _get_carddav_config() + if not _carddav_configured(cfg): + contacts = _load_local_contacts() + remaining = [c for c in contacts if c.get("uid") != uid] + _save_local_contacts(remaining) + return True + + try: + url = _resolve_resource_url(uid) + auth = (cfg["username"], cfg["password"]) if cfg["username"] else None + r = httpx.delete(url, auth=auth, timeout=10) + if r.status_code in (200, 204, 404): + # Invalidate cache so the next fetch sees the server truth. + _contact_cache["fetched_at"] = None + # Verify: force a fresh fetch and check the UID is actually gone. + # A 404 on the guessed URL ({uid}.vcf) can mean the contact + # lives at a different resource URL — the DELETE missed it but + # we'd silently report success. This check catches that. + fresh = _fetch_contacts(force=True) + still_there = any(c.get("uid") == uid for c in fresh) + if still_there: + logger.warning( + f"CardDAV DELETE reported success for {uid} " + f"but UID still present after re-fetch — " + f"resource URL may differ from {redact_url(url)}" + ) + return False + if r.status_code == 404: + logger.info(f"CardDAV DELETE 404 for {uid} — already gone") + return True + logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}") + return False + except Exception as e: + logger.error(f"Failed to delete contact: {e}") + return False + + +# ── Routes ── + +def setup_contacts_routes(): + router = APIRouter(prefix="/api/contacts", tags=["contacts"]) + + @router.get("/list") + async def list_contacts(_admin: str = Depends(require_admin)): + """List all contacts.""" + contacts = _fetch_contacts() + return {"contacts": contacts, "count": len(contacts)} + + @router.get("/search") + async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)): + """Search contacts by name or email. Returns up to 10 matches.""" + contacts = _fetch_contacts() + if not q: + return {"results": []} + q_lower = q.lower() + results = [] + for c in contacts: + if q_lower in c["name"].lower(): + results.append(c) + continue + for em in c["emails"]: + if q_lower in em.lower(): + results.append(c) + break + return {"results": results[:10]} + + @router.post("/add") + async def add_contact(data: dict, _admin: str = Depends(require_admin)): + """Add a new contact.""" + name = (data.get("name") or "").strip() + email = (data.get("email") or "").strip() + phone = (data.get("phone") or "").strip() + phones = [str(p or "").strip() for p in (data.get("phones") or []) if str(p or "").strip()] + if phone and phone not in phones: + phones.insert(0, phone) + address = (data.get("address") or "").strip() + if not name and email: + name = email.split("@")[0] + if not name and not email and not phones and not address: + return {"success": False, "error": "Name, email, phone, or address required"} + if not name: + name = email.split("@")[0] if email else (phones[0] if phones else "Contact") + contacts = _fetch_contacts() + for c in contacts: + if email and email.lower() in [e.lower() for e in c.get("emails", [])]: + return {"success": True, "message": "Already exists", "contact": c} + if phones and any(p in (c.get("phones") or []) for p in phones): + return {"success": True, "message": "Already exists", "contact": c} + create_params = inspect.signature(_create_contact).parameters + if "phones" in create_params: + ok = _create_contact(name, email, address, phones=phones) + elif len(create_params) >= 3: + ok = _create_contact(name, email, address) + else: + ok = _create_contact(name, email) + # If a phone was provided, do an immediate update to thread it + # through (the simple _create_contact signature only takes name + + # email + address; phones happen via update). + if ok and phones and "phones" not in create_params: + try: + fresh = _fetch_contacts(force=True) + created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None) + if created: + _update_contact( + created["uid"], name, + created.get("emails", []), + phones, + address, + ) + except Exception: + pass + return {"success": ok} + + @router.post("/import") + async def import_vcf(data: dict, _admin: str = Depends(require_admin)): + """Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}.""" + # Coerce defensively: a non-string vcf/text/csv (e.g. a number or list + # in the JSON body) would otherwise reach .strip() and 500 with an + # AttributeError instead of degrading to a clean "no data" response. + text = str(data.get("vcf") or data.get("text") or "") + csv_text = str(data.get("csv") or "") + if text.strip(): + if "BEGIN:VCARD" not in text.upper(): + return {"success": False, "error": "No vCard data found"} + result = _import_vcards(text) + elif csv_text.strip(): + result = _import_csv_contacts(csv_text) + else: + return {"success": False, "error": "No contact data found"} + result["success"] = result.get("imported", 0) > 0 + return result + + @router.get("/export") + async def export_contacts( + format: str = Query("vcf", pattern="^(vcf|csv)$"), + _admin: str = Depends(require_admin), + ): + """Export all contacts as vCard or CSV.""" + contacts = _fetch_contacts(force=True) + if format == "csv": + content = _contacts_to_csv(contacts) + media_type = "text/csv; charset=utf-8" + filename = "odysseus-contacts.csv" + else: + content = _contacts_to_vcf(contacts) + media_type = "text/vcard; charset=utf-8" + filename = "odysseus-contacts.vcf" + return Response( + content=content, + media_type=media_type, + headers={"Content-Disposition": f'attachment; filename="{filename}"'}, + ) + + @router.get("/config") + async def get_config(_admin: str = Depends(require_admin)): + cfg = _get_carddav_config() + # Mask password + if cfg["password"]: + cfg["password"] = "***" + return cfg + + @router.put("/config") + async def update_config(data: dict, _admin: str = Depends(require_admin)): + settings = _load_settings() + for key in ("carddav_url", "carddav_username", "carddav_password"): + if key in data: + if key == "carddav_url" and str(data[key] or "").strip(): + try: + settings[key] = _validate_carddav_url(data[key]) + except ValueError as e: + raise HTTPException(400, str(e)) + else: + value = data[key] + if key == "carddav_password" and value: + from src.secret_storage import encrypt + value = encrypt(value) + settings[key] = value + _save_settings(settings) + # Force re-fetch + _contact_cache["fetched_at"] = None + return {"success": True} + + @router.delete("/clear") + async def clear_contacts(_admin: str = Depends(require_admin)): + """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache.""" + _save_local_contacts([]) + return {"success": True} + + # NOTE: the /{uid} routes are declared LAST so the literal paths above + # (/list, /search, /add, /config) win — otherwise PUT /config would + # match PUT /{uid} with uid="config". + @router.put("/{uid}") + async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)): + """Edit an existing contact — name / emails / phones / address.""" + name = (data.get("name") or "").strip() + emails = data.get("emails") + phones = data.get("phones") + if emails is None and data.get("email"): + emails = [data["email"]] + emails = [e.strip() for e in (emails or []) if e and e.strip()] + phones = [p.strip() for p in (phones or []) if p and p.strip()] + address = (data.get("address") or "").strip() + if not name and not emails and not address: + return {"success": False, "error": "Name, email, or address required"} + if not name and emails: + name = emails[0].split("@")[0] + ok = _update_contact(uid, name, emails, phones, address) + return {"success": ok} + + @router.delete("/{uid}") + async def delete_contact(uid: str, _admin: str = Depends(require_admin)): + """Delete a contact by UID.""" + if not uid: + return {"success": False, "error": "UID required"} + ok = _delete_contact(uid) + return {"success": ok} + + return router diff --git a/routes/contacts_routes.py b/routes/contacts_routes.py index 4d5595956..5a00acc40 100644 --- a/routes/contacts_routes.py +++ b/routes/contacts_routes.py @@ -1,783 +1,13 @@ -""" -contacts_routes.py +"""Backward-compat shim — canonical location is routes/contacts/contacts_routes.py. -CardDAV contacts integration. Reads from local Radicale, supports -search and adding new contacts. +This module is replaced in ``sys.modules`` by the canonical module object so +that ``import routes.contacts_routes``, ``from routes.contacts_routes import X``, +``importlib.import_module("routes.contacts_routes")``, and string-targeted +monkeypatches all operate on the same object the application actually uses. """ -import re -import logging -import uuid -import json -import csv -import io -import httpx -from pathlib import Path -from datetime import datetime -from fastapi import APIRouter, Query, Depends, Response -from typing import List, Dict, Optional - -from src.auth_helpers import require_user -from core.middleware import require_admin - -logger = logging.getLogger(__name__) - -DATA_DIR = Path(__file__).resolve().parent.parent / "data" -SETTINGS_FILE = DATA_DIR / "settings.json" -LOCAL_CONTACTS_FILE = DATA_DIR / "contacts.json" - - -def _load_settings(): - if SETTINGS_FILE.exists(): - return json.loads(SETTINGS_FILE.read_text()) - return {} - - -def _save_settings(settings): - from core.atomic_io import atomic_write_json - atomic_write_json(str(SETTINGS_FILE), settings, indent=2) - - -def _get_carddav_config(): - import os - settings = _load_settings() - return { - "url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")), - "username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")), - "password": settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", "")), - } - - -def _carddav_configured(cfg: Optional[Dict] = None) -> bool: - cfg = cfg or _get_carddav_config() - return bool((cfg.get("url") or "").strip()) - - -def _normalize_contact(contact: Dict) -> Dict: - emails = [] - for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]): - e = str(e or "").strip() - if e and e not in emails: - emails.append(e) - phones = [] - for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]): - p = str(p or "").strip() - if p and p not in phones: - phones.append(p) - name = str(contact.get("name") or "").strip() - if not name and emails: - name = emails[0].split("@")[0] - return { - "uid": str(contact.get("uid") or uuid.uuid4()), - "name": name, - "emails": emails, - "phones": phones, - } - - -def _load_local_contacts() -> List[Dict]: - try: - if not LOCAL_CONTACTS_FILE.exists(): - return [] - data = json.loads(LOCAL_CONTACTS_FILE.read_text()) - rows = data.get("contacts", data) if isinstance(data, dict) else data - return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)] - except Exception as e: - logger.error(f"Failed to load local contacts: {e}") - return [] - - -def _save_local_contacts(contacts: List[Dict]) -> None: - from core.atomic_io import atomic_write_json - DATA_DIR.mkdir(parents=True, exist_ok=True) - atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2) - _contact_cache["contacts"] = [_normalize_contact(c) for c in contacts] - _contact_cache["fetched_at"] = datetime.utcnow() - - -# ── vCard parsing ── - -def _vunesc(value: str) -> str: - """Reverse _vesc() — turn escaped vCard text back into the raw value. - Order matters: handle \\n/\\, /\\; first, backslash-unescape last.""" - if not value: - return value - out = [] - i = 0 - while i < len(value): - ch = value[i] - if ch == "\\" and i + 1 < len(value): - nxt = value[i + 1] - if nxt in ("n", "N"): - out.append("\n") - elif nxt in (",", ";", "\\"): - out.append(nxt) - else: - out.append(nxt) - i += 2 - else: - out.append(ch) - i += 1 - return "".join(out) - - -def _parse_vcards(text: str) -> List[Dict]: - """Parse a stream of vCards into dicts with name, email, phone.""" - contacts = [] - for block in re.split(r"BEGIN:VCARD", text): - if not block.strip(): - continue - contact = {"name": "", "emails": [], "phones": [], "uid": ""} - for line in block.split("\n"): - line = line.strip() - if line.startswith("FN:") or line.startswith("FN;"): - contact["name"] = _vunesc(line.split(":", 1)[1]) if ":" in line else "" - elif line.startswith("EMAIL"): - # Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar - if ":" in line: - email_addr = _vunesc(line.split(":", 1)[1]) - if email_addr and email_addr not in contact["emails"]: - contact["emails"].append(email_addr) - elif line.startswith("TEL"): - if ":" in line: - phone = _vunesc(line.split(":", 1)[1]) - if phone and phone not in contact["phones"]: - contact["phones"].append(phone) - elif line.startswith("UID:"): - contact["uid"] = _vunesc(line[4:]) - if contact["name"] or contact["emails"]: - contacts.append(contact) - return contacts - - -def _vesc(value: str) -> str: - """Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma, - semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd' - or any value containing a newline produces a malformed vCard (broken - N/FN fields) or could inject arbitrary properties.""" - return ( - (value or "") - .replace("\\", "\\\\") - .replace("\n", "\\n") - .replace("\r", "") - .replace(",", "\\,") - .replace(";", "\\;") - ) - - -def _build_vcard(name: str, email: str, uid: Optional[str] = None, - emails: Optional[List[str]] = None, - phones: Optional[List[str]] = None) -> str: - """Build a vCard. Accepts either a single `email` (legacy callers) or - full `emails`/`phones` lists (edit path). The first email is marked - PREF=1. All values are RFC-6350-escaped.""" - if not uid: - uid = str(uuid.uuid4()) - # Normalize email lists — `email` arg is a convenience for single-email - # creation; `emails` (if given) is authoritative. - email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()] - phone_list = [p.strip() for p in (phones or []) if p and p.strip()] - # Try to split name into first/last - parts = name.strip().split() - if len(parts) >= 2: - first = parts[0] - last = " ".join(parts[1:]) - else: - first = name - last = "" - # N field is structured (5 components separated by ';') — escape each - # component individually so a comma in the name doesn't split it. - n_field = f"{_vesc(last)};{_vesc(first)};;;" - lines = [ - "BEGIN:VCARD", - "VERSION:4.0", - f"UID:{_vesc(uid)}", - f"FN:{_vesc(name)}", - f"N:{n_field}", - ] - for i, em in enumerate(email_list): - # First email is the preferred one. - lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}") - for ph in phone_list: - lines.append(f"TEL:{_vesc(ph)}") - lines.append("END:VCARD") - return "\r\n".join(lines) + "\r\n" - - -# ── In-memory cache ── - -_contact_cache = {"contacts": [], "fetched_at": None} - - -def _abs_url(href: str) -> str: - """Combine a multistatus (an absolute path like - /user/contacts/x.vcf) with the configured CardDAV server origin so we - get a fully-qualified URL to PUT/DELETE. If href is already absolute - (http...), return it as-is.""" - from urllib.parse import urlparse, urlunparse - if href.startswith("http://") or href.startswith("https://"): - return href - cfg = _get_carddav_config() - p = urlparse(cfg["url"]) - return urlunparse((p.scheme, p.netloc, href, "", "", "")) - - -# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request, -# alongside the resource href. Lets us map each contact's UID to the real -# server resource path (which is NOT always .vcf for contacts created -# by other clients). -_ADDRESSBOOK_QUERY = ( - '' - '' - '' - '' - '' -) - - -def _fetch_via_report(cfg, auth): - """Try a CardDAV REPORT addressbook-query — returns contacts WITH an - `href` field, or None if the server doesn't support it / errors.""" - from defusedxml import ElementTree as ET - try: - r = httpx.request( - "REPORT", cfg["url"], - content=_ADDRESSBOOK_QUERY.encode("utf-8"), - headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"}, - auth=auth, timeout=10, - ) - if r.status_code not in (207, 200): - return None - root = ET.fromstring(r.text) - ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"} - out = [] - for resp in root.findall("D:response", ns): - href_el = resp.find("D:href", ns) - data_el = resp.find(".//C:address-data", ns) - if href_el is None or data_el is None or not (data_el.text or "").strip(): - continue - parsed = _parse_vcards(data_el.text) - if not parsed: - continue - c = parsed[0] - c["href"] = href_el.text.strip() - out.append(c) - # If the REPORT parsed to ZERO contacts, don't trust it — some - # CardDAV servers treat an empty as "match nothing" and - # return a valid-but-empty 207. Return None so the caller falls - # back to the plain GET (which lists everything). A genuinely empty - # address book just costs one extra GET that also returns nothing. - if not out: - return None - return out - except Exception as e: - logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}") - return None - - -def _fetch_contacts(force=False): - """Fetch all contacts. Uses CardDAV when configured, otherwise local JSON.""" - if not force and _contact_cache["fetched_at"]: - age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds() - if age < 60: - return _contact_cache["contacts"] - - cfg = _get_carddav_config() - if not _carddav_configured(cfg): - contacts = _load_local_contacts() - _contact_cache["contacts"] = contacts - _contact_cache["fetched_at"] = datetime.utcnow() - return contacts - - try: - auth = None - if cfg["username"]: - auth = (cfg["username"], cfg["password"]) - # Preferred path: REPORT gives us hrefs for reliable edit/delete. - contacts = _fetch_via_report(cfg, auth) - if contacts is None: - # Fallback: plain GET, concatenated vCards, no hrefs. - r = httpx.get(cfg["url"], auth=auth, timeout=10) - if r.status_code != 200: - logger.warning(f"CardDAV returned {r.status_code}") - return _contact_cache["contacts"] - contacts = _parse_vcards(r.text) - _contact_cache["contacts"] = contacts - _contact_cache["fetched_at"] = datetime.utcnow() - return contacts - except Exception as e: - logger.error(f"Failed to fetch contacts: {e}") - return _contact_cache["contacts"] - - -def _resolve_resource_url(uid: str) -> str: - """Map a contact UID to its real CardDAV resource URL. Uses the href - captured during fetch when available (handles contacts whose filename - != UID); falls back to the .vcf guess for app-created contacts or - when no href is known.""" - def _lookup(): - for c in _contact_cache.get("contacts", []): - if c.get("uid") == uid and c.get("href"): - return _abs_url(c["href"]) - return None - found = _lookup() - if found: - return found - # Not in cache (or no href) — refresh once and retry before guessing. - try: - _fetch_contacts(force=True) - except Exception: - pass - return _lookup() or _vcard_url(uid) - - -def _create_contact(name: str, email: str) -> bool: - """Add a new contact via CardDAV or local contacts.""" - cfg = _get_carddav_config() - if not _carddav_configured(cfg): - contacts = _load_local_contacts() - email_l = (email or "").strip().lower() - for c in contacts: - if email_l and email_l in [e.lower() for e in c.get("emails", [])]: - return True - contacts.append(_normalize_contact({"name": name, "emails": [email]})) - _save_local_contacts(contacts) - return True - - contact_uid = str(uuid.uuid4()) - vcard = _build_vcard(name, email, contact_uid) - url = cfg["url"].rstrip("/") + "/" + contact_uid + ".vcf" - try: - auth = None - if cfg["username"]: - auth = (cfg["username"], cfg["password"]) - r = httpx.put( - url, - data=vcard.encode("utf-8"), - headers={"Content-Type": "text/vcard; charset=utf-8"}, - auth=auth, - timeout=10, - ) - if r.status_code in (200, 201, 204): - # Invalidate cache - _contact_cache["fetched_at"] = None - return True - logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}") - return False - except Exception as e: - logger.error(f"Failed to create contact: {e}") - return False - - -def _vcard_url(uid: str) -> str: - """The CardDAV resource URL for a given contact UID. The uid is URL- - encoded so a value containing '/', '..' or other path chars can't - escape the collection and target an arbitrary CardDAV resource.""" - from urllib.parse import quote - cfg = _get_carddav_config() - return cfg["url"].rstrip("/") + "/" + quote(uid, safe="") + ".vcf" - - -def _import_vcards(text: str) -> Dict: - """Import a (possibly multi-card) .vcf blob. Each card is PUT to the - CardDAV server PRESERVING its full original content (ADR/ORG/photo/ - etc.) — we don't rebuild it, just ensure it has VERSION + UID and - normalize line endings. Returns {imported, failed, total}.""" - from urllib.parse import quote - cfg = _get_carddav_config() - if not cfg.get("url"): - parsed = _parse_vcards(text) - contacts = _load_local_contacts() - existing = { - e.lower() - for c in contacts - for e in (c.get("emails") or []) - if e - } - imported = 0 - for c in parsed: - emails = [e for e in (c.get("emails") or []) if e] - if emails and any(e.lower() in existing for e in emails): - continue - contacts.append(_normalize_contact(c)) - for e in emails: - existing.add(e.lower()) - imported += 1 - if imported: - _save_local_contacts(contacts) - return {"imported": imported, "failed": 0, "total": len(parsed)} - auth = (cfg["username"], cfg["password"]) if cfg["username"] else None - # Split into individual cards. re.split drops the BEGIN line, so we - # re-add it. Normalize CRLF. - raw = (text or "").replace("\r\n", "\n").replace("\r", "\n") - blocks = [] - for chunk in raw.split("BEGIN:VCARD"): - chunk = chunk.strip() - if not chunk: - continue - # Trim anything after END:VCARD (defensive). - end = chunk.upper().find("END:VCARD") - body = chunk[: end + len("END:VCARD")] if end != -1 else chunk - blocks.append("BEGIN:VCARD\n" + body) - imported = 0 - failed = 0 - for block in blocks: - # Extract or assign a UID. - m = re.search(r"^UID:(.+)$", block, re.MULTILINE) - uid = (m.group(1).strip() if m else "") or str(uuid.uuid4()) - if not m: - # Inject a UID right after the VERSION line (or after BEGIN). - if re.search(r"^VERSION:", block, re.MULTILINE): - block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE) - else: - block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1) - elif not re.search(r"^VERSION:", block, re.MULTILINE): - block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1) - vcard = block.replace("\n", "\r\n") + "\r\n" - url = cfg["url"].rstrip("/") + "/" + quote(uid, safe="") + ".vcf" - try: - r = httpx.put( - url, data=vcard.encode("utf-8"), - headers={"Content-Type": "text/vcard; charset=utf-8"}, - auth=auth, timeout=15, - ) - if r.status_code in (200, 201, 204): - imported += 1 - else: - failed += 1 - logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}") - except Exception as e: - failed += 1 - logger.error(f"Import PUT {uid} failed: {e}") - if imported: - _contact_cache["fetched_at"] = None - return {"imported": imported, "failed": failed, "total": len(blocks)} - - -def _import_csv_contacts(text: str) -> Dict: - """Import contacts from CSV. Supports common headers: - name/full_name/display_name, email/email_address/e-mail, phone/tel. - Falls back to first columns as name,email,phone when no headers exist.""" - raw = (text or "").strip() - if not raw: - return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"} - - try: - sample = raw[:2048] - dialect = csv.Sniffer().sniff(sample) - except Exception: - dialect = csv.excel - - stream = io.StringIO(raw) - try: - has_header = csv.Sniffer().has_header(raw[:2048]) - except Exception: - has_header = True - - rows = [] - if has_header: - reader = csv.DictReader(stream, dialect=dialect) - for row in reader: - lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()} - name = ( - lowered.get("name") or lowered.get("full name") or lowered.get("full_name") - or lowered.get("display name") or lowered.get("display_name") - or lowered.get("fn") or "" - ) - email = ( - lowered.get("email") or lowered.get("email address") - or lowered.get("email_address") or lowered.get("e-mail") - or lowered.get("mail") or "" - ) - phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or "" - rows.append((name, email, phone)) - else: - stream.seek(0) - reader = csv.reader(stream, dialect=dialect) - for row in reader: - cols = [(c or "").strip() for c in row] - if not any(cols): - continue - rows.append(( - cols[0] if len(cols) > 0 else "", - cols[1] if len(cols) > 1 else "", - cols[2] if len(cols) > 2 else "", - )) - - imported = 0 - failed = 0 - total = 0 - existing_emails = { - e.lower() - for c in _fetch_contacts() - for e in (c.get("emails") or []) - if e - } - for name, email, phone in rows: - email = (email or "").strip() - name = (name or "").strip() or (email.split("@")[0] if email else "") - if not email: - continue - total += 1 - if email.lower() in existing_emails: - continue - ok = _create_contact(name, email) - if ok: - imported += 1 - existing_emails.add(email.lower()) - # If the CSV had a phone number, rewrite the just-created row - # through the richer update path so phone lands in CardDAV too. - if phone: - try: - contacts = _fetch_contacts(force=True) - created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None) - if created and created.get("uid"): - _update_contact(created["uid"], name, [email], [phone]) - except Exception: - pass - else: - failed += 1 - - if imported: - _contact_cache["fetched_at"] = None - return {"imported": imported, "failed": failed, "total": total} - - -def _contacts_to_vcf(contacts: List[Dict]) -> str: - return "".join( - _build_vcard( - c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"), - "", - uid=c.get("uid") or str(uuid.uuid4()), - emails=c.get("emails") or [], - phones=c.get("phones") or [], - ) - for c in contacts - ) - - -def _contacts_to_csv(contacts: List[Dict]) -> str: - out = io.StringIO() - writer = csv.writer(out) - writer.writerow(["name", "email", "phone"]) - for c in contacts: - emails = c.get("emails") or [""] - phones = c.get("phones") or [""] - max_len = max(len(emails), len(phones), 1) - for i in range(max_len): - writer.writerow([ - c.get("name") or "", - emails[i] if i < len(emails) else "", - phones[i] if i < len(phones) else "", - ]) - return out.getvalue() - - -def _update_contact(uid: str, name: str, emails: List[str], phones: List[str]) -> bool: - """Rewrite an existing contact via CardDAV or local contacts.""" - cfg = _get_carddav_config() - if not _carddav_configured(cfg): - contacts = _load_local_contacts() - found = False - out = [] - for c in contacts: - if c.get("uid") == uid: - out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones})) - found = True - else: - out.append(c) - if not found: - out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones})) - _save_local_contacts(out) - return True - - vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones) - # Use the real resource href (handles externally-created contacts whose - # filename != UID); falls back to the .vcf guess. - url = _resolve_resource_url(uid) - try: - auth = (cfg["username"], cfg["password"]) if cfg["username"] else None - r = httpx.put( - url, - data=vcard.encode("utf-8"), - headers={"Content-Type": "text/vcard; charset=utf-8"}, - auth=auth, - timeout=10, - ) - if r.status_code in (200, 201, 204): - _contact_cache["fetched_at"] = None - return True - logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}") - return False - except Exception as e: - logger.error(f"Failed to update contact: {e}") - return False - - -def _delete_contact(uid: str) -> bool: - """Delete a contact via CardDAV or local contacts.""" - cfg = _get_carddav_config() - if not _carddav_configured(cfg): - contacts = _load_local_contacts() - remaining = [c for c in contacts if c.get("uid") != uid] - _save_local_contacts(remaining) - return True - - url = _resolve_resource_url(uid) - try: - auth = (cfg["username"], cfg["password"]) if cfg["username"] else None - r = httpx.delete(url, auth=auth, timeout=10) - if r.status_code in (200, 204): - _contact_cache["fetched_at"] = None - return True - if r.status_code == 404: - # Resource not found at the resolved URL. With href resolution - # this should be rare (genuinely already deleted). Invalidate - # the cache and report success so the UI doesn't keep a ghost. - logger.info(f"CardDAV DELETE 404 for {uid} — treating as already gone") - _contact_cache["fetched_at"] = None - return True - logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}") - return False - except Exception as e: - logger.error(f"Failed to delete contact: {e}") - return False - - -# ── Routes ── - -def setup_contacts_routes(): - router = APIRouter(prefix="/api/contacts", tags=["contacts"]) - - @router.get("/list") - async def list_contacts(_admin: str = Depends(require_admin)): - """List all contacts.""" - contacts = _fetch_contacts() - return {"contacts": contacts, "count": len(contacts)} - - @router.get("/search") - async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)): - """Search contacts by name or email. Returns up to 10 matches.""" - contacts = _fetch_contacts() - if not q: - return {"results": []} - q_lower = q.lower() - results = [] - for c in contacts: - if q_lower in c["name"].lower(): - results.append(c) - continue - for em in c["emails"]: - if q_lower in em.lower(): - results.append(c) - break - return {"results": results[:10]} - - @router.post("/add") - async def add_contact(data: dict, _admin: str = Depends(require_admin)): - """Add a new contact.""" - name = data.get("name", "").strip() - email = data.get("email", "").strip() - if not email: - return {"success": False, "error": "Email required"} - # Check if already exists - contacts = _fetch_contacts() - for c in contacts: - if email.lower() in [e.lower() for e in c["emails"]]: - return {"success": True, "message": "Already exists", "contact": c} - if not name: - name = email.split("@")[0] - ok = _create_contact(name, email) - return {"success": ok} - - @router.post("/import") - async def import_vcf(data: dict, _admin: str = Depends(require_admin)): - """Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}.""" - text = data.get("vcf") or data.get("text") or "" - csv_text = data.get("csv") or "" - if text.strip(): - if "BEGIN:VCARD" not in text.upper(): - return {"success": False, "error": "No vCard data found"} - result = _import_vcards(text) - elif csv_text.strip(): - result = _import_csv_contacts(csv_text) - else: - return {"success": False, "error": "No contact data found"} - result["success"] = result.get("imported", 0) > 0 - return result - - @router.get("/export") - async def export_contacts( - format: str = Query("vcf", pattern="^(vcf|csv)$"), - _admin: str = Depends(require_admin), - ): - """Export all contacts as vCard or CSV.""" - contacts = _fetch_contacts(force=True) - if format == "csv": - content = _contacts_to_csv(contacts) - media_type = "text/csv; charset=utf-8" - filename = "odysseus-contacts.csv" - else: - content = _contacts_to_vcf(contacts) - media_type = "text/vcard; charset=utf-8" - filename = "odysseus-contacts.vcf" - return Response( - content=content, - media_type=media_type, - headers={"Content-Disposition": f'attachment; filename="{filename}"'}, - ) - - @router.get("/config") - async def get_config(_admin: str = Depends(require_admin)): - cfg = _get_carddav_config() - # Mask password - if cfg["password"]: - cfg["password"] = "***" - return cfg - - @router.put("/config") - async def update_config(data: dict, _admin: str = Depends(require_admin)): - settings = _load_settings() - for key in ("carddav_url", "carddav_username", "carddav_password"): - if key in data: - settings[key] = data[key] - _save_settings(settings) - # Force re-fetch - _contact_cache["fetched_at"] = None - return {"success": True} - - @router.delete("/clear") - async def clear_contacts(_admin: str = Depends(require_admin)): - """Clear all local contacts. If CardDAV is configured, only clears the local fallback cache.""" - _save_local_contacts([]) - return {"success": True} - - # NOTE: the /{uid} routes are declared LAST so the literal paths above - # (/list, /search, /add, /config) win — otherwise PUT /config would - # match PUT /{uid} with uid="config". - @router.put("/{uid}") - async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)): - """Edit an existing contact — name / emails / phones.""" - name = (data.get("name") or "").strip() - emails = data.get("emails") - phones = data.get("phones") - if emails is None and data.get("email"): - emails = [data["email"]] - emails = [e.strip() for e in (emails or []) if e and e.strip()] - phones = [p.strip() for p in (phones or []) if p and p.strip()] - if not name and not emails: - return {"success": False, "error": "Name or email required"} - if not name and emails: - name = emails[0].split("@")[0] - ok = _update_contact(uid, name, emails, phones) - return {"success": ok} +import sys as _sys - @router.delete("/{uid}") - async def delete_contact(uid: str, _admin: str = Depends(require_admin)): - """Delete a contact by UID.""" - if not uid: - return {"success": False, "error": "UID required"} - ok = _delete_contact(uid) - return {"success": ok} +from routes.contacts import contacts_routes as _canonical # noqa: F401 - return router +_sys.modules[__name__] = _canonical diff --git a/routes/cookbook_helpers.py b/routes/cookbook_helpers.py index 97ef2ca49..c784f2c25 100644 --- a/routes/cookbook_helpers.py +++ b/routes/cookbook_helpers.py @@ -1,36 +1,65 @@ """cookbook_helpers.py — validators + small helpers shared by the cookbook routes. Extracted from cookbook_routes.py; the routes module imports the symbols it needs.""" +import json import logging +import ntpath import os +import posixpath import re import shlex +from pathlib import Path from fastapi import HTTPException from pydantic import BaseModel +from routes._validators import validate_remote_host, validate_ssh_port +from core.platform_compat import _ssh_exec_argv + logger = logging.getLogger(__name__) # HuggingFace repo IDs are /, both alphanumerics plus ._- # Rejecting anything else up front closes off shell-interpolation vectors. _REPO_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*/[A-Za-z0-9][A-Za-z0-9._-]*$") +# Cached models scanned from a custom/local model dir are keyed by their leaf +# folder name (no slash), e.g. `DeepSeek-R1-UD-IQ4_XS`. The serve command uses +# the real on-disk path separately; this identifier is only for UI/task +# bookkeeping, so serving should accept the same safe glyph set as repo IDs. +_LOCAL_MODEL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +# Ollama model names include tags, e.g. `qwen2.5:0.5b` or `llama3.2:latest`. +# Some registries also use a namespace path. Keep this shell-safe: no spaces, +# quotes, `$`, `;`, `&`, pipes, or redirects. +_OLLAMA_MODEL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,200}$") # Include pattern is a glob: allow typical safe glyphs only. _INCLUDE_RE = re.compile(r"^[A-Za-z0-9._\-*?/\[\]]+$") -# Remote host: user@host (optionally with :port-free hostname parts). -_REMOTE_HOST_RE = re.compile(r"^[A-Za-z0-9._-]+@[A-Za-z0-9._-]+$") # HF tokens and API tokens are url-safe base64-like. _TOKEN_RE = re.compile(r"^[A-Za-z0-9._~+/=-]+$") # Session IDs we mint look like "cookbook-deadbeef" or "serve-deadbeef". # Anything beyond plain alphanumerics + dash + underscore could break out # of the shell/PowerShell contexts the value lands in. _SESSION_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$") -_SSH_PORT_RE = re.compile(r"^\d{1,5}$") _GPU_LIST_RE = re.compile(r"^\d+(?:,\d+)*$") # A download target directory. Absolute or ~-relative path; safe path glyphs -# only (no quotes, shell metacharacters, or spaces) since it lands in a shell -# command. A leading ~ is expanded to $HOME at command-build time. -_LOCAL_DIR_RE = re.compile(r"^~?/[A-Za-z0-9._/-]*$|^~$") +# only (no quotes or shell metacharacters). Spaces are allowed because command +# builders pass the value through quoted shell/Python contexts. The character +# class uses ``\w`` — Unicode word characters under Python 3's default str +# matching — so non-ASCII folder names pass validation too: Cyrillic, accented +# Latin, CJK, e.g. ``/Volumes/Модели`` or ``D:\AI Models\Модели``. This stays +# shell-safe: none of ``; & | ` $ '' "" () {}`` newlines etc. are in ``[\w. -]``, +# so injection vectors remain rejected. A leading ~ is expanded to $HOME at +# command-build time. (Drive letters stay ASCII: ``[A-Za-z]:``.) +_LOCAL_DIR_RE = re.compile(r"^~?(?:/[\w. -]*)+$|^~$") +_WINDOWS_LOCAL_DIR_RE = re.compile(r"^[A-Za-z]:[\\/](?:[\w. -]+(?:[\\/][\w. -]+)*[\\/]?)?$") +_WINDOWS_DRIVE_PATH_RE = re.compile(r"^[A-Za-z]:[\\/]") + + +def _git_bash_path(path: str) -> str: + m = re.match(r"^([A-Za-z]):[\\/](.*)$", path) + if not m: + return path + drive, rest = m.groups() + return f"/{drive.lower()}/{rest.replace(chr(92), '/')}" def _validate_repo_id(v: str | None) -> str: @@ -39,6 +68,14 @@ def _validate_repo_id(v: str | None) -> str: return v +def _validate_serve_model_id(v: str | None) -> str: + if not v: + raise HTTPException(400, "repo_id is required") + if _REPO_ID_RE.match(v) or _LOCAL_MODEL_ID_RE.match(v) or _OLLAMA_MODEL_ID_RE.match(v): + return v + raise HTTPException(400, "Invalid repo_id — must be /, an Ollama name:tag, or a cached local model id") + + def _validate_include(v: str | None) -> str | None: if v is None or v == "": return None @@ -47,14 +84,6 @@ def _validate_include(v: str | None) -> str | None: return v -def _validate_remote_host(v: str | None) -> str | None: - if v is None or v == "": - return None - if not _REMOTE_HOST_RE.match(v): - raise HTTPException(400, "Invalid remote_host — must be user@host, no SSH option syntax") - return v - - def _validate_token(v: str | None) -> str | None: if v is None or v == "": return None @@ -63,26 +92,43 @@ def _validate_token(v: str | None) -> str | None: return v +def load_stored_hf_token(*, state_path: Path | str | None = None) -> str: + """Return the decrypted HF token from cookbook_state.json, else env fallback.""" + path = Path(state_path) if state_path else Path(os.environ.get("DATA_DIR", "data")) / "cookbook_state.json" + token = "" + if path.exists(): + try: + state = json.loads(path.read_text(encoding="utf-8")) + env = state.get("env") if isinstance(state, dict) else {} + if isinstance(env, dict) and env.get("hfToken"): + from src.secret_storage import decrypt + token = decrypt(env.get("hfToken") or "") + except Exception: + token = "" + if not token: + token = (os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "").strip() + return token + + def _validate_local_dir(v: str | None) -> str | None: if v is None or v == "": return None + if len(v) >= 2 and v[0] == v[-1] and v[0] in {"'", '"'}: + v = v[1:-1] v = v.rstrip("/") or "/" - if not _LOCAL_DIR_RE.match(v): - raise HTTPException(400, "Invalid local_dir — must be an absolute or ~ path with no spaces or shell metacharacters") + if not (_LOCAL_DIR_RE.match(v) or _WINDOWS_LOCAL_DIR_RE.match(v)): + raise HTTPException(400, "Invalid local_dir — must be an absolute or ~ path with no shell metacharacters") + # Reject path segments that start with '-' (option injection). '-' is in the + # allowlist, so a dir like ``/models/-rf`` or ``D:\models\-rf`` could be read + # as a CLI flag by hf/etc. — and quoting does NOT stop a value from being + # parsed as an option. This is the one residual that command-build-time + # quoting can't cover, so the guard lives here, keeping the safety wholly + # inside the validator rather than relying on consumers. + if any(seg.startswith("-") for seg in re.split(r"[\\/]", v) if seg): + raise HTTPException(400, "Invalid local_dir — path segments cannot start with '-'") return v -def _validate_ssh_port(v: str | None) -> str | None: - if v is None or v == "": - return None - if not _SSH_PORT_RE.fullmatch(str(v)): - raise HTTPException(400, "Invalid ssh_port") - port = int(v) - if port < 1 or port > 65535: - raise HTTPException(400, "Invalid ssh_port") - return str(port) - - def _validate_gpus(v: str | None) -> str | None: if v is None or v == "": return None @@ -94,7 +140,7 @@ def _validate_gpus(v: str | None) -> str | None: def _shell_path(p: str) -> str: """Render a validated path for a double-quoted shell context, expanding a leading ~ to $HOME (single quotes wouldn't expand it). Safe because - _validate_local_dir already restricts the charset.""" + _validate_local_dir already rejects quotes and shell metacharacters.""" if p == "~": return '"$HOME"' if p.startswith("~/"): @@ -102,6 +148,430 @@ def _shell_path(p: str) -> str: return '"' + p + '"' +def _local_tooling_path_export(executable: str) -> str: + """Bash line prepending the running interpreter's bin dir to PATH. + + When Odysseus runs from a virtualenv, that bin dir holds the tools the + cookbook runners shell out to (`hf`, `python`). tmux runners start from a + fresh login shell with the venv NOT activated, so without this they can't + find `hf` and downloads fail with "hf: command not found" — notably on + macOS, where the `pip --user` self-heal also misses (`pip` isn't a command, + only `pip3`/`python3 -m pip`). Local runs only; meaningless over SSH. + """ + # This builds a bash snippet, so an explicit POSIX absolute path should keep + # POSIX semantics even when the app/tests run on Windows. Otherwise + # os.path.abspath("/opt/...") would incorrectly turn it into "D:\\opt\\...". + if executable.startswith("/"): + bin_dir = posixpath.dirname(executable) + elif _WINDOWS_DRIVE_PATH_RE.match(executable): + bin_dir = ntpath.dirname(executable) + else: + bin_dir = os.path.dirname(os.path.abspath(executable)) + bin_dir = _git_bash_path(bin_dir) + # Escape for a double-quoted context: $PATH must still expand, but spaces + # and shell metacharacters in the path must be preserved literally. + esc = ( + bin_dir.replace("\\", "\\\\") + .replace('"', '\\"') + .replace("$", "\\$") + .replace("`", "\\`") + ) + return f'export PATH="{esc}:$PATH"' + + +def _pip_install_no_cache(cmd: str) -> str: + """Add ``--no-cache-dir`` to a pip install command. + + Cookbook dependency installs (vLLM, llama-cpp-python, …) build large wheels; + pip's default cache lives under ``$HOME/.cache/pip`` and these builds can fill + a small home filesystem with ``[Errno 28] No space left on device`` mid-build + (issue #1219), leaving the dependency "installed" but unusable (#1459). + Disabling the cache for these one-off installs keeps them off the home disk + (the maintainer's suggested ``PIP_CACHE_DIR=`` workaround, made the default). + Idempotent; leaves non-pip-install commands untouched.""" + if not cmd or "pip install" not in cmd or "--no-cache-dir" in cmd: + return cmd + return cmd.replace("pip install", "pip install --no-cache-dir", 1) + + +def _pip_install_attempt(pip_cmd: str) -> str: + """Wrap a single pip install command so its exit status survives the + fallback chain and its stderr is visible in the tmux log on failure. + + Without this wrapper, `pip … 2>&1 | tail -5` returns ``tail``'s exit + code (0), masking pip's real failure and preventing the next fallback + from running. The generated snippet captures all output to a temp + file, prints the last 5 lines on failure (so the Cookbook log panel + shows useful diagnostics), cleans up, and exits with pip's original + status. + """ + return ( + "bash -c '" + f'_out=$(mktemp) && {pip_cmd} >"$_out" 2>&1; _rc=$?; ' + 'tail -5 "$_out"; rm -f "$_out"; exit $_rc' + "'" + ) + + +def _pip_command(python_cmd: str) -> str: + """Return a pip command for either a pip executable or a Python executable.""" + cmd = python_cmd.strip() + if " -m pip" in cmd or cmd in {"pip", "pip3"}: + return python_cmd + if cmd in {"python", "python3", "python.exe"} or cmd.endswith(("/python", "/python3", "\\python.exe")): + return f"{python_cmd} -m pip" + return python_cmd + + +def _pip_break_system_packages_check(pip_cmd: str) -> str: + return f"{pip_cmd} install --help 2>/dev/null | grep -q -- --break-system-packages" + + +def _pip_install_fallback_chain(package: str, *, python_cmd: str = "python3 -m pip", upgrade: bool = False) -> str: + """Build a bash pip install fallback chain that surfaces errors. + + Try the active interpreter/environment first. ``--user`` is invalid + inside many venvs, so only attempt the ``--user`` fallback when NOT + inside a venv. + + Each attempt is wrapped via :func:`_pip_install_attempt` so pip's real + exit code is preserved (no ``| tail`` masking) and the last 5 lines of + pip output appear in the Cookbook log on failure. + """ + from core.platform_compat import IS_WINDOWS + upgrade_flag = " -U" if upgrade else "" + # Shell-quote the package spec: an extras spec like ``llama-cpp-python[server]`` + # contains brackets that bash would treat as a glob, so it must be quoted + # before being embedded in the install command. Plain names (e.g. + # ``huggingface_hub``) are returned unchanged by ``shlex.quote``. + pkg = shlex.quote(package) + # llama-cpp-python source builds are brittle on older distro pip/packaging + # stacks (common on WSL images). Prefer the prebuilt wheel index whenever + # this package is requested so dependency-install tasks are reliable. + if "llama-cpp-python" in package: + pkg += " --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu" + + pip_cmd = _pip_command(python_cmd) + base = _pip_install_attempt(f"{pip_cmd} install -q{upgrade_flag} {pkg}") + user = _pip_install_attempt(f"{pip_cmd} install --user -q{upgrade_flag} {pkg}") + user_break_system = _pip_install_attempt(f"{pip_cmd} install --user --break-system-packages -q{upgrade_flag} {pkg}") + user_fallback = f"( {user} || {{ {_pip_break_system_packages_check(pip_cmd)} && {user_break_system}; }} )" + # Derive the python executable for the venv detection check. + # Must use the same interpreter that pip belongs to; hardcoding + # python3 breaks when pip lives in a venv that only has "python". + if " -m pip" in pip_cmd: + python_exe = pip_cmd.replace(" -m pip", "") + elif pip_cmd.strip() == "pip": + python_exe = "python" + elif pip_cmd.strip() == "pip3": + python_exe = "python3" + else: + python_exe = "python3" + venv_check = f'{python_exe} -c "import sys; sys.exit(0 if sys.prefix != sys.base_prefix else 1)"' + # Negated: `! venv_check` succeeds (exit 0) when NOT in a venv -> `&&` tries + # --user. When IN a venv `! venv_check` fails -> `&&` skips --user and the + # group exits non-zero, propagating the base-install failure instead of + # masking it as success (the `|| { venv_check || … }` shape from #903 + # swallowed the exit code because venv_check's exit-0 became the group's + # result). `--break-system-packages` is only attempted when the active pip + # supports it; older pip versions abort with "no such option" otherwise. + return f"{base} || {{ ! {venv_check} && {user_fallback}; }}" + + +def _venv_safe_local_pip_install_cmd(cmd: str, *, local: bool, in_venv: bool) -> str: + """Drop pip user-install flags that are invalid for local venv installs. + + Cookbook dependency installs run through the model-serve task path so users + can watch progress in the same log UI. For local POSIX runs, that task + prepends Odysseus' own interpreter directory to PATH. If Odysseus itself is + running from a venv, `python3` resolves to the venv Python and pip rejects + `--user` with "User site-packages are not visible in this virtualenv". + + Keep remote and non-venv installs unchanged: remotes may intentionally use + system Python, and Docker/non-venv installs still need user-site fallback. + """ + if not local or not in_venv: + return cmd + if "pip install" not in (cmd or ""): + return cmd + try: + parts = shlex.split(cmd) + except ValueError: + return cmd + stripped = [ + part + for part in parts + if part not in {"--user", "--break-system-packages"} + ] + return shlex.join(stripped) + + +def _pip_install_command_without_break_system_packages(cmd: str) -> str: + try: + parts = shlex.split(cmd) + except ValueError: + return cmd + stripped = [part for part in parts if part != "--break-system-packages"] + return shlex.join(stripped) + + +def _pip_install_help_check_from_cmd(cmd: str) -> str | None: + try: + parts = shlex.split(cmd) + except ValueError: + return None + try: + install_index = parts.index("install") + except ValueError: + return None + if install_index <= 0: + return None + pip_prefix = parts[:install_index] + return f"{shlex.join(pip_prefix + ['install', '--help'])} 2>/dev/null | grep -q -- --break-system-packages" + + +def _append_pip_install_runner_lines(runner_lines: list[str], cmd: str) -> None: + """Append a pip install command, guarding --break-system-packages support. + + The Dependencies UI may submit ``python3 -m pip install --user + --break-system-packages ...`` for non-venv installs. That flag is useful on + PEP-668-locked distros, but older pip (including Ubuntu 22.04's apt pip in + the NVIDIA CUDA base image) aborts with "no such option". Branch at runner + time so stale browser JS and remote targets are handled by the server too. + """ + if "--break-system-packages" not in (cmd or ""): + runner_lines.append(cmd) + return + help_check = _pip_install_help_check_from_cmd(cmd) + without_break = _pip_install_command_without_break_system_packages(cmd) + if not help_check or without_break == cmd: + runner_lines.append(cmd) + return + runner_lines.append(f"if {help_check}; then") + runner_lines.append(f" {cmd}") + runner_lines.append("else") + runner_lines.append(' echo "[odysseus] pip does not support --break-system-packages; installing without it."') + runner_lines.append(f" {without_break}") + runner_lines.append("fi") + + +def _user_shell_path_bootstrap() -> list[str]: + return [ + 'ODYSSEUS_USER_SHELL="${SHELL:-}"', + 'if [ -n "$ODYSSEUS_USER_SHELL" ] && [ -x "$ODYSSEUS_USER_SHELL" ]; then', + ' ODYSSEUS_USER_PATH="$("$ODYSSEUS_USER_SHELL" -ic \'printf "__ODYSSEUS_PATH__%s\\n" "$PATH"\' 2>/dev/null | sed -n \'s/^__ODYSSEUS_PATH__//p\' | tail -n 1 || true)"', + ' if [ -n "$ODYSSEUS_USER_PATH" ]; then export PATH="$ODYSSEUS_USER_PATH:$PATH"; fi', + 'fi', + # Windows can expose python3 as a Microsoft Store App Execution Alias + # under WindowsApps. Git Bash sees that stub as present, but it exits + # before running Python. A Windows venv usually has python.exe, not + # python3.exe, so treat a missing or WindowsApps python3 as absent. + '_odys_py3="$(command -v python3 2>/dev/null || true)"', + 'case "$_odys_py3" in ""|*[Ww]indows[Aa]pps*) python3() { python "$@"; } ;; esac', + 'command -v python >/dev/null 2>&1 || python() { python3 "$@"; }', + ] + + +def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache: str | None = None) -> str: + """Build the standalone Python scanner used by /api/model/cached. + Allows for an additional HuggingFace cache path to be scanned (i.e. Windows HF cache for local WSL envs.) + """ + lines = [ + "import json, os, re, shutil, subprocess, urllib.request", + "models = []", + "seen = set()", + "BLOCKED_ROOTS = ('/sys', '/proc', '/dev', '/run', '/var/run')", + "def safe_path(p):", + " try:", + " rp = os.path.realpath(os.path.expanduser(p))", + " return not any(rp == b or rp.startswith(b + os.sep) for b in BLOCKED_ROOTS)", + " except Exception:", + " return False", + "def safe_walk(top):", + " if not safe_path(top): return", + " for root, dirs, fns in os.walk(top, followlinks=False):", + " dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(root, d)) and safe_path(os.path.join(root, d))]", + " yield root, dirs, fns", + "def gguf_role(name):", + " n = name.lower()", + " if n.startswith('mmproj') or 'mmproj' in n: return 'projector'", + " return 'model'", + "def gguf_quant(name):", + " m = re.search(r'(?i)(UD-)?(IQ[0-9]_[A-Z0-9_]+|Q[0-9](?:_[A-Z0-9]+)+|BF16|F16|FP16|F32|Q8_0)', name)", + " return m.group(0).upper() if m else ''", + "def collect_ggufs(base):", + " files = []", + " split_groups = {}", + " if not os.path.isdir(base) or not safe_path(base): return files", + " for root, dirs, fns in safe_walk(base):", + " for fn in sorted(fns):", + " if not fn.lower().endswith('.gguf'): continue", + " if fn.startswith('._'): continue # macOS AppleDouble sidecar, not a real GGUF", + " fp = os.path.join(root, fn)", + " try: size = os.path.getsize(fp)", + " except Exception: size = 0", + " try: rel = os.path.relpath(fp, base).replace(os.sep, '/')", + " except Exception: rel = fn", + " sm = re.match(r'(?i)^(.+)-(\\d+)-of-(\\d+)\\.gguf$', fn)", + " if sm:", + " prefix, part_s, total_s = sm.group(1), sm.group(2), sm.group(3)", + " key = (root, prefix, total_s)", + " g = split_groups.setdefault(key, {'name':fn,'rel_path':rel,'size_bytes':0,'role':gguf_role(fn),'quant':gguf_quant(fn),'parts':int(total_s),'split':True})", + " g['size_bytes'] += size", + " if int(part_s) == 1:", + " g.update({'name':fn,'rel_path':rel,'role':gguf_role(fn),'quant':gguf_quant(fn)})", + " continue", + " files.append({'name':fn,'rel_path':rel,'size_bytes':size,'role':gguf_role(fn),'quant':gguf_quant(fn)})", + " files.extend(split_groups.values())", + " files.sort(key=lambda f: (f.get('role') != 'model', f.get('rel_path', '')))", + " return files", + "def scan_hf(cache):", + " if not os.path.isdir(cache): return", + " for d in sorted(os.listdir(cache)):", + " if not d.startswith('models--'): continue", + " rid = d.replace('models--','').replace('--','/')", + " if rid in seen: continue", + " seen.add(rid)", + " blobs = os.path.join(cache, d, 'blobs')", + " sz, nf, ic = 0, 0, False", + " if os.path.isdir(blobs):", + " for f in os.scandir(blobs):", + " if f.is_file(): nf += 1; sz += f.stat().st_size", + " if f.name.endswith('.incomplete'): ic = True", + " snap = os.path.join(cache, d, 'snapshots')", + " def snapshot_size():", + " total, count, incomplete = 0, 0, False", + " seen_real = set()", + " for sd in os.listdir(snap):", + " sf = os.path.join(snap, sd)", + " if not os.path.isdir(sf): continue", + " for root, dirs, fns in safe_walk(sf):", + " for fn in fns:", + " fp = os.path.join(root, fn)", + " if fn.endswith('.incomplete'): incomplete = True", + " try:", + " real = os.path.realpath(fp)", + " if real in seen_real: continue", + " seen_real.add(real)", + " total += os.path.getsize(real)", + " count += 1", + " except Exception:", + " pass", + " return total, count, incomplete", + " # Some HF caches (macOS/MLX/Xet-style) keep blobs elsewhere or expose", + " # snapshot symlinks only. Size snapshots too when blob accounting is empty.", + " if sz == 0 and os.path.isdir(snap):", + " sz2, nf2, ic2 = snapshot_size()", + " sz, nf, ic = sz2, nf2, ic or ic2", + " is_diffusion = False; gguf_files = []", + " if os.path.isdir(snap):", + " for sd in os.listdir(snap):", + " sf = os.path.join(snap, sd)", + " if not os.path.isdir(sf): continue", + " if os.path.exists(os.path.join(sf, 'model_index.json')): is_diffusion = True", + " for f in collect_ggufs(sf): f['rel_path'] = sd + '/' + f['rel_path']; gguf_files.append(f)", + " models.append({'repo_id':rid,'size_bytes':sz,'nb_files':nf,'has_incomplete':ic,'path':cache,'is_diffusion':is_diffusion,'is_gguf':bool(gguf_files),'gguf_files':gguf_files})", + "def hf_cache_paths():", + " candidates = []", + " def add(p):", + " if not p: return", + " p = os.path.expanduser(p)", + " if p not in candidates: candidates.append(p)", + " add(os.environ.get('HUGGINGFACE_HUB_CACHE'))", + " hf_home = os.environ.get('HF_HOME')", + " if hf_home: add(os.path.join(hf_home, 'hub'))", + " add('~/.cache/huggingface/hub')", + " # Docker images mount ./data/huggingface at /app/.cache/huggingface.", + " # When HOME is /root, expanduser() misses that persisted cache.", + " add('/app/.cache/huggingface/hub')", + f" add({add_hf_cache!r})" if add_hf_cache else "", + " return candidates", + "def normalize_model_dir(p):", + " p = os.path.expanduser((p or '').strip())", + " if not p: return p", + " if os.path.isdir(p) or os.path.isabs(p): return p", + " # Users often paste Linux absolute paths without the leading slash.", + " # Treat home//... as /home//... so remote scans work.", + " if p.startswith(('home/', 'mnt/', 'media/', 'data/', 'opt/', 'srv/', 'var/')):", + " prefixed = '/' + p", + " if os.path.isdir(prefixed): return prefixed", + " return p", + "def scan_dir(p):", + " p = normalize_model_dir(p)", + " if not os.path.isdir(p) or not safe_path(p): return", + " for d in sorted(os.listdir(p)):", + " if d.startswith('.'): continue", + " if d.startswith('models--'): continue", + " fp = os.path.join(p, d)", + " if not os.path.isdir(fp) or os.path.islink(fp) or not safe_path(fp): continue", + " if d in seen: continue", + " is_model = False; gguf_files = []", + " for root, dirs, fns in safe_walk(fp):", + " for fn in fns:", + " if fn.lower().endswith('.gguf'): is_model = True", + " elif fn == 'config.json' or fn.endswith('.safetensors') or fn.endswith('.bin'): is_model = True", + " if is_model: break", + " if not is_model: continue", + " gguf_files = collect_ggufs(fp)", + " seen.add(d)", + " sz, nf = 0, 0", + " for dp, _, fns in safe_walk(fp):", + " for fn in fns:", + " try: nf += 1; sz += os.path.getsize(os.path.join(dp, fn))", + " except Exception: pass", + " is_diff = os.path.exists(os.path.join(fp, 'model_index.json'))", + " models.append({'repo_id':d,'size_bytes':sz,'nb_files':nf,'has_incomplete':False,'path':p,'is_local_dir':True,'is_diffusion':is_diff,'is_gguf':bool(gguf_files),'gguf_files':gguf_files})", + "def parse_size(num, unit):", + " try: n = float(num)", + " except Exception: return 0", + " u = (unit or '').upper()", + " if u.startswith('TB'): return int(n * 1024 ** 4)", + " if u.startswith('GB'): return int(n * 1024 ** 3)", + " if u.startswith('MB'): return int(n * 1024 ** 2)", + " if u.startswith('KB'): return int(n * 1024)", + " return int(n)", + "def scan_ollama():", + " if any(m.get('is_ollama') for m in models): return", + " if os.name == 'nt' and not os.environ.get('ODYSSEUS_ALLOW_OLLAMA_CLI_SCAN'): return", + " if not shutil.which('ollama'): return", + " try:", + " p = subprocess.run(['ollama', 'list'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=6)", + " except Exception:", + " return", + " if p.returncode != 0: return", + " for line in (p.stdout or '').splitlines()[1:]:", + " parts = line.split()", + " if len(parts) < 4: continue", + " name = parts[0]", + " if not name or name in seen: continue", + " size_bytes = parse_size(parts[2], parts[3])", + " seen.add(name)", + " models.append({'repo_id':name,'size_bytes':size_bytes,'nb_files':1,'has_incomplete':False,'path':'ollama','backend':'ollama','is_ollama':True})", + "def scan_ollama_api():", + " urls = ['http://127.0.0.1:11434/api/tags', 'http://localhost:11434/api/tags', 'http://host.docker.internal:11434/api/tags']", + " for url in urls:", + " try:", + " with urllib.request.urlopen(url, timeout=2) as r:", + " data = json.loads(r.read().decode('utf-8', 'replace'))", + " except Exception:", + " continue", + " for item in data.get('models', []):", + " name = item.get('name') or item.get('model')", + " if not name or name in seen: continue", + " size_bytes = int(item.get('size') or item.get('size_bytes') or 0)", + " seen.add(name)", + " models.append({'repo_id':name,'size_bytes':size_bytes,'nb_files':1,'has_incomplete':False,'path':'ollama','backend':'ollama','is_ollama':True})", + " return", + "for _hf_cache in hf_cache_paths(): scan_hf(_hf_cache)", + "scan_ollama_api()", + "scan_ollama()", + ] + for model_dir in model_dirs or []: + lines.append(f"scan_dir({model_dir!r})") + lines.append("print(json.dumps(models))") + return "\n".join(lines) + "\n" + + def _ps_squote(v: str) -> str: """Escape a value for PowerShell single-quoted string interpolation. Belt-and-suspenders on top of _validate_token's regex — if the regex @@ -114,10 +584,22 @@ def _bash_squote(v: str) -> str: return v.replace("'", "'\\''") +# Shown by generated runner scripts when the ollama binary is missing on the +# target host. Must stay free of backticks/$( ) and be emitted single-quoted: +# an earlier version wrapped the install one-liner in backticks inside a +# double-quoted echo, which bash executed as command substitution and ran the +# system-wide installer (including on remote SSH hosts) instead of printing +# the hint. +OLLAMA_MISSING_HINT = ( + "ERROR: Ollama not found on this server. Install it from " + "https://ollama.com/download or run: curl -fsSL https://ollama.com/install.sh | sh" +) + + # Allow-list of binaries permitted as the leading token of `req.cmd` for /api/model/serve. # Anything else is rejected before the cmd is interpolated into a tmux/PowerShell wrapper. _SERVE_CMD_ALLOWLIST = { - "vllm", "llama-server", "llama_server", "llama.cpp", "ollama", + "vllm", "llama-server", "llama-server.exe", "llama_server", "llama.cpp", "ollama", "python", "python3", "sglang", "lmdeploy", "node", "npx", @@ -133,6 +615,94 @@ def _bash_squote(v: str) -> str: _GGUF_PRELUDE_RE = re.compile( r'^MODEL_FILE=\$\([^\n]*?\)\s*&&\s*\{[^{}]*\}\s*\|\|\s*\{[^{}]*\}\s*&&\s*' ) +_SAFE_SUBSHELL_TEXT = r"[^'\n;&|`$()<>]+" +_SAFE_SUBSHELL_DQ_HOME_PATH = r'"\$HOME/[^"\n;&|`()<>]*"' +_SAFE_PRINTF_SUBSHELL_RE = re.compile( + rf"^\$\(printf[ \t]+%s[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|\$\{{HOME\}}'/{_SAFE_SUBSHELL_TEXT}')\)$" +) +_SAFE_FIND_MMPROJ_SUBSHELL_RE = re.compile( + rf"^\$\(find[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|{_SAFE_SUBSHELL_DQ_HOME_PATH}|{_SAFE_SUBSHELL_TEXT})" + r"[ \t]+-iname[ \t]+'mmproj\*\.gguf'" + r"(?:[ \t]+2>/dev/null)?[ \t]*\|[ \t]*sort[ \t]*\|[ \t]*head[ \t]+-1\)$" +) +_OLLAMA_HOST_ASSIGNMENT_RE = re.compile(r"(?:^|\s)OLLAMA_HOST=([^\s]+)") +_OLLAMA_BIND_RE = re.compile(r"^\[([^\]]+)\]:(\d+)$|^([^:]+):(\d+)$") +_OLLAMA_BIND_HOST_RE = re.compile(r"^[A-Za-z0-9._:-]+$") +_LLAMA_CPP_PYTHON_GGML_TYPES = { + "f32": "0", + "f16": "1", + "q4_0": "2", + "q4_1": "3", + "q5_0": "6", + "q5_1": "7", + "q8_0": "8", + "q8_1": "9", + "q2_k": "10", + "q3_k": "11", + "q4_k": "12", + "q5_k": "13", + "q6_k": "14", + "q8_k": "15", + "iq2_xxs": "16", + "iq2_xs": "17", + "iq3_xxs": "18", + "iq1_s": "19", + "iq4_nl": "20", + "iq3_s": "21", + "iq2_s": "22", + "iq4_xs": "23", + "mxfp4": "39", + "nvfp4": "40", + "q1_0": "41", +} +_LLAMA_CPP_PYTHON_TYPE_FLAG_RE = re.compile( + r"(?P--type_[kv])(?P\s+|=)(?P['\"]?)(?P[A-Za-z0-9_]+)(?P=quote)" +) + + +def _ollama_bind_from_cmd(cmd: str | None, *, default_host: str = "127.0.0.1") -> tuple[str, str]: + """Return the Ollama bind host/port requested by a serve command. + + Plain local `ollama serve` defaults to loopback. Remote callers can pass a + wider default host so the resulting API is reachable by Odysseus. + """ + if not cmd: + return default_host, "11434" + match = _OLLAMA_HOST_ASSIGNMENT_RE.search(cmd) + if not match: + return default_host, "11434" + value = match.group(1).strip("'\"") + bind_match = _OLLAMA_BIND_RE.match(value) + if not bind_match: + return "127.0.0.1", "11434" + bracketed_host = bind_match.group(1) + host = bracketed_host or bind_match.group(3) or "127.0.0.1" + port = bind_match.group(2) or bind_match.group(4) or "11434" + if not _OLLAMA_BIND_HOST_RE.match(host): + return "127.0.0.1", "11434" + try: + port_num = int(port, 10) + except ValueError: + return "127.0.0.1", "11434" + if port_num < 1 or port_num > 65535: + return "127.0.0.1", "11434" + return f"[{host}]" if bracketed_host else host, port + + +def _normalize_llama_cpp_python_cache_types(cmd: str | None) -> str | None: + """Map llama.cpp KV cache type names to llama-cpp-python's integer enum.""" + if not cmd or "llama_cpp.server" not in cmd: + return cmd + + def repl(match: re.Match[str]) -> str: + value = match.group("value") + mapped = _LLAMA_CPP_PYTHON_GGML_TYPES.get(value.lower()) + if not mapped: + return match.group(0) + quote = match.group("quote") + return f"{match.group('flag')}{match.group('sep')}{quote}{mapped}{quote}" + + return _LLAMA_CPP_PYTHON_TYPE_FLAG_RE.sub(repl, cmd) def _check_serve_binary(seg: str) -> None: @@ -155,6 +725,13 @@ def _check_serve_binary(seg: str) -> None: ) +def _is_safe_serve_subshell(subshell: str) -> bool: + return bool( + _SAFE_PRINTF_SUBSHELL_RE.fullmatch(subshell) + or _SAFE_FIND_MMPROJ_SUBSHELL_RE.fullmatch(subshell) + ) + + def _validate_serve_cmd(v: str | None) -> str | None: """Reject serve commands that aren't in the allowlist or contain shell metachars. @@ -176,6 +753,7 @@ def _validate_serve_cmd(v: str | None) -> str | None: # Backticks and raw newlines are never legitimate here. if any(c in v for c in ("`", "\n", "\r")): raise HTTPException(400, "Invalid characters in cmd") + # Known GGUF launcher prelude → validate the serve invocation(s) it guards. m = _GGUF_PRELUDE_RE.match(v) if m: @@ -184,16 +762,306 @@ def _validate_serve_cmd(v: str | None) -> str | None: for part in rest.split("||"): _check_serve_binary(part.strip()) return v - # Otherwise: a single invocation — no shell metacharacters allowed. + + # Otherwise: a single invocation — no shell metacharacters allowed. Replace + # only the exact command substitutions emitted by the Cookbook UI: + # $(printf %s 'safe-path') and the mmproj lookup + # $(find -iname 'mmproj*.gguf' 2>/dev/null | sort | head -1). + def _replace_safe_subshell(match: re.Match[str]) -> str: + subshell = match.group(0) + return "/placeholder/safe/path" if _is_safe_serve_subshell(subshell) else subshell + + cleaned_v = re.sub(r"\$\([^()]*\)", _replace_safe_subshell, v) + # (`$(` was the original intent; bare `$` is fine for shell-safe paths.) - if any(c in v for c in (";", "&&", "||", "$(")): + if any(c in cleaned_v for c in (";", "&&", "||", "$(")): raise HTTPException(400, "Invalid characters in cmd") _check_serve_binary(v) return v +def _append_serve_preflight_exit_lines(runner_lines: list[str], *, keep_shell_open: bool) -> None: + """Append serve-runner lines that surface preflight failures before exit.""" + runner_lines.append('if [ -n "$ODYSSEUS_PREFLIGHT_EXIT" ]; then') + runner_lines.append(' echo ""; echo "=== Process exited with code $ODYSSEUS_PREFLIGHT_EXIT ==="') + if keep_shell_open: + # Decouple the post-crash interactive shell from the persistent log + # file. fds 3/4 were saved BEFORE the tee redirect at the top of + # the runner; restoring them here means the neofetch banner the + # user's .zshrc prints lands on the tmux pane only, not in the + # log file the agent's tail_serve_output reads. + runner_lines.append(' exec 1>&3 2>&4 3>&- 4>&- 2>/dev/null || true') + runner_lines.append(' sleep 0.2 # let tee child flush + exit') + runner_lines.append(' exec "${SHELL:-/bin/bash}"') + else: + runner_lines.append(' exit "$ODYSSEUS_PREFLIGHT_EXIT"') + runner_lines.append('fi') + + +def _append_vllm_linux_preflight_lines(runner_lines: list[str]) -> None: + """Append Linux vLLM readiness lines that identify the runtime being used.""" + # Keep the user install bin visible for Odysseus-managed `pip install --user` + # installs, but then report the actual CLI path so external runtimes are clear. + runner_lines.append('export PATH="$HOME/.local/bin:$PATH"') + runner_lines.append('ODYSSEUS_VLLM_BIN="$(command -v vllm 2>/dev/null || true)"') + runner_lines.append('if [ -z "$ODYSSEUS_VLLM_BIN" ]; then') + runner_lines.append(' echo "ERROR: vLLM is not installed."') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') + runner_lines.append('else') + runner_lines.append(' echo "[odysseus] vLLM CLI: $ODYSSEUS_VLLM_BIN"') + runner_lines.append(' ODYSSEUS_VLLM_VERSION="$("$ODYSSEUS_VLLM_BIN" --version 2>&1 | head -n 1 || true)"') + runner_lines.append(' if [ -n "$ODYSSEUS_VLLM_VERSION" ]; then echo "[odysseus] vLLM version: $ODYSSEUS_VLLM_VERSION"; fi') + runner_lines.append('fi') + +def _append_serve_exit_code_lines( + runner_lines: list[str], + *, + keep_shell_open: bool, + is_pip_install: bool = False, +) -> None: + """Append serve-runner lines that preserve and report the command exit code.""" + runner_lines.append('ODYSSEUS_CMD_EXIT=$?') + if is_pip_install: + runner_lines.append('if [ $ODYSSEUS_CMD_EXIT -eq 0 ]; then echo ""; echo "DOWNLOAD_OK"; fi') + if keep_shell_open: + runner_lines.append('echo ""; echo "=== Process exited with code $ODYSSEUS_CMD_EXIT ==="') + # See preflight branch above for the rationale on restoring fds 3/4. + runner_lines.append('exec 1>&3 2>&4 3>&- 4>&- 2>/dev/null || true') + runner_lines.append('sleep 0.2 # let tee child flush + exit') + runner_lines.append('exec "${SHELL:-/bin/bash}"') + else: + runner_lines.append('echo ""; echo "=== Process exited with code $ODYSSEUS_CMD_EXIT ==="') + runner_lines.append('exit "$ODYSSEUS_CMD_EXIT"') + + +def _append_llama_cpp_linux_accel_build_lines(runner_lines: list[str]) -> None: + """Append Linux llama.cpp build lines that prefer ROCm/HIP when available. + + Cookbook already detects AMD GPUs elsewhere, but the llama.cpp bootstrap used + to hard-wire CUDA on Linux. That made ROCm hosts attempt a CUDA configure and + fail with "CUDA Toolkit not found" instead of building with HIP. + """ + # Try a prebuilt binary from llama.cpp's GitHub releases FIRST — no + # cmake/build-essential/git/CUDA-headers needed at all. The from-source + # build below stays as a fallback (custom flags, esoteric arch, no + # internet, etc). 30 seconds vs 5+ minutes of compile, and removes + # every OS-package dep from the launch path. Sets _odysseus_have_prebuilt=1 + # on success; the existing build-tier if/elif chain below is gated on + # that variable so we never compile twice or shadow the prebuilt symlink. + runner_lines.append(' _odysseus_have_prebuilt=""') + runner_lines.append(' _odysseus_arch="$(uname -m)"') + runner_lines.append(' _odysseus_prebuilt_url=""') + runner_lines.append(' if command -v curl >/dev/null 2>&1 && [ "$_odysseus_arch" = "x86_64" ]; then') + runner_lines.append(' _odysseus_pat=""') + runner_lines.append(' _odysseus_has_nv_inline() { command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q "GPU "; }') + runner_lines.append(' _odysseus_has_vk_inline() { ldconfig -p 2>/dev/null | grep -q "libvulkan\\.so" || command -v vulkaninfo >/dev/null 2>&1 || [ -e /usr/lib/x86_64-linux-gnu/libvulkan.so.1 ]; }') + runner_lines.append(' _odysseus_has_vkdev_inline() { ls /dev/dri/renderD* >/dev/null 2>&1 || (lspci 2>/dev/null | grep -Ei \'VGA|3D|Display\' | grep -Eiq \'AMD|ATI|Radeon\'); }') + runner_lines.append(' if _odysseus_has_nv_inline; then') + runner_lines.append(' _odysseus_pat="ubuntu.*cuda"') + runner_lines.append(' elif _odysseus_has_vkdev_inline && _odysseus_has_vk_inline; then') + runner_lines.append(' _odysseus_pat="ubuntu.*vulkan"') + runner_lines.append(' else') + runner_lines.append(' _odysseus_pat="ubuntu-x64\\\\.zip"') + runner_lines.append(' fi') + runner_lines.append(' _odysseus_prebuilt_url="$(curl -fsSL --max-time 15 https://api.github.com/repos/ggml-org/llama.cpp/releases/latest 2>/dev/null | grep \'"browser_download_url"\' | cut -d\'"\' -f4 | grep -iE "$_odysseus_pat" | grep -iv "arm\\|aarch64" | head -1)"') + runner_lines.append(' fi') + # Preserve the release asset extension so the extractor can select + # the correct format. Linux llama.cpp assets are commonly .tar.gz, + # while some release assets are .zip. + runner_lines.append(' if [ -n "$_odysseus_prebuilt_url" ] && command -v python3 >/dev/null 2>&1; then') + runner_lines.append(' echo "[odysseus] Found prebuilt llama-server: $_odysseus_prebuilt_url"') + runner_lines.append(' mkdir -p ~/bin "$HOME/.cache/odysseus/llama-cpp-prebuilt" && cd "$HOME/.cache/odysseus/llama-cpp-prebuilt"') + runner_lines.append(' case "$_odysseus_prebuilt_url" in') + runner_lines.append(' *.tar.gz|*.tgz) _odysseus_archive="llama-cpp.tar.gz" ;;') + runner_lines.append(' *.zip) _odysseus_archive="llama-cpp.zip" ;;') + runner_lines.append(' *) _odysseus_archive="" ;;') + runner_lines.append(' esac') + runner_lines.append(' rm -f llama-cpp.zip llama-cpp.tar.gz') + runner_lines.append(' if [ -n "$_odysseus_archive" ] && curl -fsSL --max-time 120 "$_odysseus_prebuilt_url" -o "$_odysseus_archive" && [ -s "$_odysseus_archive" ]; then') + runner_lines.append(' rm -rf build && mkdir -p build') + runner_lines.append(' python3 -c "import shutil, sys; shutil.unpack_archive(sys.argv[1], sys.argv[2])" "$_odysseus_archive" build') + runner_lines.append(' _odysseus_extracted="$(find build -type f -name llama-server 2>/dev/null | head -1)"') + runner_lines.append(' if [ -n "$_odysseus_extracted" ]; then') + runner_lines.append(' chmod +x "$_odysseus_extracted"') + runner_lines.append(' ln -sf "$_odysseus_extracted" ~/bin/llama-server') + runner_lines.append(' _odysseus_libdir="$(dirname "$_odysseus_extracted")"') + runner_lines.append(' mkdir -p ~/.config && echo "export LD_LIBRARY_PATH=\\"$_odysseus_libdir:\\${LD_LIBRARY_PATH:-}\\"" > ~/.config/odysseus-llama-cpp-env') + runner_lines.append(' _odysseus_have_prebuilt=1') + runner_lines.append(' echo "[odysseus] Prebuilt llama-server installed at $_odysseus_extracted"') + runner_lines.append(' fi') + runner_lines.append(' fi') + runner_lines.append(' [ -z "$_odysseus_have_prebuilt" ] && echo "[odysseus] Prebuilt download/extract failed — falling back to from-source build."') + runner_lines.append(' elif [ -z "$_odysseus_prebuilt_url" ]; then') + runner_lines.append(' echo "[odysseus] No matching prebuilt llama-server for this host (arch=$_odysseus_arch) — will build from source."') + runner_lines.append(' fi') + runner_lines.append(' if [ -z "$_odysseus_have_prebuilt" ]; then') + # Detect pip-installed nvcc (from vLLM/nvidia CUDA wheels) and put it on PATH + # so cmake's CUDA configure can find it — BUT only when actual NVIDIA + # hardware is present. On AMD/Intel hosts the pip nvcc is a misleading + # leftover (no libcudart, no GPU it could target) and would otherwise + # send the build down the CUDA branch and fail with "CUDA Toolkit not + # found" instead of trying Vulkan. + runner_lines.append(' _odysseus_has_nvidia_hw() {') + runner_lines.append(' command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q "GPU " && return 0') + runner_lines.append(' ls /dev/nvidia* >/dev/null 2>&1 && return 0') + runner_lines.append(' lspci 2>/dev/null | grep -iE \'VGA|3D|Display\' | grep -iq nvidia && return 0') + runner_lines.append(' return 1') + runner_lines.append(' }') + runner_lines.append(' if _odysseus_has_nvidia_hw; then') + runner_lines.append(' for _cudir in ~/.local/lib/python*/site-packages/nvidia/cu13 ~/.local/lib/python*/site-packages/nvidia/cu12 ~/.local/lib/python*/site-packages/nvidia/cuda_nvcc; do') + runner_lines.append(' [ -x "$_cudir/bin/nvcc" ] && export CUDA_HOME="$_cudir" && export PATH="$_cudir/bin:$PATH" && break') + runner_lines.append(' done') + runner_lines.append(' fi') + # rm -rf build so a prior poisoned CMakeCache.txt (e.g. from a failed CUDA + # or HIP attempt) doesn't cause the next configure to reuse stale settings. + runner_lines.append(' mkdir -p ~/bin') + # Try to install cmake / build-essential / git automatically before the + # build, but ONLY via passwordless sudo (`sudo -n`) — interactive sudo + # would hang a tmux-backgrounded serve task waiting for a password. If + # sudo asks for a password the install is skipped silently and the + # diagnosis pattern (cookbook_routes.py / cookbook_helpers.py) surfaces + # an explicit "install cmake" suggestion in the Cookbook diagnosis + # toolbar after the inevitable build failure. + runner_lines.append(' _odysseus_apt_bootstrap() {') + runner_lines.append(' local _missing=""') + runner_lines.append(' command -v cmake >/dev/null 2>&1 || _missing="$_missing cmake"') + runner_lines.append(' command -v g++ >/dev/null 2>&1 || command -v gcc >/dev/null 2>&1 || _missing="$_missing build-essential"') + runner_lines.append(' command -v git >/dev/null 2>&1 || _missing="$_missing git"') + runner_lines.append(' [ -z "$_missing" ] && return 0') + runner_lines.append(' if command -v apt-get >/dev/null 2>&1 && sudo -n true 2>/dev/null; then') + runner_lines.append(' echo "[odysseus] Auto-installing missing build deps via apt:$_missing"') + runner_lines.append(' sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq 2>&1 | tail -3') + runner_lines.append(' sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $_missing 2>&1 | tail -5 || true') + runner_lines.append(' elif command -v pacman >/dev/null 2>&1 && sudo -n true 2>/dev/null; then') + runner_lines.append(' echo "[odysseus] Auto-installing missing build deps via pacman:$_missing"') + runner_lines.append(' local _pacpkgs="$(echo "$_missing" | sed -e \'s/build-essential/base-devel/g\')"') + runner_lines.append(' sudo -n pacman -Sy --needed --noconfirm $_pacpkgs 2>&1 | tail -5 || true') + runner_lines.append(' elif command -v dnf >/dev/null 2>&1 && sudo -n true 2>/dev/null; then') + runner_lines.append(' echo "[odysseus] Auto-installing missing build deps via dnf:$_missing"') + runner_lines.append(' local _dnfpkgs="$(echo "$_missing" | sed -e \'s/build-essential/gcc gcc-c++ make/g\')"') + runner_lines.append(' sudo -n dnf install -y $_dnfpkgs 2>&1 | tail -5 || true') + runner_lines.append(' else') + runner_lines.append(' echo "[odysseus] WARNING: missing build deps ($_missing) — passwordless sudo is unavailable, cannot auto-install. Cookbook Diagnosis will explain the fix after the build fails."') + runner_lines.append(' fi') + runner_lines.append(' }') + runner_lines.append(' _odysseus_apt_bootstrap') + runner_lines.append(' _odysseus_missing_build_deps=""') + runner_lines.append(' command -v cmake >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps cmake"') + runner_lines.append(' command -v git >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps git"') + runner_lines.append(' command -v g++ >/dev/null 2>&1 || command -v gcc >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps build-essential"') + runner_lines.append(' if [ -n "$_odysseus_missing_build_deps" ]; then') + runner_lines.append(' echo "ERROR: llama.cpp source build needs missing packages:$_odysseus_missing_build_deps"') + runner_lines.append(' if command -v apt-get >/dev/null 2>&1; then') + runner_lines.append(' echo "Install on this host: sudo apt-get update && sudo apt-get install -y cmake build-essential git"') + runner_lines.append(' elif command -v pacman >/dev/null 2>&1; then') + runner_lines.append(' echo "Install on this host: sudo pacman -Sy --needed cmake base-devel git"') + runner_lines.append(' elif command -v dnf >/dev/null 2>&1; then') + runner_lines.append(' echo "Install on this host: sudo dnf install -y cmake gcc gcc-c++ make git"') + runner_lines.append(' fi') + runner_lines.append(' echo "Alternative: install a native llama-server on PATH, then relaunch."') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') + runner_lines.append(' fi') + runner_lines.append(' cd ~/llama.cpp') + runner_lines.append(' _odysseus_has_vulkan() {') + runner_lines.append(' ldconfig -p 2>/dev/null | grep -q \'libvulkan\\.so\' && return 0') + runner_lines.append(' [ -e /usr/lib/libvulkan.so.1 ] && return 0') + runner_lines.append(' [ -e /usr/lib/x86_64-linux-gnu/libvulkan.so.1 ] && return 0') + runner_lines.append(' command -v vulkaninfo >/dev/null 2>&1 && return 0') + runner_lines.append(' return 1') + runner_lines.append(' }') + runner_lines.append(' _odysseus_has_vulkan_device() {') + runner_lines.append(' ls /dev/dri/renderD* >/dev/null 2>&1 && return 0') + runner_lines.append(' lspci 2>/dev/null | grep -Ei \'VGA|3D|Display\' | grep -Eiq \'AMD|ATI|Radeon\' && return 0') + runner_lines.append(' return 1') + runner_lines.append(' }') + # Backend preference: native ROCm/HIP > native CUDA > Vulkan > CPU. + # Vulkan is a portable fallback that works on AMD when ROCm isn't + # installed (e.g. Strix Halo) and on any vendor's discrete GPU, but + # it's ~30-40% slower than native HIP/CUDA for LLM inference — only + # pick it when no native toolchain is present. + runner_lines.append(' if command -v hipconfig &>/dev/null || [ -d /opt/rocm ] || [ -n "$ROCM_PATH" ] || [ -n "$HIP_PATH" ]; then') + runner_lines.append(' rm -rf build') + runner_lines.append(' if command -v hipconfig &>/dev/null; then') + runner_lines.append(' export HIPCXX="${HIPCXX:-$(hipconfig -l)/clang}"') + runner_lines.append(' export HIP_PATH="${HIP_PATH:-$(hipconfig -R)}"') + runner_lines.append(' fi') + runner_lines.append(' echo "[odysseus] ROCm/HIP detected — building llama-server with HIP support..."') + runner_lines.append(' cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_HIP=ON && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server') + runner_lines.append(' elif command -v nvcc &>/dev/null && _odysseus_has_nvidia_hw; then') + runner_lines.append(' rm -rf build') + # nvcc alone is not sufficient — pip-installed CUDA wheels or incomplete + # tooling can expose nvcc without shipping libcudart, causing cmake to fail + # mid-build with "CUDA runtime library not found". Check cudart explicitly + # via a small helper so the guard stays readable. + runner_lines.append(' _odysseus_has_cudart() {') + runner_lines.append(' ldconfig -p 2>/dev/null | grep -q \'libcudart\\.so\' && return 0') + runner_lines.append(' local _cuh="${CUDA_HOME:-/usr/local/cuda}"') + runner_lines.append(' ls "$_cuh/lib64/libcudart.so"* &>/dev/null && return 0') + runner_lines.append(' ls "$_cuh/lib/libcudart.so"* &>/dev/null && return 0') + runner_lines.append(' ls /usr/local/cuda/lib64/libcudart.so* &>/dev/null && return 0') + runner_lines.append(' ls /usr/local/cuda/lib/libcudart.so* &>/dev/null && return 0') + runner_lines.append(' ls "${_cuh%/cuda_nvcc}/cuda_runtime/lib/libcudart.so"* &>/dev/null && return 0') + runner_lines.append(' return 1') + runner_lines.append(' }') + runner_lines.append(' if _odysseus_has_cudart; then') + runner_lines.append(' echo "[odysseus] CUDA nvcc + cudart found — building llama-server with CUDA (GPU) support..."') + runner_lines.append(' cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_CUDA=ON && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server') + runner_lines.append(' else') + runner_lines.append(' echo "[odysseus] WARNING: nvcc found but CUDA runtime (libcudart.so) is not visible — building llama-server for CPU only."') + runner_lines.append(' echo "[odysseus] GPU inference will not be available for this llama.cpp build."') + runner_lines.append(' echo "[odysseus] Ensure libcudart is installed (e.g. cuda-runtime package) and visible via ldconfig or CUDA_HOME."') + runner_lines.append(' cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server') + runner_lines.append(' fi') + runner_lines.append(' elif _odysseus_has_vulkan_device && _odysseus_has_vulkan; then') + runner_lines.append(' echo "[odysseus] Vulkan-capable GPU detected (no ROCm/CUDA toolchain installed) — building llama-server with Vulkan support..."') + runner_lines.append(' rm -rf build-vulkan') + runner_lines.append(' cmake -B build-vulkan -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON && cmake --build build-vulkan -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build-vulkan/bin/llama-server ~/bin/llama-server') + runner_lines.append(' else') + runner_lines.append(' echo "[odysseus] WARNING: no HIP/CUDA/Vulkan toolchain found — building llama-server for CPU only."') + runner_lines.append(' echo "[odysseus] GPU inference will not be available for this llama.cpp build."') + runner_lines.append(' echo "[odysseus] Install Vulkan (libvulkan-dev) / ROCm for AMD GPUs or CUDA tooling for NVIDIA, then re-launch this serve task."') + runner_lines.append(' rm -rf build') + runner_lines.append(' cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server') + runner_lines.append(' fi') + runner_lines.append(' fi # end _odysseus_have_prebuilt guard') + + +def _llama_cpp_rebuild_cmd(update_source: bool = False) -> str: + """Shell command that clears the Cookbook-managed llama.cpp build. + + Removes the cached ``llama-server`` symlink and the ``~/llama.cpp/build*`` + directory so the next llama.cpp serve recompiles from source, picking up a + CUDA or HIP toolchain if one is now available. The serve bootstrap only + builds when ``llama-server`` is missing from PATH, so without this an + existing CPU-only build is reused forever. When ``update_source`` is true, + the command also fast-forwards the Cookbook-managed ``~/llama.cpp`` checkout + if it exists. The rebuild itself happens on the next serve. + """ + update_cmd = '' + if update_source: + update_cmd = ( + 'if [ -d "$HOME/llama.cpp/.git" ]; then ' + 'git -C "$HOME/llama.cpp" pull --ff-only --depth 1 || ' + 'echo "[odysseus] WARNING: llama.cpp source update failed; clearing cached build anyway."; ' + 'elif command -v git >/dev/null 2>&1; then ' + 'git clone --depth 1 https://github.com/ggml-org/llama.cpp "$HOME/llama.cpp" || ' + 'echo "[odysseus] WARNING: llama.cpp clone failed; clearing cached build anyway."; ' + 'fi && ' + ) + return ( + 'mkdir -p "$HOME/bin" && ' + f'{update_cmd}' + 'rm -f "$HOME/bin/llama-server" && ' + 'rm -rf "$HOME/llama.cpp/build" "$HOME/llama.cpp/build-vulkan" && ' + 'echo "[odysseus] Cleared the cached llama.cpp build. ' + 'Re-launch the serve task to rebuild llama-server from source ' + '(Vulkan, HIP, or CUDA will be used if a matching toolchain is now available)."' + ) + + class ModelDownloadRequest(BaseModel): repo_id: str + backend: str | None = None # "hf" (default) or "ollama" include: str | None = None # glob pattern e.g. "*Q4_K_M*" hf_token: str | None = None env_prefix: str | None = None # e.g. "source ~/venv/bin/activate" @@ -254,6 +1122,8 @@ def _parse_serve_phase(snapshot: str, task_type: str = "serve") -> dict: } if "Application startup complete" in flat: return {"phase": "ready", "status": "ready"} + if re.search(r'Ollama API ready on port\s+\d+', flat, re.I): + return {"phase": "ready", "status": "ready"} # HTTP access logs (e.g. GET /v1/models 200 OK) mean the server is up and serving if re.search(r'(?:GET|POST)\s+/[^\s]*\s+HTTP/[\d.]+"\s*\d{3}', flat): return {"phase": "idle", "status": "ready"} @@ -338,3 +1208,216 @@ def _ssh_ps(host, script_path, port=None): # Windows session dir — stored in user's temp on the remote WIN_SESSION_DIR = "$env:TEMP\\\\odysseus-sessions" + + +def _diagnose_serve_output(text: str) -> dict | None: + """Server-side mirror of the Cookbook UI's common serve diagnoses. + + The browser uses cookbook-diagnosis.js for clickable fixes. This gives + the agent/tool path the same structured signal so it can retry with an + adjusted command instead of guessing from raw tmux output. + """ + if not text: + return None + tail = text[-6000:] + patterns = [ + ( + r"No available memory for the cache blocks|Available KV cache memory:.*-", + "No GPU memory left for KV cache after loading model.", + [ + {"label": "retry with GPU memory utilization 0.95", "op": "replace", "flag": "--gpu-memory-utilization", "value": "0.95"}, + {"label": "retry with context 2048", "op": "replace", "flag": "--max-model-len", "value": "2048"}, + ], + ), + ( + r"CUDA out of memory|torch\.cuda\.OutOfMemoryError|CUDA error: out of memory|warming up sampler|max_num_seqs.*gpu_memory_utilization", + "GPU ran out of memory during startup or warmup.", + [ + {"label": "retry with context 4096", "op": "replace", "flag": "--max-model-len", "value": "4096"}, + {"label": "retry with GPU memory utilization 0.80", "op": "replace", "flag": "--gpu-memory-utilization", "value": "0.80"}, + {"label": "retry with --enforce-eager", "op": "append", "arg": "--enforce-eager"}, + ], + ), + ( + r"not divisib|must be divisible|attention heads.*divisible", + "Tensor parallel size is incompatible with the model.", + [ + {"label": "retry with tensor parallel size 1", "op": "replace", "flag": "--tensor-parallel-size", "value": "1"}, + {"label": "retry with tensor parallel size 2", "op": "replace", "flag": "--tensor-parallel-size", "value": "2"}, + ], + ), + ( + r"KV cache.*too (small|large)|max_model_len.*exceeds|maximum.*context", + "Context length is too large for available GPU memory.", + [ + {"label": "retry with context 8192", "op": "replace", "flag": "--max-model-len", "value": "8192"}, + {"label": "retry with context 4096", "op": "replace", "flag": "--max-model-len", "value": "4096"}, + ], + ), + ( + r"enable-auto-tool-choice requires --tool-call-parser", + "Auto tool choice requires an explicit tool call parser.", + [{"label": "retry with Hermes tool parser", "op": "append", "arg": "--tool-call-parser hermes"}], + ), + ( + r"Please pass.*trust.remote.code=True|contains custom code which must be executed to correctly load|does not recognize this architecture|model type.*but Transformers does not", + "Model requires custom code or newer model support.", + [{"label": "retry with --trust-remote-code", "op": "append", "arg": "--trust-remote-code"}], + ), + ( + r"There is no module or parameter named ['\"]lm_head\.input_scale['\"]|lm_head\.input_scale|weight_scale_2", + "vLLM cannot load this ModelOpt LM-head quantized checkpoint with the current runtime.", + [ + { + "label": "upgrade vLLM through the environment that provides this CLI, or use a compatible checkpoint", + "op": "manual", + } + ], + ), + ( + r"Either a revision or a version must be specified|transformers\.integrations\.hub_kernels|kernels/layer", + "vLLM/Transformers kernel package mismatch.", + [{"label": "update vLLM, Transformers, and kernels on this server", "op": "dependency", "package": "vllm transformers kernels"}], + ), + ( + r"Address already in use|bind.*address.*in use", + "Port is already in use.", + [{"label": "retry on port 8001", "op": "replace", "flag": "--port", "value": "8001"}], + ), + ( + r"No CUDA GPUs are available|no GPU.*found|CUDA_VISIBLE_DEVICES.*invalid", + "No GPUs are visible to the serve process.", + [{"label": "clear Cookbook GPU selection or choose available GPUs", "op": "settings", "field": "gpus", "value": ""}], + ), + ( + r"Failed to infer device type|NVML Shared Library Not Found|No module named 'amdsmi'|platform is not available", + "vLLM could not find a supported GPU (CUDA or ROCm). " + "This machine may have integrated or unsupported graphics only.", + [ + {"label": "switch to llama.cpp (CPU/Metal, works without a discrete GPU)", "op": "manual"}, + {"label": "switch to Ollama (CPU/Metal, works without a discrete GPU)", "op": "manual"}, + ], + ), + ( + r"vllm.*command not found|No module named vllm|ERROR: vLLM is not installed", + "vLLM is not installed or not in PATH on this server.", + [{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}], + ), + ( + r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|" + r"(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|" + r"Could not load any common_ops library|" + r"Please ensure sgl_kernel is properly installed", + "SGLang native kernel/runtime is missing or mismatched on this server.", + [ + {"label": "repair sglang-kernel in this Python environment", "op": "dependency", "package": "sglang-kernel"}, + {"label": "install OS packages: libnuma-dev python3.12-dev build-essential", "op": "manual"}, + {"label": "if libnvrtc is still missing, install the matching CUDA/NVRTC runtime on this host", "op": "manual"}, + ], + ), + ( + r"sglang.*command not found|No module named sglang|SGLang is not installed", + "SGLang is not installed or not in PATH on this server.", + [{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}], + ), + ( + r"No module named ['\"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed", + "MLX LM is not installed on this server.", + [{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}], + ), + ( + r"Unable to quantize model of type |QuantizedSwitchLinear", + "MLX-LM tried to quantize an already-quantized DeepSeek switch layer.", + [ + {"label": "relaunch from the cached local Hugging Face snapshot path on this Mac", "op": "manual"}, + {"label": "Odysseus now rewrites MLX repo-id launches to a cached snapshot when one exists", "op": "manual"}, + ], + ), + # System build deps come BEFORE the generic llama.cpp catch-all so + # cmake / build-essential / git missing → a specific OS-package + # remediation instead of "install llama-cpp-python[server]" (which + # itself fails to compile when cmake is absent). + ( + r"cmake: command not found|cmake.*not found.*[Cc]ould not", + "cmake is required to build llama.cpp from source but isn't installed on this server.", + [{"label": "install build deps for llama.cpp (apt: cmake build-essential git / pacman: cmake base-devel git / dnf: cmake gcc-c++ make git / brew: cmake git)", "op": "dependency", "package": "llama-cpp-python[server]"}], + ), + ( + r"^(make|g\+\+|gcc): command not found|Could not find C\+\+ compiler", + "A C/C++ compiler (build-essential) is required to build llama.cpp from source.", + [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}], + ), + ( + r"^git: command not found", + "git is required to clone the llama.cpp source tree.", + [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}], + ), + ( + r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'", + "llama.cpp / llama-cpp-python dependencies are missing.", + [{"label": "install llama.cpp dependencies or llama-cpp-python[server]", "op": "dependency", "package": "llama-cpp-python[server]"}], + ), + ( + r"No GGUF found on this host|no \.gguf file|No GGUF file found", + "No GGUF file found for this model on this host. The llama.cpp backend needs a .gguf file.", + [{"label": "download a GGUF build of this model (repo name usually ends in -GGUF, file like Q4_K_M.gguf)", "op": "manual"}], + ), + ( + r"No module named 'torch'|No module named torch|No module named 'diffusers'|No module named diffusers", + "Diffusion serving requires PyTorch and diffusers.", + [{"label": "install diffusers[torch] in Cookbook Dependencies", "op": "dependency", "package": "diffusers[torch]"}], + ), + ( + r"403 Forbidden|401 Unauthorized|Access to model.*is restricted|gated repo|not in the authorized list|awaiting a review", + "Model access is gated or unauthorized.", + [{"label": "set HF token and request model access on HuggingFace", "op": "manual"}], + ), + ] + for pattern, message, suggestions in patterns: + if re.search(pattern, tail, re.I): + return {"message": message, "suggestions": suggestions} + if re.search(r"Traceback \(most recent call last\)", tail, re.I) and not re.search( + r"Application startup complete|GET /v1/|Uvicorn running on", tail, re.I + ): + return { + "message": "Python traceback detected during serve startup.", + "suggestions": [{"label": "inspect traceback and retry with adjusted backend/settings", "op": "manual"}], + } + return None + + +async def run_ssh_command_async( + remote: str, + ssh_port: str | None, + remote_cmd: str, + *, + timeout: float, + connect_timeout: int | None = None, + strict_host_key_checking: bool | None = None, + stdin_data: bytes | None = None, +) -> tuple[int, bytes, bytes]: + """Run an ssh command with centralized timeout and stderr/stdout capture. + Async version of core.platform_compat.run_ssh_command_sync. + """ + import asyncio + proc = await asyncio.create_subprocess_exec( + *_ssh_exec_argv( + remote, + ssh_port, + remote_cmd=remote_cmd, + connect_timeout=connect_timeout, + strict_host_key_checking=strict_host_key_checking, + ), + stdin=asyncio.subprocess.PIPE if stdin_data is not None else None, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for( + proc.communicate(input=stdin_data), timeout=timeout + ) + except asyncio.TimeoutError: + proc.kill() + await proc.communicate() + raise + return proc.returncode or 0, stdout, stderr diff --git a/routes/cookbook_output.py b/routes/cookbook_output.py new file mode 100644 index 000000000..b30b18536 --- /dev/null +++ b/routes/cookbook_output.py @@ -0,0 +1,75 @@ +"""Pure helpers for shaping cookbook task output for the status response. + +Kept dependency-free (no FastAPI / SQLAlchemy imports) so the behavior can be +unit-tested without standing up the whole app. +""" + +import re + +_FETCHING_ZERO_FILES_RE = re.compile(r"Fetching\s+0\s+files", re.IGNORECASE) + +# Probe scripts for the dead-session download check, run as +# `python3 -c ` (locally or over SSH). +# cache_root is the task's custom download dir, '' for the default HF cache. +# It has to be passed explicitly: the download runner exports +# HF_HOME=, so that task's cache lives under /hub, and +# the probe process's own environment knows nothing about it. +HF_CACHE_COMPLETE_PROBE = ( + "import os,sys;" + "repo=sys.argv[1];" + "root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';" + "base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));" + "d=os.path.join(base,'models--'+repo.replace('/','--'));" + "snap=os.path.join(d,'snapshots');" + "ok=os.path.isdir(snap) and any(os.path.isdir(os.path.join(snap,x)) and os.listdir(os.path.join(snap,x)) for x in os.listdir(snap));" + "inc=False;" + "blobs=os.path.join(d,'blobs');" + "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));" + "sys.exit(0 if ok and not inc else 1)" +) + +HF_CACHE_INCOMPLETE_PROBE = ( + "import os,sys;" + "repo=sys.argv[1];" + "root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';" + "base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));" + "d=os.path.join(base,'models--'+repo.replace('/','--'));" + "blobs=os.path.join(d,'blobs');" + "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));" + "sys.exit(0 if inc else 1)" +) + + +def classify_dead_download(full_snapshot: str): + """Resolve a dead download session's status from its runner markers. + + The runner prints DOWNLOAD_OK only after exiting 0 (and DOWNLOAD_FAILED + otherwise), so the markers stay trustworthy after the tmux pane is gone. + Returns (status, zero_files), or None when the snapshot carries no marker + and the caller has to fall back to the cache probe. Same precedence as + the live-session branch: DOWNLOAD_OK wins, except a "Fetching 0 files" + run is an error (nothing matched the include/quant pattern). + """ + if not full_snapshot: + return None + if "DOWNLOAD_OK" in full_snapshot: + if _FETCHING_ZERO_FILES_RE.search(full_snapshot): + return ("error", True) + return ("completed", False) + if "DOWNLOAD_FAILED" in full_snapshot: + return ("error", False) + return None + + +def error_aware_output_tail(full_snapshot: str, status: str) -> str: + """Return the trailing slice of a task log for the status response. + + Failed tasks return the last 50 lines so the "Copy last 50 lines" action + surfaces the actual error context (stack traces, build output). Running and + other non-error tasks keep the cheaper 12-line tail to limit the payload on + the 10s polling interval. + """ + if not full_snapshot: + return "" + tail_lines = 50 if status == "error" else 12 + return "\n".join(full_snapshot.splitlines()[-tail_lines:]) diff --git a/routes/cookbook_routes.py b/routes/cookbook_routes.py index 9ba054b32..b64f1d3c3 100644 --- a/routes/cookbook_routes.py +++ b/routes/cookbook_routes.py @@ -7,25 +7,59 @@ import re import shlex import shutil +import subprocess +import sys +import time +import urllib.request import uuid from pathlib import Path from fastapi import APIRouter, HTTPException, Request, Depends from src.auth_helpers import require_user +from src.constants import COOKBOOK_STATE_FILE from pydantic import BaseModel from core.middleware import require_admin +from routes._validators import validate_remote_host, validate_ssh_port +from core.platform_compat import ( + IS_WINDOWS, + detached_popen_kwargs, + find_bash, + kill_process_tree, + pid_alive, + safe_chmod, + which_tool, +) from routes.shell_routes import TMUX_LOG_DIR +from src.host_docker_access import ( + HOST_DOCKER_ACCESS_HINT, + HOST_DOCKER_SOCKET_PATH, + host_docker_access_enabled, + local_docker_available, + running_in_container, +) +from routes.cookbook_output import ( + error_aware_output_tail, classify_dead_download, + HF_CACHE_COMPLETE_PROBE, HF_CACHE_INCOMPLETE_PROBE, +) logger = logging.getLogger(__name__) from routes.cookbook_helpers import ( - _SSH_PORT_RE, _REMOTE_HOST_RE, _SESSION_ID_RE, - _validate_repo_id, _validate_include, _validate_remote_host, _validate_token, - _validate_local_dir, _validate_ssh_port, _validate_gpus, _shell_path, - _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, - _safe_env_prefix, + _SESSION_ID_RE, _validate_repo_id, _validate_serve_model_id, _validate_include, _validate_token, + _validate_local_dir, _validate_gpus, _shell_path, + _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, OLLAMA_MISSING_HINT, + _safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines, + _append_serve_exit_code_lines, _append_llama_cpp_linux_accel_build_lines, _cached_model_scan_script, + load_stored_hf_token, + _append_vllm_linux_preflight_lines, _ollama_bind_from_cmd, _pip_install_fallback_chain, + _pip_install_no_cache, _user_shell_path_bootstrap, _venv_safe_local_pip_install_cmd, + _diagnose_serve_output, run_ssh_command_async, + _ollama_bind_from_cmd, _pip_install_fallback_chain, _pip_install_no_cache, + _user_shell_path_bootstrap, _venv_safe_local_pip_install_cmd, + _append_pip_install_runner_lines, _pip_install_command_without_break_system_packages, + _normalize_llama_cpp_python_cache_types, ModelDownloadRequest, ServeRequest, ) @@ -34,13 +68,297 @@ 'echo "[odysseus] HF token: applied"; ' 'else ' 'echo "[odysseus] HF token: NOT SET — gated/private models will be denied. ' - 'Add one in Odysseus Settings -> Cookbook -> HuggingFace Token."; ' + 'Add one in Odysseus Cookbook -> Settings -> HuggingFace Token."; ' 'fi' ) + +def _venv_root_from_serve_cmd(cmd: str) -> str: + """Best-effort venv root from an absolute venv python in a serve command.""" + try: + parts = shlex.split(cmd or "") + except Exception: + parts = (cmd or "").split() + for part in parts: + if re.search(r"/bin/python(?:3(?:\.\d+)?)?$", part or ""): + return re.sub(r"/bin/python(?:3(?:\.\d+)?)?$", "", part) + return "" + + +def _append_venv_nvidia_library_path_lines(lines: list[str], *, cmd: str = "") -> None: + """Expose NVIDIA CUDA runtime wheels bundled inside the active venv. + + SGLang/vLLM wheels can depend on CUDA libraries shipped as Python packages + under site-packages/nvidia. Activating the venv puts Python packages on + sys.path, but the dynamic loader still cannot find libraries such as + libnvrtc.so.13 unless those package lib dirs are on LD_LIBRARY_PATH. + """ + venv_root = _venv_root_from_serve_cmd(cmd) + lines.append(f'_ODY_VENV_FOR_LIBS="${{VIRTUAL_ENV:-{_bash_squote(venv_root)}}}"') + lines.append('if [ -n "$_ODY_VENV_FOR_LIBS" ] && [ -d "$_ODY_VENV_FOR_LIBS" ]; then') + lines.append(' for _ody_nvlib in "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cu13/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cu12/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cuda_nvrtc/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cuda_runtime/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cublas/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cudnn/lib; do') + lines.append(' [ -d "$_ody_nvlib" ] && export LD_LIBRARY_PATH="$_ody_nvlib:${LD_LIBRARY_PATH:-}"') + lines.append(' done') + lines.append('fi') + + +def _serve_port_from_cmd(cmd: str) -> str: + m = re.search(r"--port(?:=|\s+)(\d+)", cmd or "") + return m.group(1) if m else "" + + +def _append_openai_port_preflight_lines(lines: list[str], *, cmd: str, expected_model: str) -> None: + port = _serve_port_from_cmd(cmd) + if not port: + return + lines.append(f"ODYSSEUS_SERVE_PORT='{_bash_squote(port)}'") + lines.append(f"ODYSSEUS_EXPECTED_MODEL='{_bash_squote(expected_model)}'") + lines.append("if [ -n \"$ODYSSEUS_SERVE_PORT\" ]; then") + lines.append(" python3 - \"$ODYSSEUS_SERVE_PORT\" \"$ODYSSEUS_EXPECTED_MODEL\" <<'PY'") + lines.append("import json, sys, urllib.request") + lines.append("port = sys.argv[1]") + lines.append("expected = (sys.argv[2] or '').strip()") + lines.append("url = f'http://127.0.0.1:{port}/v1/models'") + lines.append("try:") + lines.append(" with urllib.request.urlopen(url, timeout=1.5) as r:") + lines.append(" data = json.loads(r.read().decode('utf-8', 'replace') or '{}')") + lines.append("except Exception:") + lines.append(" raise SystemExit(0)") + lines.append("models = [str(x.get('id') or '') for x in data.get('data', []) if isinstance(x, dict)]") + lines.append("def base(s): return s.lower().split('/')[-1]") + lines.append("match = bool(expected) and any((m.lower() == expected.lower() or base(m) == base(expected) or base(expected) in m.lower() or base(m) in expected.lower()) for m in models)") + lines.append("print(f'ERROR: Port {port} is already serving {models or [\"unknown\"]}.')") + lines.append("if expected and not match:") + lines.append(" print(f'ERROR: Cookbook was about to launch {expected}, but this port is occupied by a different model. Stop the old server or choose another port.')") + lines.append("else:") + lines.append(" print('ERROR: Stop the existing server or choose another port before launching a duplicate serve.')") + lines.append("raise SystemExit(98)") + lines.append("PY") + lines.append(" _ody_port_ec=$?") + lines.append(" if [ \"$_ody_port_ec\" -ne 0 ]; then ODYSSEUS_PREFLIGHT_EXIT=\"$_ody_port_ec\"; fi") + lines.append("fi") + +_OLLAMA_SIDECAR_CONTAINERS = {"ollama-test", "ollama-rocm"} +_UNSAFE_DOCKER_EXEC_CHARS = frozenset(";&|<>$`\r\n") +_SAFE_OLLAMA_MODEL_TOKEN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]*$") +_SAFE_OLLAMA_FILE_TOKEN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") + + +def _is_generated_ollama_docker_exec_cmd(cmd: str | None) -> bool: + """Match only the fixed Docker exec shapes generated by Cookbook.""" + if not cmd or any(char in cmd for char in _UNSAFE_DOCKER_EXEC_CHARS): + return False + try: + parts = shlex.split(cmd) + except ValueError: + return False + if len(parts) < 4 or parts[:2] != ["docker", "exec"]: + return False + container, executable = parts[2:4] + if container not in _OLLAMA_SIDECAR_CONTAINERS: + return False + if container == "ollama-rocm" and executable == "ollama": + return ( + len(parts) == 6 + and parts[4] == "show" + and _SAFE_OLLAMA_MODEL_TOKEN_RE.fullmatch(parts[5]) is not None + ) + if container != "ollama-test" or executable != "ollama-import": + return False + if len(parts) not in {7, 8}: + return False + model, name, context_size = parts[4:7] + return ( + _SAFE_OLLAMA_MODEL_TOKEN_RE.fullmatch(model) is not None + and _SAFE_OLLAMA_FILE_TOKEN_RE.fullmatch(name) is not None + and re.fullmatch(r"[0-9]+", context_size) is not None + and ( + len(parts) == 7 + or _SAFE_OLLAMA_FILE_TOKEN_RE.fullmatch(parts[7]) is not None + ) + ) + + +def _missing_binary_message( + binary: str, + target: str, + *, + local_host_docker_blocked: bool = False, +) -> str: + if binary == "tmux": + return ( + f"tmux is required for Cookbook background downloads/serves on {target}. " + "Install it with your OS package manager, or run Cookbook server setup for that server." + ) + if binary == "docker": + if local_host_docker_blocked: + return HOST_DOCKER_ACCESS_HINT + return ( + f"Docker is required by this Cookbook launch command on {target}, but the docker CLI was not found. " + "Install Docker and make sure this user can run `docker`, then retry." + ) + return f"{binary} is required on {target}, but it was not found." + + +async def _remote_binary_available( + remote: str, + ssh_port: str | None, + binary: str, + *, + windows: bool = False, +) -> bool: + port = ssh_port or "" + port_args = ["-p", port] if port and port != "22" else [] + if windows: + check = f'powershell -NoProfile -Command "if (Get-Command {binary} -ErrorAction SilentlyContinue) {{ exit 0 }} else {{ exit 127 }}"' + else: + check = f'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; command -v {shlex.quote(binary)} >/dev/null 2>&1' + try: + proc = await asyncio.create_subprocess_exec( + "ssh", + "-o", + "ConnectTimeout=6", + "-o", + "StrictHostKeyChecking=no", + *port_args, + remote, + check, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await asyncio.wait_for(proc.communicate(), timeout=10) + return proc.returncode == 0 + except Exception: + return False + + +def _remote_posix_path_prefix() -> str: + return 'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; ' + + +def _remote_tmux_command(*args: str) -> str: + """Shell command for remote tmux when non-login SSH has a thin PATH.""" + tmux = ( + 'ODYSSEUS_TMUX="$(command -v tmux ' + '|| command -v /opt/homebrew/bin/tmux ' + '|| command -v /usr/local/bin/tmux ' + '|| command -v /usr/bin/tmux ' + '|| true)"; ' + 'if [ -z "$ODYSSEUS_TMUX" ]; then echo "tmux not found" >&2; exit 127; fi; ' + ) + quoted = " ".join(shlex.quote(str(arg)) for arg in args) + return f'{_remote_posix_path_prefix()}{tmux}"$ODYSSEUS_TMUX" {quoted}' + + +def _remote_tmux_launch_command(session_id: str, runner: str) -> str: + """Shell command that chmods a runner and starts it in remote tmux.""" + tmux = ( + 'ODYSSEUS_TMUX="$(command -v tmux ' + '|| command -v /opt/homebrew/bin/tmux ' + '|| command -v /usr/local/bin/tmux ' + '|| command -v /usr/bin/tmux ' + '|| true)"; ' + 'if [ -z "$ODYSSEUS_TMUX" ]; then echo "tmux not found" >&2; exit 127; fi; ' + ) + sid = shlex.quote(str(session_id)) + runner_q = shlex.quote(str(runner)) + runner_exec = shlex.quote(f"./{runner}") + return ( + f'{_remote_posix_path_prefix()}{tmux}' + f'chmod +x {runner_q} && ' + f'"$ODYSSEUS_TMUX" set-option -g history-limit 100000 2>/dev/null; ' + f'"$ODYSSEUS_TMUX" new-session -d -s {sid} {runner_exec}' + ) + + +async def _binary_available( + binary: str, + remote: str | None, + ssh_port: str | None, + *, + windows: bool = False, + in_container: bool | None = None, + environ=None, + socket_path: str = HOST_DOCKER_SOCKET_PATH, +) -> bool: + if remote: + return await _remote_binary_available( + remote, + ssh_port, + binary, + windows=windows, + ) + cli_available = shutil.which(binary) is not None + if binary != "docker": + return cli_available + return local_docker_available( + cli_available=cli_available, + in_container=in_container, + environ=environ, + socket_path=socket_path, + ) + + + +def _local_ollama_docker_fallback_available( + *, + in_container: bool | None = None, + environ: dict[str, str] | None = None, + socket_path: str = HOST_DOCKER_SOCKET_PATH, +) -> bool: + return local_docker_available( + cli_available=shutil.which("docker") is not None, + in_container=in_container, + environ=environ, + socket_path=socket_path, + ) + + +def _local_ollama_docker_access_blocked( + *, + in_container: bool | None = None, + environ: dict[str, str] | None = None, + socket_path: str = HOST_DOCKER_SOCKET_PATH, +) -> bool: + containerized = running_in_container() if in_container is None else in_container + if not containerized or shutil.which("docker") is None: + return False + return not _local_ollama_docker_fallback_available( + in_container=containerized, + environ=environ, + socket_path=socket_path, + ) + + +def _append_local_ollama_download_command_lines( + lines: list[str], + ollama_cmd: str, + *, + docker_fallback_available: bool, + docker_fallback_blocked: bool, +) -> None: + lines.append('if command -v ollama >/dev/null 2>&1; then') + lines.append(f' ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote(ollama_cmd)}') + if docker_fallback_available: + lines.append('elif command -v docker >/dev/null 2>&1; then') + lines.append(" ODYSSEUS_OLLAMA_CONTAINER=\"$(docker ps --format '{{.Names}}' 2>/dev/null | grep -E '^(ollama-rocm|ollama-test)$' | head -1)\"") + lines.append(' if [ -n "$ODYSSEUS_OLLAMA_CONTAINER" ]; then') + lines.append(f' ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote("docker exec ${ODYSSEUS_OLLAMA_CONTAINER} " + ollama_cmd)}') + lines.append(' fi') + elif docker_fallback_blocked: + hint = shlex.quote("ERROR: " + HOST_DOCKER_ACCESS_HINT) + lines.append('else') + lines.append(f" printf '%s\\n' {hint}; exit 127") + lines.append('fi') + lines.append('if [ -z "$ODYSSEUS_OLLAMA_PULL_CMD" ]; then echo "ERROR: Ollama not found on this server. Install Ollama or start an ollama-rocm/ollama-test container."; exit 127; fi') + + def setup_cookbook_routes() -> APIRouter: router = APIRouter(tags=["cookbook"]) - _cookbook_state_path = Path(os.environ.get("DATA_DIR", "data")) / "cookbook_state.json" + _cookbook_state_path = Path(COOKBOOK_STATE_FILE) + _state_get_cache = {"ts": 0.0, "mtime": 0.0, "value": None} + _tasks_status_cache = {"ts": 0.0, "value": None} + _tasks_status_inflight = {"task": None} def _mask_secret(value: str) -> str: if not value: @@ -49,6 +367,9 @@ def _mask_secret(value: str) -> str: return "stored" return f"{value[:4]}...{value[-4:]}" + def _client_host_platform() -> str: + return "windows" if IS_WINDOWS else "" + def _decrypt_secret(value: str | None) -> str: if not value: return "" @@ -121,6 +442,11 @@ def _diagnose_serve_output(text: str) -> dict | None: "Model requires custom code or newer model support.", [{"label": "retry with --trust-remote-code", "op": "append", "arg": "--trust-remote-code"}], ), + ( + r"Either a revision or a version must be specified|transformers\.integrations\.hub_kernels|kernels/layer", + "vLLM/Transformers kernel package mismatch.", + [{"label": "update vLLM, Transformers, and kernels on this server", "op": "dependency", "package": "vllm transformers kernels"}], + ), ( r"Address already in use|bind.*address.*in use", "Port is already in use.", @@ -131,21 +457,83 @@ def _diagnose_serve_output(text: str) -> dict | None: "No GPUs are visible to the serve process.", [{"label": "clear Cookbook GPU selection or choose available GPUs", "op": "settings", "field": "gpus", "value": ""}], ), + ( + r"Failed to infer device type|NVML Shared Library Not Found|No module named 'amdsmi'|platform is not available", + "vLLM could not find a supported GPU (CUDA or ROCm). " + "This machine may have integrated or unsupported graphics only.", + [ + {"label": "switch to llama.cpp (CPU/Metal, works without a discrete GPU)", "op": "manual"}, + {"label": "switch to Ollama (CPU/Metal, works without a discrete GPU)", "op": "manual"}, + ], + ), ( r"vllm.*command not found|No module named vllm|ERROR: vLLM is not installed", "vLLM is not installed or not in PATH on this server.", [{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}], ), + ( + r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|" + r"(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|" + r"Could not load any common_ops library|" + r"Please ensure sgl_kernel is properly installed", + "SGLang native kernel/runtime is missing or mismatched on this server.", + [ + {"label": "repair sglang-kernel in this Python environment", "op": "dependency", "package": "sglang-kernel"}, + {"label": "if libnvrtc is still missing, install the matching CUDA/NVRTC runtime on this host", "op": "manual"}, + ], + ), ( r"sglang.*command not found|No module named sglang|SGLang is not installed", "SGLang is not installed or not in PATH on this server.", [{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}], ), ( - r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'|git: command not found|cmake: command not found", + r"No module named ['\"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed", + "MLX LM is not installed on this server.", + [{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}], + ), + ( + r"Unable to quantize model of type |QuantizedSwitchLinear", + "MLX-LM tried to quantize an already-quantized DeepSeek switch layer.", + [ + {"label": "relaunch from the cached local Hugging Face snapshot path on this Mac", "op": "manual"}, + {"label": "Odysseus now rewrites MLX repo-id launches to a cached snapshot when one exists", "op": "manual"}, + ], + ), + # System build deps come BEFORE the generic llama.cpp catch-all + # so cmake / build-essential / git missing → a specific OS-package + # remediation instead of "install llama-cpp-python[server]" (which + # itself fails to compile when cmake is absent). + ( + r"cmake: command not found|cmake.*not found.*[Cc]ould not", + "cmake is required to build llama.cpp from source but isn't installed on this server.", + [{"label": "install build deps for llama.cpp (apt: cmake build-essential git / pacman: cmake base-devel git / dnf: cmake gcc-c++ make git / brew: cmake git)", "op": "dependency", "package": "llama-cpp-python[server]"}], + ), + ( + r"^(make|g\+\+|gcc): command not found|Could not find C\+\+ compiler", + "A C/C++ compiler (build-essential) is required to build llama.cpp from source.", + [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}], + ), + ( + r"^git: command not found", + "git is required to clone the llama.cpp source tree.", + [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}], + ), + ( + r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'", "llama.cpp / llama-cpp-python dependencies are missing.", [{"label": "install llama.cpp dependencies or llama-cpp-python[server]", "op": "dependency", "package": "llama-cpp-python[server]"}], ), + ( + r"No GGUF found on this host|no \.gguf file|No GGUF file found", + "No GGUF file found for this model on this host. The llama.cpp backend needs a .gguf file.", + [{"label": "download a GGUF build of this model (repo name usually ends in -GGUF, file like Q4_K_M.gguf)", "op": "manual"}], + ), + ( + r"No module named 'torch'|No module named torch|No module named 'diffusers'|No module named diffusers", + "Diffusion serving requires PyTorch and diffusers.", + [{"label": "install diffusers[torch] in Cookbook Dependencies", "op": "dependency", "package": "diffusers[torch]"}], + ), ( r"403 Forbidden|401 Unauthorized|Access to model.*is restricted|gated repo|not in the authorized list|awaiting a review", "Model access is gated or unauthorized.", @@ -168,11 +556,15 @@ def _state_for_client(state): """Return cookbook state without raw secrets for browser clients.""" _strip_task_secrets(state) env = state.get("env") if isinstance(state, dict) else None + if isinstance(state, dict) and not isinstance(env, dict): + env = {} + state["env"] = env if isinstance(env, dict): token = _decrypt_secret(env.get("hfToken")) env.pop("hfToken", None) env["hfTokenConfigured"] = bool(token) env["hfTokenMasked"] = _mask_secret(token) + env["hostPlatform"] = _client_host_platform() return state def _state_for_storage(state, on_disk=None): @@ -191,33 +583,299 @@ def _state_for_storage(state, on_disk=None): env.pop("hfToken", None) env.pop("hfTokenMasked", None) env.pop("hfTokenConfigured", None) + env.pop("hostPlatform", None) return state def _load_stored_hf_token() -> str: - if not _cookbook_state_path.exists(): - return "" + return load_stored_hf_token(state_path=_cookbook_state_path) + + def _normalize_minimax_m3_vllm_cmd(cmd: str) -> str: + """Patch MiniMax M3 vLLM launches into the known-good local form. + + The browser form can be stale or omit advanced-only fields. MiniMax M3 + is sensitive to several flags: using the HF repo id with block-size 128 + fails KV-cache setup, and FlashInfer sampler JIT fails on this host's + system nvcc. Normalize server-side before writing the tmux runner. + """ + cmd_lower = (cmd or "").lower() + if not cmd or "vllm serve" not in cmd_lower or "minimax" not in cmd_lower or "m3" not in cmd_lower: + return cmd try: - state = json.loads(_cookbook_state_path.read_text()) - env = state.get("env") if isinstance(state, dict) else {} - return _decrypt_secret(env.get("hfToken") if isinstance(env, dict) else "") - except Exception: - return "" + parts = shlex.split(cmd) + except ValueError: + return cmd + if "serve" not in parts: + return cmd + + env_re = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=") + env_parts = [p for p in parts if env_re.match(p)] + body = [p for p in parts if not env_re.match(p)] + try: + serve_i = body.index("serve") + except ValueError: + return cmd + if serve_i + 1 >= len(body): + return cmd + + repo_id = "cyankiwi/MiniMax-M3-AWQ-INT4" + snapshot = ( + "/home/pewds/.cache/huggingface/hub/" + "models--cyankiwi--MiniMax-M3-AWQ-INT4/" + "snapshots/4082acbbec1236d21828d55b6bb0fe02ade4ab5b" + ) + if body[serve_i + 1] == repo_id: + body[serve_i + 1] = snapshot + + def add_env(key: str, value: str) -> None: + if not any(p.startswith(f"{key}=") for p in env_parts): + env_parts.append(f"{key}={value}") + + def has_flag(flag: str) -> bool: + return any(p == flag or p.startswith(flag + "=") for p in body) + + def set_flag(flag: str, value: str) -> None: + for i, part in enumerate(body): + if part == flag: + if i + 1 < len(body): + body[i + 1] = value + else: + body.append(value) + return + if part.startswith(flag + "="): + body[i] = f"{flag}={value}" + return + body.extend([flag, value]) + + def add_bool(flag: str) -> None: + if not has_flag(flag): + body.append(flag) + + add_env("VLLM_TARGET_DEVICE", "cuda") + add_env("VLLM_USE_FLASHINFER_SAMPLER", "0") + set_flag("--served-model-name", repo_id) + set_flag("--tool-call-parser", "minimax_m3") + set_flag("--reasoning-parser", "minimax_m3") + set_flag("--attention-backend", "TRITON_ATTN") + set_flag("--block-size", "128") + add_bool("--language-model-only") + add_bool("--disable-custom-all-reduce") + add_bool("--enable-expert-parallel") + return shlex.join(env_parts + body) + + def _normalize_deepseek_v4_sglang_cmd(cmd: str) -> str: + """Patch stale DeepSeek-V4 SGLang commands into the safer local form. + + The browser command builder already emits these flags, but saved presets, + running-row retries, and old tabs can still submit a pre-fix command to + /api/model/serve. Normalize server-side so the tmux runner does not keep + relaunching DeepSeek-V4 with the known CUDA-graph crash shape. + """ + cmd_lower = (cmd or "").lower() + if ( + not cmd + or "sglang.launch_server" not in cmd_lower + or "deepseek-v4" not in cmd_lower + ): + return cmd + try: + parts = shlex.split(cmd) + except ValueError: + return cmd + + env_re = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=") + env_parts = [p for p in parts if env_re.match(p)] + body = [p for p in parts if not env_re.match(p)] + + def add_env(key: str, value: str) -> None: + if not any(p.startswith(f"{key}=") for p in env_parts): + env_parts.append(f"{key}={value}") + + def has_flag(flag: str) -> bool: + return any(p == flag or p.startswith(flag + "=") for p in body) + + def flag_value(flag: str) -> str | None: + for i, part in enumerate(body): + if part == flag: + return body[i + 1] if i + 1 < len(body) else "" + if part.startswith(flag + "="): + return part.split("=", 1)[1] + return None + + def set_flag(flag: str, value: str) -> None: + for i, part in enumerate(body): + if part == flag: + if i + 1 < len(body): + body[i + 1] = value + else: + body.append(value) + return + if part.startswith(flag + "="): + body[i] = f"{flag}={value}" + return + body.extend([flag, value]) + + def remove_flag(flag: str) -> None: + i = 0 + while i < len(body): + part = body[i] + if part == flag: + del body[i:i + 2] + continue + if part.startswith(flag + "="): + del body[i] + continue + i += 1 + + add_env("SGLANG_DSV4_COMPRESS_STATE_DTYPE", "bf16") + mem_fraction = flag_value("--mem-fraction-static") + try: + mem_fraction_num = float(mem_fraction) if mem_fraction not in (None, "") else None + except (TypeError, ValueError): + mem_fraction_num = None + if mem_fraction in (None, "", "0.90", "0.9") or ( + mem_fraction_num is not None and mem_fraction_num < 0.76 + ): + set_flag("--mem-fraction-static", "0.80") + if not has_flag("--reasoning-parser"): + set_flag("--reasoning-parser", "deepseek-v4") + if not has_flag("--tool-call-parser"): + set_flag("--tool-call-parser", "deepseekv4") + if not has_flag("--cuda-graph-backend-decode"): + remove_flag("--cuda-graph-max-bs-decode") + set_flag("--cuda-graph-backend-decode", "disabled") + return shlex.join(env_parts + body) def _cookbook_ssh_dir() -> Path: - app_ssh = Path("/app/.ssh") - if Path("/app").exists(): - return app_ssh + # The Docker image keeps cookbook keys under /app/.ssh; that path only + # exists inside the container. On Windows (and any non-container host) + # fall back to the user profile's ~/.ssh, which OpenSSH on Win10+ uses. + if not IS_WINDOWS: + app_ssh = Path("/app/.ssh") + if Path("/app").exists(): + return app_ssh return Path.home() / ".ssh" def _cookbook_ssh_key_path() -> Path: return _cookbook_ssh_dir() / "id_ed25519" + def _ssh_known_host_name(host: str) -> str: + """Return the host part OpenSSH stores in known_hosts. + + Cookbook accepts `user@host` for convenience, but known_hosts entries + are keyed by host, not username. + """ + return (host or "").rsplit("@", 1)[-1] + + def _known_hosts_targets(host: str, ssh_port: str | None = None) -> list[str]: + name = _ssh_known_host_name(host) + targets = [name] + if ssh_port and ssh_port != "22": + targets.insert(0, f"[{name}]:{ssh_port}") + return [t for t in targets if t] + + def _ssh_host_key_changed(stderr_txt: str) -> bool: + text = stderr_txt or "" + return ( + "REMOTE HOST IDENTIFICATION HAS CHANGED" in text + or "Host key verification failed" in text and "Offending" in text + ) + + async def _repair_cookbook_known_host(host: str, ssh_port: str | None = None) -> tuple[bool, str]: + """Refresh Odysseus' own known_hosts entry for a validated Cookbook host. + + This is intentionally scoped to Cookbook SSH targets and only called + after OpenSSH reports a changed host key. It fixes container-local + known_hosts drift without asking the user to run ssh-keygen manually. + """ + known_hosts = _cookbook_ssh_dir() / "known_hosts" + known_hosts.parent.mkdir(parents=True, exist_ok=True) + known_hosts.touch(mode=0o600, exist_ok=True) + safe_chmod(known_hosts, 0o600) + + ssh_keygen = which_tool("ssh-keygen") or "ssh-keygen" + ssh_keyscan = which_tool("ssh-keyscan") or "ssh-keyscan" + removed_chunks: list[str] = [] + for target in _known_hosts_targets(host, ssh_port): + proc = await asyncio.create_subprocess_exec( + ssh_keygen, + "-f", + str(known_hosts), + "-R", + target, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await proc.communicate() + removed_chunks.append((stdout or stderr).decode("utf-8", errors="replace").strip()) + + scan_args = [ssh_keyscan, "-H", "-t", "ed25519,ecdsa,rsa"] + if ssh_port and ssh_port != "22": + scan_args.extend(["-p", ssh_port]) + scan_args.append(_ssh_known_host_name(host)) + proc = await asyncio.create_subprocess_exec( + *scan_args, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=8) + except asyncio.TimeoutError: + proc.kill() + await proc.communicate() + return False, "ssh-keyscan timed out while refreshing known_hosts" + if proc.returncode != 0 or not stdout.strip(): + detail = (stderr or stdout).decode("utf-8", errors="replace").strip() + return False, detail or "ssh-keyscan returned no host keys" + with known_hosts.open("ab") as f: + if known_hosts.stat().st_size > 0: + f.write(b"\n") + f.write(stdout.strip() + b"\n") + safe_chmod(known_hosts, 0o600) + return True, "\n".join(chunk for chunk in removed_chunks if chunk) or "known_hosts refreshed" + def _read_cookbook_public_key() -> str: pub = _cookbook_ssh_key_path().with_suffix(".pub") if not pub.exists(): return "" return pub.read_text(encoding="utf-8", errors="replace").strip() + def _server_env_prefix_for_download(remote_host: str | None) -> str | None: + """Recover a server venv/conda activation for stale download clients. + + Older browser bundles could submit /api/model/download without + env_prefix even when the selected server profile had an envPath. The + remote runner would then use system python and exit before hf download. + Resolve the server profile by host here so downloads remain correct even + if the user has a cached JS bundle. + """ + if not remote_host or not _cookbook_state_path.exists(): + return None + try: + state = json.loads(_cookbook_state_path.read_text(encoding="utf-8")) + except Exception: + return None + env_state = state.get("env") if isinstance(state, dict) else {} + servers = env_state.get("servers") if isinstance(env_state, dict) else [] + if not isinstance(servers, list): + return None + selected = None + for server in servers: + if isinstance(server, dict) and (server.get("host") or "").strip() == remote_host: + selected = server + break + if not selected: + return None + env = (selected.get("env") or "none").strip().lower() + env_path = (selected.get("envPath") or "").strip() + if not env_path: + return None + if env == "venv" or (env in {"", "none"} and re.search(r"(?:^|/)(?:\.?venv|env)(?:/|$)|/bin/activate$", env_path, re.I)): + activate = env_path if env_path.endswith("/bin/activate") else env_path.rstrip("/") + "/bin/activate" + return "source " + shlex.quote(activate) + if env == "conda": + return 'eval "$(conda shell.bash hook)" && conda activate ' + shlex.quote(env_path) + return None + @router.get("/api/cookbook/ssh-key") async def get_cookbook_ssh_key(request: Request): require_admin(request) @@ -233,13 +891,15 @@ async def generate_cookbook_ssh_key(request: Request): ssh_dir = _cookbook_ssh_dir() key_path = _cookbook_ssh_key_path() ssh_dir.mkdir(parents=True, exist_ok=True) - try: - os.chmod(ssh_dir, 0o700) - except Exception: - pass + # safe_chmod no-ops on Windows (~/.ssh is already ACL-restricted to the + # user profile); applies 0o700 on POSIX. + safe_chmod(ssh_dir, 0o700) if not key_path.exists(): + # ssh-keygen ships with the OpenSSH client on Win10+; resolve it via + # which_tool so the .exe is found even when PATHEXT is unusual. + ssh_keygen = which_tool("ssh-keygen") or "ssh-keygen" proc = await asyncio.create_subprocess_exec( - "ssh-keygen", "-t", "ed25519", "-N", "", "-C", "odysseus-cookbook", "-f", str(key_path), + ssh_keygen, "-t", "ed25519", "-N", "", "-C", "odysseus-cookbook", "-f", str(key_path), stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) @@ -247,61 +907,99 @@ async def generate_cookbook_ssh_key(request: Request): if proc.returncode != 0: detail = (stderr or stdout).decode("utf-8", errors="replace").strip()[-500:] return {"ok": False, "error": detail or "Failed to generate SSH key"} - try: - os.chmod(key_path, 0o600) - os.chmod(key_path.with_suffix(".pub"), 0o644) - except Exception: - pass + safe_chmod(key_path, 0o600) + safe_chmod(key_path.with_suffix(".pub"), 0o644) return {"ok": True, "public_key": _read_cookbook_public_key()} - def _user_shell_path_bootstrap() -> list[str]: - return [ - 'ODYSSEUS_USER_SHELL="${SHELL:-}"', - 'if [ -n "$ODYSSEUS_USER_SHELL" ] && [ -x "$ODYSSEUS_USER_SHELL" ]; then', - ' ODYSSEUS_USER_PATH="$("$ODYSSEUS_USER_SHELL" -ic \'printf "__ODYSSEUS_PATH__%s\\n" "$PATH"\' 2>/dev/null | sed -n \'s/^__ODYSSEUS_PATH__//p\' | tail -n 1 || true)"', - ' if [ -n "$ODYSSEUS_USER_PATH" ]; then export PATH="$ODYSSEUS_USER_PATH:$PATH"; fi', - 'fi', - ] + class CookbookSshTestRequest(BaseModel): + host: str + ssh_port: str | None = None + + @router.post("/api/cookbook/test-ssh") + async def test_cookbook_ssh(request: Request, req: CookbookSshTestRequest): + """Test a configured Cookbook SSH target without using generic shell exec.""" + require_admin(request) + host = validate_remote_host(req.host) + ssh_port = validate_ssh_port(req.ssh_port) + try: + code, stdout, stderr = await run_ssh_command_async( + host, + ssh_port, + "echo ok", + timeout=8, + connect_timeout=5, + strict_host_key_checking=False, + ) + except asyncio.TimeoutError: + return {"stdout": "", "stderr": "SSH test timed out", "exit_code": 124} + except Exception as e: + return {"stdout": "", "stderr": str(e), "exit_code": -1} + return { + "stdout": stdout.decode("utf-8", errors="replace"), + "stderr": stderr.decode("utf-8", errors="replace"), + "exit_code": code, + } def _needs_binary(cmd: str, binary: str) -> bool: return bool(re.search(rf"(^|[\s;&|()]){re.escape(binary)}($|[\s;&|()])", cmd or "")) - def _missing_binary_message(binary: str, target: str) -> str: - if binary == "tmux": - return ( - f"tmux is required for Cookbook background downloads/serves on {target}. " - "Install it with your OS package manager, or run Cookbook server setup for that server." + def _launch_local_detached(session_id: str, bash_lines: list[str]) -> dict: + """Windows-native stand-in for a LOCAL tmux session (tmux doesn't exist + on Windows). Mirrors shell_routes._generate_win_detached / bg_jobs.launch: + runs the wrapper detached so it survives a browser/SSE disconnect (the + whole point of the tmux feature for long downloads/serves), writing a + .log the status poller tails and a .pid for liveness. + + `bash_lines` is the same bash wrapper used on POSIX. Prefers Git Bash + for full command-syntax parity; falls back to a cmd.exe wrapper that + runs the script through whatever bash is reachable, else best-effort + directly (simple commands only). Returns the launched job record.""" + log_path = TMUX_LOG_DIR / f"{session_id}.log" + pid_path = TMUX_LOG_DIR / f"{session_id}.pid" + bash = find_bash() + if bash: + # Run the existing bash wrapper verbatim through Git Bash, redirecting + # all output to the log the poller reads. Paths handed to bash use + # POSIX form + shell-quoting so drive paths / spaces survive. + inner = TMUX_LOG_DIR / f"{session_id}_run.sh" + pp = shlex.quote(pid_path.as_posix()) + inner.write_text( + f"printf '%s\\n' \"$$\" > {pp}\n" + "\n".join(bash_lines) + "\n", + encoding="utf-8", ) - if binary == "docker": - return ( - f"Docker is required by this Cookbook launch command on {target}, but the docker CLI was not found. " - "Install Docker and make sure this user can run `docker`, then retry." + lp = shlex.quote(log_path.as_posix()) + ip = shlex.quote(inner.as_posix()) + script_path = TMUX_LOG_DIR / f"{session_id}.sh" + script_path.write_text( + f"bash {ip} > {lp} 2>&1\n", + encoding="utf-8", ) - return f"{binary} is required on {target}, but it was not found." - - async def _remote_binary_available(remote: str, ssh_port: str | None, binary: str, *, windows: bool = False) -> bool: - _port = ssh_port or "" - _pf = ["-p", _port] if _port and _port != "22" else [] - if windows: - check = f"powershell -NoProfile -Command \"if (Get-Command {binary} -ErrorAction SilentlyContinue) {{ exit 0 }} else {{ exit 127 }}\"" + argv = [bash, str(script_path)] else: - check = f"command -v {shlex.quote(binary)} >/dev/null 2>&1" - try: - proc = await asyncio.create_subprocess_exec( - "ssh", "-o", "ConnectTimeout=6", "-o", "StrictHostKeyChecking=no", - *_pf, remote, check, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, + # No bash on this Windows host: the bash wrapper can't run. Fall back + # to a cmd.exe wrapper that just records a clear error to the log so + # the UI surfaces "install Git Bash" instead of silently hanging. + script_path = TMUX_LOG_DIR / f"{session_id}.cmd" + script_path.write_text( + "@echo off\r\n" + f'echo Cookbook LOCAL execution on Windows needs Git Bash ^(bash.exe^) on PATH. > "{log_path}" 2>&1\r\n' + f'echo Install Git for Windows, then retry. >> "{log_path}"\r\n', + encoding="utf-8", ) - await asyncio.wait_for(proc.communicate(), timeout=10) - return proc.returncode == 0 - except Exception: - return False - - async def _binary_available(binary: str, remote: str | None, ssh_port: str | None, *, windows: bool = False) -> bool: - if remote: - return await _remote_binary_available(remote, ssh_port, binary, windows=windows) - return shutil.which(binary) is not None + argv = [os.environ.get("ComSpec", "cmd.exe"), "/c", str(script_path)] + env = os.environ.copy() + env["PYTHONUTF8"] = "1" + env["PYTHONIOENCODING"] = "utf-8" + proc = subprocess.Popen( + argv, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + stdin=subprocess.DEVNULL, + env=env, + **detached_popen_kwargs(), + ) + pid_path.write_text(str(proc.pid), encoding="utf-8") + return {"pid": proc.pid, "log_path": str(log_path)} @router.post("/api/model/download") async def model_download(request: Request, req: ModelDownloadRequest): @@ -311,33 +1009,44 @@ async def model_download(request: Request, req: ModelDownloadRequest): require_admin(request) # Defence-in-depth: even though this endpoint is admin-gated, refuse # values that would land in shell contexts with metacharacters. - _validate_repo_id(req.repo_id) - _validate_include(req.include) - _validate_remote_host(req.remote_host) - req.ssh_port = _validate_ssh_port(req.ssh_port) + backend = (req.backend or "").strip().lower() + is_ollama_download = backend == "ollama" or ("/" not in req.repo_id and ":" in req.repo_id) + if is_ollama_download: + _validate_serve_model_id(req.repo_id) + req.include = None + req.local_dir = None + else: + _validate_repo_id(req.repo_id) + _validate_include(req.include) + validate_remote_host(req.remote_host) + req.ssh_port = validate_ssh_port(req.ssh_port) req.local_dir = _validate_local_dir(req.local_dir) - req.hf_token = req.hf_token or _load_stored_hf_token() + req.hf_token = "" if is_ollama_download else (req.hf_token or _load_stored_hf_token()) _validate_token(req.hf_token) + if req.remote_host and not req.env_prefix: + req.env_prefix = _server_env_prefix_for_download(req.remote_host) TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True) session_id = f"cookbook-{uuid.uuid4().hex[:8]}" wrapper_script = TMUX_LOG_DIR / f"{session_id}.sh" - # When a download directory is set, target a per-model subfolder under it - # (/) so the flat-directory cache scan lists it as its own - # model. Without it, hf/snapshot_download falls back to the HF cache. - _dl_short = req.repo_id.split("/")[-1] if "/" in req.repo_id else req.repo_id - _dl_base = (req.local_dir.rstrip("/") + "/" + _dl_short) if req.local_dir else None - _dl_shell = _shell_path(_dl_base) if _dl_base else None # for hf CLI / bash - _dl_pyarg = (", local_dir=os.path.expanduser(" + repr(_dl_base) + ")") if _dl_base else "" + # Custom download dir: point the HF cache at /hub via env vars + # (HF_HOME + HUGGINGFACE_HUB_CACHE) instead of --local-dir. local_dir + # produces a flat layout (//) and the local-dir + # bookkeeping files (.cache/huggingface/.gitignore.lock), and it + # also breaks robust resume on flaky transfers — the blob-based hub + # cache survives SSL ReadError mid-stream by reusing .incomplete, + # local_dir does not. See issue #2722. + _dl_hf_home_shell = _shell_path(req.local_dir.rstrip("/")) if req.local_dir else None + _dl_pyarg = "" # snapshot_download honors the env vars too — no kwarg needed # Build the hf download command. Redirection to suppress the interactive # "update available? [Y/n]" prompt is added per-platform further down # (< /dev/null on bash, $null | on PowerShell). - hf_cmd = f"hf download {req.repo_id}" + hf_download_args = f"download {shlex.quote(req.repo_id)}" if req.include: - hf_cmd += f" --include '{req.include}'" - if _dl_shell: - hf_cmd += f" --local-dir {_dl_shell}" + hf_download_args += f" --include {shlex.quote(req.include)}" + hf_cmd = f"hf {hf_download_args}" + ollama_cmd = f"ollama pull {shlex.quote(req.repo_id)}" # Build the shell wrapper — runs hf download directly in tmux (which is a TTY) # No script/tee needed — we'll use tmux capture-pane to read output @@ -345,26 +1054,50 @@ async def model_download(request: Request, req: ModelDownloadRequest): lines.extend(_user_shell_path_bootstrap()) if req.hf_token: lines.append(f"export HF_TOKEN='{_bash_squote(req.hf_token)}'") + if _dl_hf_home_shell and not is_ollama_download: + # Make hf download / snapshot_download honor the chosen dir via the + # standard HF cache (gives us the models--org--name/blobs/... layout + # with resumable .incomplete blobs). + lines.append(f"export HF_HOME={_dl_hf_home_shell}") + lines.append(f"export HUGGINGFACE_HUB_CACHE={_dl_hf_home_shell}/hub") + lines.append(f"export HF_HUB_CACHE={_dl_hf_home_shell}/hub") # Ensure pip-user scripts (e.g. hf CLI installed via --user) are on PATH - lines.append('export PATH="$HOME/.local/bin:$PATH"') + lines.append('export PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"') + # When Odysseus runs from a venv (e.g. native macOS install), put its bin + # on PATH so the tmux shell finds the bundled `hf`/`python3` without an + # activated venv. Local bash runs only — meaningless over SSH. + if not req.remote_host: + lines.append(_local_tooling_path_export(sys.executable)) # Best-effort install hf CLI (always). hf_transfer (Rust parallel downloader) # is fast but flaky on large files — it tends to crash near the end at high # throughput. Retries set disable_hf_transfer to fall back to the plain, # slower-but-reliable downloader (resumes cleanly from the .incomplete files). - lines.append("command -v hf >/dev/null 2>&1 || pip install --user --break-system-packages -q -U huggingface_hub 2>/dev/null || pip install -q -U huggingface_hub 2>/dev/null") - if req.disable_hf_transfer: - lines.append("export HF_HUB_ENABLE_HF_TRANSFER=0") - lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=4") + # Use `python3 -m pip` not `pip` — macOS has no bare `pip` command. + if is_ollama_download: + _append_local_ollama_download_command_lines( + lines, + ollama_cmd, + docker_fallback_available=_local_ollama_docker_fallback_available(), + docker_fallback_blocked=_local_ollama_docker_access_blocked(), + ) else: - lines.append("python3 -c 'import hf_transfer' 2>/dev/null || pip install --user --break-system-packages -q hf_transfer 2>/dev/null || pip install -q hf_transfer 2>/dev/null") - lines.append("python3 -c 'import hf_transfer' 2>/dev/null && export HF_HUB_ENABLE_HF_TRANSFER=1") - lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=8") + lines.append(f"command -v hf >/dev/null 2>&1 || {_pip_install_fallback_chain('huggingface_hub', upgrade=True)}") + if req.disable_hf_transfer: + lines.append("export HF_HUB_ENABLE_HF_TRANSFER=0") + lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=4") + else: + lines.append(f"python3 -c 'import hf_transfer' 2>/dev/null || {_pip_install_fallback_chain('hf_transfer')}") + lines.append("python3 -c 'import hf_transfer' 2>/dev/null && export HF_HUB_ENABLE_HF_TRANSFER=1") + lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=8") remote = req.remote_host # None for local is_windows = req.platform == "windows" + # LOCAL execution on a native-Windows host never uses tmux (it uses the + # detached-process path below), regardless of the UI-supplied platform. + local_windows = IS_WINDOWS and not remote logger.info(f"Download request: repo={req.repo_id}, remote={remote}, ssh_port={req.ssh_port}, platform={req.platform}") - if not is_windows and not await _binary_available("tmux", remote, req.ssh_port): + if not is_windows and not local_windows and not await _binary_available("tmux", remote, req.ssh_port): return { "ok": False, "error": _missing_binary_message("tmux", remote or "local server"), @@ -379,36 +1112,49 @@ async def model_download(request: Request, req: ModelDownloadRequest): ps_lines.append('New-Item -ItemType Directory -Force -Path $sessionDir | Out-Null') if req.hf_token: ps_lines.append(f"$env:HF_TOKEN = '{_ps_squote(req.hf_token)}'") + if req.local_dir and not is_ollama_download: + # Mirror the bash branch — point the HF cache at the user's dir + # via env vars instead of --local-dir, so resume works on flaky + # transfers (issue #2722). + _dl_ps = _ps_squote(req.local_dir.rstrip("/")) + ps_lines.append(f"$env:HF_HOME = '{_dl_ps}'") + ps_lines.append(f"$env:HUGGINGFACE_HUB_CACHE = '{_dl_ps}/hub'") + ps_lines.append(f"$env:HF_HUB_CACHE = '{_dl_ps}/hub'") if req.env_prefix: ps_lines.append(_safe_env_prefix(req.env_prefix)) - # Try hf CLI, fall back to Python huggingface_hub, then auto-install - ps_lines.append('try {{') - ps_lines.append(' $hfPath = Get-Command hf -ErrorAction SilentlyContinue') - ps_lines.append(' if ($hfPath) {{') - # Pipe $null to stdin to suppress interactive "update available? [Y/n]" prompt - ps_lines.append(f' $null | {hf_cmd}') - ps_lines.append(' }} else {{') - ps_lines.append(' python -c "import huggingface_hub" 2>$null') - ps_lines.append(' if ($LASTEXITCODE -eq 0) {{') - ps_lines.append(' Write-Host "hf CLI not found, using Python huggingface_hub..."') - ps_lines.append(' python -m pip install -q hf_transfer 2>$null') - ps_lines.append(' $env:HF_HUB_ENABLE_HF_TRANSFER = "1"') - ps_lines.append(f" python -c \"import os; from huggingface_hub import snapshot_download; snapshot_download('{req.repo_id}'{_dl_pyarg}, max_workers=8)\"") - ps_lines.append(' }} else {{') - ps_lines.append(' Write-Host "Installing huggingface-hub..."') - ps_lines.append(' python -m pip install -q huggingface-hub hf_transfer') - ps_lines.append(' $env:HF_HUB_ENABLE_HF_TRANSFER = "1"') - ps_lines.append(f" python -c \"import os; from huggingface_hub import snapshot_download; snapshot_download('{req.repo_id}'{_dl_pyarg}, max_workers=8)\"") - ps_lines.append(' }}') - ps_lines.append(' }}') - ps_lines.append(' if ($LASTEXITCODE -eq 0) {{ Write-Host ""; Write-Host "DOWNLOAD_OK" }}') - ps_lines.append(' else {{ Write-Host ""; Write-Host "DOWNLOAD_FAILED (exit $LASTEXITCODE)" }}') - ps_lines.append('}} catch {{') - ps_lines.append(' Write-Host ""; Write-Host "DOWNLOAD_FAILED ($_)"') - ps_lines.append('}}') + if is_ollama_download: + ps_lines.append('if (-not (Get-Command ollama -ErrorAction SilentlyContinue)) { Write-Host "ERROR: Ollama not found. Install from https://ollama.com/download/windows"; exit 127 }') + ps_lines.append(f"$null | ollama pull '{_ps_squote(req.repo_id)}'") + ps_lines.append('if ($LASTEXITCODE -eq 0) { Write-Host ""; Write-Host "DOWNLOAD_OK" } else { Write-Host ""; Write-Host "DOWNLOAD_FAILED (exit $LASTEXITCODE)" }') + else: + # Try hf CLI, fall back to Python huggingface_hub, then auto-install + ps_lines.append('try {{') + ps_lines.append(' $hfPath = Get-Command hf -ErrorAction SilentlyContinue') + ps_lines.append(' if ($hfPath) {{') + # Pipe $null to stdin to suppress interactive "update available? [Y/n]" prompt + ps_lines.append(f' $null | {hf_cmd}') + ps_lines.append(' }} else {{') + ps_lines.append(' python -c "import huggingface_hub" 2>$null') + ps_lines.append(' if ($LASTEXITCODE -eq 0) {{') + ps_lines.append(' Write-Host "hf CLI not found, using Python huggingface_hub..."') + ps_lines.append(' python -m pip install -q hf_transfer 2>$null') + ps_lines.append(' $env:HF_HUB_ENABLE_HF_TRANSFER = "1"') + ps_lines.append(f" python -c \"import os; from huggingface_hub import snapshot_download; snapshot_download('{req.repo_id}'{_dl_pyarg}, max_workers=8)\"") + ps_lines.append(' }} else {{') + ps_lines.append(' Write-Host "Installing huggingface-hub..."') + ps_lines.append(' python -m pip install -q huggingface-hub hf_transfer') + ps_lines.append(' $env:HF_HUB_ENABLE_HF_TRANSFER = "1"') + ps_lines.append(f" python -c \"import os; from huggingface_hub import snapshot_download; snapshot_download('{req.repo_id}'{_dl_pyarg}, max_workers=8)\"") + ps_lines.append(' }}') + ps_lines.append(' }}') + ps_lines.append(' if ($LASTEXITCODE -eq 0) {{ Write-Host ""; Write-Host "DOWNLOAD_OK" }}') + ps_lines.append(' else {{ Write-Host ""; Write-Host "DOWNLOAD_FAILED (exit $LASTEXITCODE)" }}') + ps_lines.append('}} catch {{') + ps_lines.append(' Write-Host ""; Write-Host "DOWNLOAD_FAILED ($_)"') + ps_lines.append('}}') ps_lines.append(f'Remove-Item -Force "$HOME\\{remote_runner}" -ErrorAction SilentlyContinue') runner_path = TMUX_LOG_DIR / f"{session_id}_run.ps1" - runner_path.write_text("\r\n".join(ps_lines) + "\r\n") + runner_path.write_text("\r\n".join(ps_lines) + "\r\n", encoding="utf-8") # scp the .ps1 script, then launch it as a detached process with log + pid files _port = req.ssh_port @@ -436,6 +1182,10 @@ async def model_download(request: Request, req: ModelDownloadRequest): runner_lines.append("deactivate 2>/dev/null; hash -r") if req.hf_token: runner_lines.append(f"export HF_TOKEN='{_bash_squote(req.hf_token)}'") + if _dl_hf_home_shell and not is_ollama_download: + runner_lines.append(f"export HF_HOME={_dl_hf_home_shell}") + runner_lines.append(f"export HUGGINGFACE_HUB_CACHE={_dl_hf_home_shell}/hub") + runner_lines.append(f"export HF_HUB_CACHE={_dl_hf_home_shell}/hub") if req.env_prefix: runner_lines.append(_safe_env_prefix(req.env_prefix)) else: @@ -446,37 +1196,73 @@ async def model_download(request: Request, req: ModelDownloadRequest): 'done' ) # Ensure pip-user scripts (e.g. hf CLI installed via --user) are on PATH - runner_lines.append('export PATH="$HOME/.local/bin:$PATH"') - # Install hf CLI + hf_transfer best-effort so future runs get the fast path. + runner_lines.append('export PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"') + runner_lines.append('ODYSSEUS_PY="$(command -v python3 || command -v python || true)"') + runner_lines.append('if [ -z "$ODYSSEUS_PY" ]; then echo "ERROR: python3/python not found on this server."; exit 127; fi') + # Install hf CLI + optional hf_transfer best-effort. Retries disable + # hf_transfer because the Rust parallel path is fast but has been + # flaky near the end of very large multi-file downloads. # Use --break-system-packages on PEP-668 systems (Arch, newer Debian) so it doesn't bail. - runner_lines.append("command -v hf >/dev/null 2>&1 || pip install --user --break-system-packages -q -U huggingface_hub 2>/dev/null || pip install -q -U huggingface_hub 2>/dev/null") - runner_lines.append("python3 -c 'import hf_transfer' 2>/dev/null || pip install --user --break-system-packages -q hf_transfer 2>/dev/null || pip install -q hf_transfer 2>/dev/null") - runner_lines.append("python3 -c 'import hf_transfer' 2>/dev/null && export HF_HUB_ENABLE_HF_TRANSFER=1") - runner_lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=8") - # Surface whether the HF token actually reached THIS server, so a gated - # download's "not authorized" failure can be told apart from a missing - # token (the token is masked — we only print applied / not-set). - runner_lines.append(_HF_TOKEN_STATUS_SNIPPET) - # Try hf CLI first, fall back to Python huggingface_hub, then auto-install - runner_lines.append('if command -v hf &>/dev/null; then') - # < /dev/null suppresses interactive "update available? [Y/n]" prompt - runner_lines.append(f' {hf_cmd} < /dev/null') - runner_lines.append('elif python3 -c "import huggingface_hub" 2>/dev/null; then') - runner_lines.append(' echo "hf CLI not found, using Python huggingface_hub..."') - runner_lines.append(f' python3 -c "import os; from huggingface_hub import snapshot_download; snapshot_download(\'{req.repo_id}\'{_dl_pyarg}, max_workers=8)"') - runner_lines.append('else') - runner_lines.append(' echo "Installing huggingface-hub and dependencies..."') - runner_lines.append(' pip install --no-deps -q huggingface-hub 2>/dev/null') - runner_lines.append(' pip install -q filelock fsspec packaging pyyaml tqdm typer httpx requests hf_transfer 2>/dev/null') - runner_lines.append(" python3 -c 'import hf_transfer' 2>/dev/null && export HF_HUB_ENABLE_HF_TRANSFER=1") - runner_lines.append(f' python3 -c "import os; from huggingface_hub import snapshot_download; snapshot_download(\'{req.repo_id}\'{_dl_pyarg}, max_workers=8)"') - runner_lines.append('fi') - runner_lines.append('if [ $? -eq 0 ]; then echo ""; echo "DOWNLOAD_OK"; else echo ""; echo "DOWNLOAD_FAILED (exit $?)"; fi') + if is_ollama_download: + runner_lines.append('if command -v ollama >/dev/null 2>&1; then') + runner_lines.append(f' ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote(ollama_cmd)}') + runner_lines.append('elif command -v docker >/dev/null 2>&1; then') + runner_lines.append(' ODYSSEUS_OLLAMA_CONTAINER="$(docker ps --format \'{{.Names}}\' 2>/dev/null | grep -E \'^(ollama-rocm|ollama-test)$\' | head -1)"') + runner_lines.append(' if [ -n "$ODYSSEUS_OLLAMA_CONTAINER" ]; then') + runner_lines.append(f' ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote("docker exec ${ODYSSEUS_OLLAMA_CONTAINER} " + ollama_cmd)}') + runner_lines.append(' fi') + runner_lines.append('fi') + runner_lines.append('if [ -z "$ODYSSEUS_OLLAMA_PULL_CMD" ]; then echo "ERROR: Ollama not found on this server. Install Ollama or start an ollama-rocm/ollama-test container."; exit 127; fi') + else: + hf_hub_install = _pip_install_fallback_chain( + "huggingface_hub", + python_cmd='"$ODYSSEUS_PY" -m pip', + upgrade=True, + ) + runner_lines.append(f"command -v hf >/dev/null 2>&1 || command -v huggingface-cli >/dev/null 2>&1 || {hf_hub_install}") + runner_lines.append('hash -r 2>/dev/null || true') + runner_lines.append('ODYSSEUS_HF_CLI="$(command -v hf || command -v huggingface-cli || true)"') + runner_lines.append('if [ -z "$ODYSSEUS_HF_CLI" ]; then echo "ERROR: HF CLI not found after installing huggingface_hub."; exit 127; fi') + if req.disable_hf_transfer: + runner_lines.append("export HF_HUB_ENABLE_HF_TRANSFER=0") + runner_lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=4") + else: + hf_transfer_install = _pip_install_fallback_chain( + "hf_transfer", + python_cmd='"$ODYSSEUS_PY" -m pip', + ) + runner_lines.append(f"\"$ODYSSEUS_PY\" -c 'import hf_transfer' 2>/dev/null || {hf_transfer_install}") + runner_lines.append("\"$ODYSSEUS_PY\" -c 'import hf_transfer' 2>/dev/null && export HF_HUB_ENABLE_HF_TRANSFER=1") + runner_lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=8") + # Surface whether the HF token actually reached THIS server, so a gated + # download's "not authorized" failure can be told apart from a missing + # token (the token is masked — we only print applied / not-set). + runner_lines.append(_HF_TOKEN_STATUS_SNIPPET) + # Wrap the download in a retry loop. Large HF/Ollama transfers can + # hit transient network failures; both backends resume cached partials. + mw = 4 if req.disable_hf_transfer else 8 + runner_lines.append('_max_retries=10; _attempt=0; _ec=0') + runner_lines.append('while [ $_attempt -lt $_max_retries ]; do') + runner_lines.append(' _attempt=$((_attempt+1))') + if is_ollama_download: + runner_lines.append(' eval "$ODYSSEUS_OLLAMA_PULL_CMD" < /dev/null') + else: + runner_lines.append(f' "$ODYSSEUS_HF_CLI" {hf_download_args} < /dev/null') + runner_lines.append(' _ec=$?') + runner_lines.append(' if [ $_ec -eq 0 ]; then break; fi') + runner_lines.append(' if [ $_attempt -lt $_max_retries ]; then') + runner_lines.append(' echo ""; echo "Download attempt $_attempt failed (exit $_ec) — retrying in 30s..."') + runner_lines.append(' sleep 30') + runner_lines.append(' fi') + runner_lines.append('done') + runner_lines.append('if [ $_ec -eq 0 ]; then echo ""; echo "DOWNLOAD_OK"; else echo ""; echo "DOWNLOAD_FAILED (exit $_ec after $_attempt attempts)"; fi') runner_lines.append(f"rm -f {remote_runner}") runner_lines.append('exec "${SHELL:-/bin/bash}"') runner_path = TMUX_LOG_DIR / f"{session_id}_run.sh" - runner_path.write_text("\n".join(runner_lines) + "\n") - runner_path.chmod(0o755) + runner_path.write_text("\n".join(runner_lines) + "\n", encoding="utf-8") + # Local temp file is scp'd then chmod'd on the remote; the local bit + # is irrelevant (no-op on Windows). + safe_chmod(runner_path, 0o755) # scp the runner script, then create tmux session on the remote _port = req.ssh_port @@ -484,40 +1270,62 @@ async def model_download(request: Request, req: ModelDownloadRequest): _spf = f"-p {_port} " if _port and _port != "22" else "" setup_cmd = ( f"scp -O {_pf}-q '{runner_path}' {remote}:{remote_runner} && " - f"ssh {_spf}{remote} 'chmod +x {remote_runner} && tmux new-session -d -s {session_id} \"./{remote_runner}\"'" + f"ssh {_spf}{remote} {shlex.quote(_remote_tmux_launch_command(session_id, remote_runner))}" ) else: - # Local: run hf download in a local tmux session + # Local: run hf download in the background (tmux on POSIX, a detached + # process + logfile on Windows where tmux doesn't exist). if req.env_prefix: lines.append(_safe_env_prefix(req.env_prefix)) else: lines.append("deactivate 2>/dev/null; hash -r") # Show whether the HF token reached this run (masked) — tells a gated # "not authorized" failure apart from a missing token. - lines.append(_HF_TOKEN_STATUS_SNIPPET) - # < /dev/null suppresses interactive "update available? [Y/n]" prompt - lines.append(f"{hf_cmd} < /dev/null") - lines.append('if [ $? -eq 0 ]; then echo ""; echo "DOWNLOAD_OK"; else echo ""; echo "DOWNLOAD_FAILED (exit $?)"; fi') - lines.append(f"rm -f '{wrapper_script}'") - lines.append('exec "${SHELL:-/bin/bash}"') - wrapper_script.write_text("\n".join(lines) + "\n") - wrapper_script.chmod(0o755) - setup_cmd = f"tmux new-session -d -s {session_id} {shlex.quote(str(wrapper_script))}" - - logger.info(f"Model download: {req.repo_id} (include={req.include}, session={session_id}, remote={remote})") + if not is_ollama_download: + lines.append(_HF_TOKEN_STATUS_SNIPPET) + # Retry loop — same rationale as the remote-bash path. Issue #2722. + _hf_invoke = 'eval "$ODYSSEUS_OLLAMA_PULL_CMD" < /dev/null' if is_ollama_download else (hf_cmd if IS_WINDOWS else f"{hf_cmd} < /dev/null") + lines.append('_max_retries=10; _attempt=0; _ec=0') + lines.append('while [ $_attempt -lt $_max_retries ]; do') + lines.append(' _attempt=$((_attempt+1))') + lines.append(f' {_hf_invoke}') + lines.append(' _ec=$?') + lines.append(' if [ $_ec -eq 0 ]; then break; fi') + lines.append(' if [ $_attempt -lt $_max_retries ]; then') + lines.append(' echo ""; echo "Download attempt $_attempt failed (exit $_ec) — retrying in 30s..."') + lines.append(' sleep 30') + lines.append(' fi') + lines.append('done') + lines.append('if [ $_ec -eq 0 ]; then echo ""; echo "DOWNLOAD_OK"; else echo ""; echo "DOWNLOAD_FAILED (exit $_ec after $_attempt attempts)"; fi') + if not IS_WINDOWS: + lines.append(f"rm -f '{wrapper_script}'") + lines.append('exec "${SHELL:-/bin/bash}"') + wrapper_script.write_text("\n".join(lines) + "\n", encoding="utf-8") + wrapper_script.chmod(0o755) + setup_cmd = None if IS_WINDOWS else f"tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} {shlex.quote(str(wrapper_script))}" + + logger.info(f"Model download: {req.repo_id} (backend={'ollama' if is_ollama_download else 'hf'}, include={req.include}, session={session_id}, remote={remote})") logger.info(f"Download setup_cmd: {setup_cmd}") - proc = await asyncio.create_subprocess_shell( - setup_cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - await proc.wait() + if setup_cmd is None: + # LOCAL Windows: launch the bash wrapper detached; no tmux setup_cmd. + try: + _launch_local_detached(session_id, lines) + except Exception as e: + logger.error(f"Local detached download launch failed: {e}") + return {"ok": False, "error": str(e), "session_id": session_id} + else: + proc = await asyncio.create_subprocess_shell( + setup_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await proc.wait() - if proc.returncode != 0: - stderr = (await proc.stderr.read()).decode(errors="replace") - logger.error(f"Download failed (rc={proc.returncode}): {stderr}") - return {"ok": False, "error": stderr, "session_id": session_id} + if proc.returncode != 0: + stderr = (await proc.stderr.read()).decode(errors="replace") + logger.error(f"Download failed (rc={proc.returncode}): {stderr}") + return {"ok": False, "error": stderr, "session_id": session_id} # Log to assistant try: @@ -541,114 +1349,78 @@ async def model_cached(request: Request, host: str | None = None, model_dir: str # Validate shell-bound inputs, matching the sibling list_gpus endpoint — # `host`/`ssh_port` are interpolated into an ssh command below, so an # unvalidated value (e.g. "x'; rm -rf ~ #") would be command injection. - host = _validate_remote_host(host) - if ssh_port is not None and ssh_port != "" and not _SSH_PORT_RE.fullmatch(ssh_port): - raise HTTPException(400, "Invalid ssh_port") + host = validate_remote_host(host) + ssh_port = validate_ssh_port(ssh_port) TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True) - paths_code = "import json, os\n" - paths_code += "models = []\n" - paths_code += "seen = set()\n" - paths_code += "BLOCKED_ROOTS = ('/sys', '/proc', '/dev', '/run', '/var/run')\n" - paths_code += "def safe_path(p):\n" - paths_code += " try:\n" - paths_code += " rp = os.path.realpath(os.path.expanduser(p))\n" - paths_code += " return not any(rp == b or rp.startswith(b + os.sep) for b in BLOCKED_ROOTS)\n" - paths_code += " except Exception:\n" - paths_code += " return False\n" - paths_code += "def safe_walk(top):\n" - paths_code += " if not safe_path(top): return\n" - paths_code += " for root, dirs, fns in os.walk(top, followlinks=False):\n" - paths_code += " dirs[:] = [d for d in dirs if not os.path.islink(os.path.join(root, d)) and safe_path(os.path.join(root, d))]\n" - paths_code += " yield root, dirs, fns\n" - # Scan HF cache format (models-- directories with blobs/) - paths_code += "def scan_hf(cache):\n" - paths_code += " if not os.path.isdir(cache): return\n" - paths_code += " for d in sorted(os.listdir(cache)):\n" - paths_code += " if not d.startswith('models--'): continue\n" - paths_code += " rid = d.replace('models--','').replace('--','/')\n" - paths_code += " if rid in seen: continue\n" - paths_code += " seen.add(rid)\n" - paths_code += " blobs = os.path.join(cache, d, 'blobs')\n" - paths_code += " sz, nf, ic = 0, 0, False\n" - paths_code += " if os.path.isdir(blobs):\n" - paths_code += " for f in os.scandir(blobs):\n" - paths_code += " if f.is_file(): nf += 1; sz += f.stat().st_size\n" - paths_code += " if f.name.endswith('.incomplete'): ic = True\n" - paths_code += " # Check if it's an LLM (has config.json with model_type) vs diffusion (has model_index.json)\n" - paths_code += " snap = os.path.join(cache, d, 'snapshots')\n" - paths_code += " is_diffusion = False; is_gguf = False\n" - paths_code += " if os.path.isdir(snap):\n" - paths_code += " for sd in os.listdir(snap):\n" - paths_code += " sf = os.path.join(snap, sd)\n" - paths_code += " if not os.path.isdir(sf): continue\n" - paths_code += " if os.path.exists(os.path.join(sf, 'model_index.json')): is_diffusion = True\n" - paths_code += " try:\n" - paths_code += " if any(x.endswith('.gguf') for x in os.listdir(sf)): is_gguf = True\n" - paths_code += " except Exception: pass\n" - paths_code += " models.append({'repo_id':rid,'size_bytes':sz,'nb_files':nf,'has_incomplete':ic,'path':cache,'is_diffusion':is_diffusion,'is_gguf':is_gguf})\n" - # Scan plain directory (each subdirectory = a model if it has model files) - paths_code += "def scan_dir(p):\n" - paths_code += " if not os.path.isdir(p) or not safe_path(p): return\n" - paths_code += " for d in sorted(os.listdir(p)):\n" - paths_code += " if d.startswith('.'): continue\n" - paths_code += " fp = os.path.join(p, d)\n" - paths_code += " if not os.path.isdir(fp) or os.path.islink(fp) or not safe_path(fp): continue\n" - paths_code += " if d in seen: continue\n" - paths_code += " # Check if it looks like a model (has config.json, safetensors, bin, or gguf)\n" - paths_code += " is_model = False; is_gguf = False\n" - paths_code += " for root, dirs, fns in safe_walk(fp):\n" - paths_code += " for fn in fns:\n" - paths_code += " if fn.endswith('.gguf'): is_gguf = True; is_model = True\n" - paths_code += " elif fn == 'config.json' or fn.endswith('.safetensors') or fn.endswith('.bin'): is_model = True\n" - paths_code += " if is_model: break\n" - paths_code += " if not is_model: continue\n" - paths_code += " seen.add(d)\n" - paths_code += " sz, nf = 0, 0\n" - paths_code += " for dp, _, fns in safe_walk(fp):\n" - paths_code += " for fn in fns:\n" - paths_code += " try: nf += 1; sz += os.path.getsize(os.path.join(dp, fn))\n" - paths_code += " except Exception: pass\n" - paths_code += " is_diff = os.path.exists(os.path.join(fp, 'model_index.json'))\n" - paths_code += " models.append({'repo_id':d,'size_bytes':sz,'nb_files':nf,'has_incomplete':False,'path':p,'is_local_dir':True,'is_diffusion':is_diff,'is_gguf':is_gguf})\n" - # Always scan HF cache - paths_code += "scan_hf(os.path.expanduser('~/.cache/huggingface/hub'))\n" - # Also scan custom model dirs (comma-separated) if specified + model_dirs = [] if model_dir: for d in model_dir.split(','): d = d.strip() - if d and d != '~/.cache/huggingface/hub': - # repr() encodes the dir as a properly-escaped Python string - # literal. The old f"...'{d}'..." broke out of the quotes on - # any `'` in the value, injecting arbitrary Python that then - # ran locally or over ssh. - paths_code += f"scan_dir(os.path.expanduser({d!r}))\n" - paths_code += "print(json.dumps(models))\n" + if d: + if d.startswith(("home/", "mnt/", "media/", "data/", "opt/", "srv/", "var/")): + d = "/" + d + model_dirs.append(d) + paths_code = _cached_model_scan_script(model_dirs) scan_py = TMUX_LOG_DIR / "scan_cache.py" - scan_py.write_text(paths_code) + scan_py.write_text(paths_code, encoding="utf-8") - if host: - _pf = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else "" - if platform == "windows": - # Windows: use 'python' and pipe via stdin with double-quote wrapping - cmd = f'ssh {_pf}{host} "python -" < \'{scan_py}\'' + async def _run_cached_scan_once(): + if host: + _ssh_opts = "-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=4 -o ServerAliveCountMax=1 " + _pf = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else "" + if platform == "windows": + # Windows: use 'python' and pipe via stdin with double-quote wrapping + cmd = f'ssh {_ssh_opts}{_pf}{host} "python -" < \'{scan_py}\'' + else: + cmd = f"ssh {_ssh_opts}{_pf}{host} 'python3 -' < '{scan_py}'" + proc = await asyncio.create_subprocess_shell( + cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=str(Path.home()), + ) else: - cmd = f"ssh {_pf}{host} 'python3 -' < '{scan_py}'" - else: - cmd = f"python3 '{scan_py}'" + # LOCAL scan: use sys.executable (the venv Python Odysseus is already + # running under) — it's guaranteed real Python on all platforms. + # Falling back to which_tool on Windows risks hitting the Microsoft + # Store stub alias for "python3"/"python", which prints + # "Python was not found; run without arguments to install from the + # Microsoft Store" and exits 9009, producing empty stdout and a + # JSON parse error. sys.executable bypasses PATH entirely. + local_py = sys.executable or ( + which_tool("python3") or which_tool("python") + or which_tool("py") or "python" + ) + proc = await asyncio.create_subprocess_exec( + local_py, str(scan_py), + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=str(Path.home()), + ) + return await asyncio.wait_for(proc.communicate(), timeout=60), proc.returncode - proc = await asyncio.create_subprocess_shell( - cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - cwd=str(Path.home()), - ) - stdout_b, stderr_b = await asyncio.wait_for(proc.communicate(), timeout=60) + (stdout_b, stderr_b), returncode = await _run_cached_scan_once() + stderr_txt = stderr_b.decode(errors="replace").strip() + stdout_txt = stdout_b.decode(errors="replace").strip() + if host and returncode != 0 and _ssh_host_key_changed(stderr_txt): + ok, detail = await _repair_cookbook_known_host(host, ssh_port) + if ok: + logger.info("Repaired Cookbook known_hosts for %s after host-key-change scan failure", host) + (stdout_b, stderr_b), returncode = await _run_cached_scan_once() + stderr_txt = stderr_b.decode(errors="replace").strip() + stdout_txt = stdout_b.decode(errors="replace").strip() + else: + logger.warning("Failed to repair Cookbook known_hosts for %s: %s", host, detail[:300]) + if returncode != 0: + msg = stderr_txt or f"Cached model scan failed with exit code {returncode}" + logger.warning(f"Cached model scan failed host={host or 'local'} rc={returncode}: {msg[:500]}") + return {"models": [], "host": host or "local", "error": msg} models = [] try: - raw = json.loads(stdout_b.decode(errors="replace").strip()) + raw = json.loads(stdout_txt) for m in raw: size_gb = m["size_bytes"] / (1024 ** 3) if size_gb >= 1: @@ -666,10 +1438,21 @@ async def model_cached(request: Request, host: str | None = None, model_dir: str } if m.get("is_local_dir"): entry["is_local_dir"] = True + if m.get("is_gguf"): + entry["is_gguf"] = True + if m.get("backend"): + entry["backend"] = m.get("backend") + if m.get("is_ollama"): + entry["is_ollama"] = True + if isinstance(m.get("gguf_files"), list): + entry["gguf_files"] = m["gguf_files"] models.append(entry) except Exception as e: - logger.warning(f"Failed to parse cached models: {e}") - logger.warning(f"stderr: {stderr_b.decode(errors='replace')[:500]}") + logger.warning(f"Failed to parse cached models host={host or 'local'}: {e}") + if stderr_txt: + logger.warning(f"stderr: {stderr_txt[:500]}") + msg = stderr_txt or stdout_txt[:500] or str(e) + return {"models": [], "host": host or "local", "error": msg} return {"models": models, "host": host or "local"} @@ -727,31 +1510,430 @@ def _auto_register_image_endpoint(req: ServeRequest, remote: str | None) -> str finally: db.close() + def _pick_free_port_for_ollama( + remote: str | None, ssh_port: str | None, start_port: int, max_offset: int + ) -> int | None: + """Return the first free port in [start_port, start_port+max_offset] on + the target host. Used to pick a real bind for `ollama serve` so we + don't reattach to an external systemd ollama (or other listener) the + Cookbook Stop button can't kill.""" + import socket + if remote: + # Probe over SSH. Bash's /dev/tcp gives a portable "is anything + # listening" check without requiring ss/netstat/nmap. + ssh_base = ["ssh", "-o", "ConnectTimeout=4", "-o", "StrictHostKeyChecking=no"] + if ssh_port and str(ssh_port) != "22": + try: + ssh_port = validate_ssh_port(ssh_port) + except HTTPException: + return None + ssh_base.extend(["-p", str(ssh_port)]) + try: + host_arg = validate_remote_host(remote) + except HTTPException: + return None + if not host_arg: + return None + probe_ports = " ".join(str(start_port + i) for i in range(max_offset + 1)) + script = ( + f"for p in {probe_ports}; do " + "if ! (exec 3<>/dev/tcp/127.0.0.1/$p) 2>/dev/null; then " + "echo $p; exit 0; fi; exec 3<&-; exec 3>&-; done; exit 1" + ) + try: + import subprocess + r = subprocess.run( + ssh_base + [host_arg, script], + capture_output=True, text=True, timeout=8, + ) + if r.returncode == 0: + out = (r.stdout or "").strip().splitlines() + if out and out[0].isdigit(): + return int(out[0]) + except Exception: + return None + return None + # Local: just try to connect. + for off in range(max_offset + 1): + p = start_port + off + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(0.25) + try: + s.connect(("127.0.0.1", p)) + except (ConnectionRefusedError, socket.timeout, OSError): + return p + return None + + async def _serve_crash_watchdog( + endpoint_id: str, + session_id: str, + remote: str | None, + ssh_port: str | None, + is_windows: bool, + ) -> None: + """Drop a freshly-registered endpoint when the cookbook serve dies early. + + The runner script always emits ``=== Process exited with code N ===`` + when the launched cmd terminates (success or failure). We poll the + tmux pane periodically; on a non-zero exit detected within the watch + window, the endpoint row is deleted so the picker doesn't keep a + dead model around. A zero exit (rare for a long-running serve, but + possible for fast-failing builds that the runner reports as code 0) + and "missing exit marker" both leave the endpoint alone — that's + the loading-but-not-yet-bound state, which the probe-marks-offline + logic already handles. + + Times are picked to outlast realistic vLLM load times (Qwen3.5-122B + takes ~3 min to load) without burning resources on a stuck-forever + wait. After the last check, the watchdog gives up — the picker's + per-endpoint probe takes over from there. + """ + # Cumulative wait points: 25 s, 60 s, 2 min, 5 min. + _waits = [25, 35, 60, 180] + # Tmux capture-pane equivalent of the polling path used elsewhere in + # this file. Build it once and reuse on each tick. Skip the watchdog + # entirely on native-Windows local runs (no tmux). The Windows + # detached-process path writes its log to a known file and has its + # own lifecycle tracking; punting here keeps the code simple. + local_win = is_windows and not remote + if local_win: + return + if remote: + ssh_args = ["ssh"] + if ssh_port and ssh_port != "22": + ssh_args.extend(["-p", str(ssh_port)]) + capture_cmd = ssh_args + [remote, _remote_tmux_command("capture-pane", "-t", session_id, "-p", "-S", "-2000")] + else: + capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-2000"] + + _exit_re = re.compile(r"=== Process exited with code (-?\d+) ===") + for wait_s in _waits: + await asyncio.sleep(wait_s) + try: + proc = await asyncio.create_subprocess_exec( + *capture_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.DEVNULL, + ) + stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=8) + output = stdout.decode("utf-8", errors="replace") + except Exception as e: + logger.debug(f"crash-watchdog: capture-pane failed (will retry): {e!r}") + continue + # Last occurrence wins — a serve that exits/restarts under the + # runner's "exec bash -i" trail will emit multiple markers; the + # most-recent code is the one that matters. + matches = list(_exit_re.finditer(output)) + if not matches: + continue + try: + exit_code = int(matches[-1].group(1)) + except (ValueError, IndexError): + continue + if exit_code == 0: + # Exit 0 on a long-running serve is unusual (a normal "loaded + # then ready" path keeps the process alive) but it happens for + # commands like "ollama pull" the user might launch through + # the same form. Don't drop the endpoint on a clean exit; + # let the probe layer mark it offline if nothing's listening. + logger.info(f"crash-watchdog: serve {session_id} exited cleanly (0); leaving endpoint {endpoint_id}") + return + # Non-zero exit — drop the endpoint. + try: + from core.database import SessionLocal as _SL, ModelEndpoint as _ME + db = _SL() + try: + ep = db.query(_ME).filter(_ME.id == endpoint_id).first() + if ep: + # A scheduled serve can leave old non-zero exit markers + # in tmux scrollback while the current OpenAI endpoint is + # actually alive. Verify reachability before deleting the + # endpoint row; otherwise chats fall back even though the + # served model is ready. + try: + probe_url = ep.base_url.rstrip("/") + "/models" + with urllib.request.urlopen(probe_url, timeout=3) as resp: + if 200 <= getattr(resp, "status", 0) < 300: + logger.info( + f"crash-watchdog: serve {session_id} has exit marker {exit_code} " + f"but endpoint {ep.id} is reachable; leaving it registered" + ) + return + except Exception: + pass + logger.info( + f"crash-watchdog: dropping endpoint {endpoint_id} " + f"({ep.name} @ {ep.base_url}) — serve exited {exit_code}" + ) + db.delete(ep) + db.commit() + finally: + db.close() + except Exception as e: + logger.warning(f"crash-watchdog: endpoint cleanup failed: {e!r}") + return + logger.debug(f"crash-watchdog: no exit marker for {session_id} within window; leaving endpoint {endpoint_id}") + + def _auto_register_llm_endpoint(req: ServeRequest, remote: str | None) -> str | None: + """Register a freshly-served LLM as a model endpoint so it appears in the + model picker without a manual /setup step — the text-model sibling of + _auto_register_image_endpoint. + + Cookbook serve commands launch an OpenAI-compatible server (llama.cpp's + llama-server, vLLM, SGLang, or Ollama) on a known port. We point an + endpoint at that server's /v1; the picker auto-discovers the model id by + probing /v1/models and dims the endpoint until the server is reachable, + so registering immediately (before the server finishes loading) is safe. + """ + logger.info( + f"_auto_register_llm_endpoint: ENTRY repo_id={req.repo_id!r} " + f"remote={remote!r} cmd_prefix={req.cmd[:80]!r}" + ) + import re + from core.database import SessionLocal, ModelEndpoint + + # Port: ordered fallbacks so we match whatever the user actually + # asked for, not a hardcoded default: + # 1. explicit `--port N` (vllm / sglang / llama-server) + # 2. `OLLAMA_HOST=host:port` (the way Ollama specifies its bind) + # 3. fallback by backend (11434 ollama / 8080 llama.cpp) + # Previously the OLLAMA_HOST form was silently ignored and we + # registered every Ollama endpoint at 11434 — even if the user + # set OLLAMA_HOST=0.0.0.0:11435 to avoid colliding with an + # existing systemd Ollama, the registered endpoint pointed at + # the OLD port and showed as offline. + port_match = re.search(r'--port\s+(\d+)', req.cmd) + ollama_host_match = re.search(r'OLLAMA_HOST=[^\s]*?:(\d+)', req.cmd) + if port_match: + port = int(port_match.group(1)) + elif ollama_host_match: + port = int(ollama_host_match.group(1)) + elif "ollama" in req.cmd: + port = 11434 + else: + port = 8080 # llama.cpp's llama-server default — the Apple Silicon path + + # Determine host. The cookbook tmux for `local=true` serves runs INSIDE + # the odysseus container — so the right URL for the in-container + # backend to reach it is `localhost`, NOT `host.docker.internal` + # (the latter points at the docker HOST, which doesn't have a server + # on that port). The previous host.docker.internal fallback only made + # sense for /setup-added external services like systemd Ollama on the + # host — and those go through manual setup, not this auto-register + # code path. For remote serves we still use the SSH host alias. + if remote: + host = remote.split("@")[-1] if "@" in remote else remote + elif re.search(r"\bdocker\s+exec\s+(?:ollama-rocm|ollama-test)\b", req.cmd or ""): + host = "host.docker.internal" + else: + host = "localhost" + + base_url = f"http://{host}:{port}/v1" + + short_name = req.repo_id.split("/")[-1] if "/" in req.repo_id else req.repo_id + display_name = short_name or "Local model" + is_mlx_deepseek_v4 = ( + "mlx_lm.server" in (req.cmd or "") + and "deepseek-v4" in ((req.repo_id or "") + " " + (req.cmd or "")).lower() + ) + mlx_shim_model_id = "" + if is_mlx_deepseek_v4 and short_name: + home_match = re.search(r"((?:/Users|/home)/[^/\s'\"]+)", req.cmd or "") + remote_home = home_match.group(1) if home_match else "" + if remote_home: + mlx_shim_model_id = f"{remote_home}/.cache/odysseus/mlx-shims/{short_name}" + + # If the serve command opts models into OpenAI tool-calling, record it so + # agent_loop trusts emitted tool_calls instead of the name heuristic. + is_ollama_endpoint = "ollama" in (req.cmd or "").lower() + supports_tools = True if "--enable-auto-tool-choice" in req.cmd else None + # Pin the model the user launched for every Cookbook-created LLM + # endpoint, not just Ollama. Some OpenAI-compatible servers report a + # deployment alias from /v1/models, and a stale server can answer on the + # same port while the new launch failed. Keeping the requested model id + # pinned makes the picker reflect the actual launch intent. + pinned_models = [mlx_shim_model_id] if mlx_shim_model_id else ([req.repo_id] if req.repo_id else []) + + db = SessionLocal() + try: + # Reuse an endpoint already pointed at this URL instead of duplicating. + existing = db.query(ModelEndpoint).filter(ModelEndpoint.base_url == base_url).first() + if existing: + existing.is_enabled = True + existing.model_type = "llm" + existing.name = display_name + existing.endpoint_kind = "local" + existing.model_refresh_mode = "auto" + if pinned_models: + try: + existing_pinned = json.loads(existing.pinned_models or "[]") + except Exception: + existing_pinned = [] + merged_pinned = [] + for mid in [*existing_pinned, *pinned_models]: + if mid and mid not in merged_pinned: + merged_pinned.append(mid) + existing.pinned_models = json.dumps(merged_pinned) if merged_pinned else None + if is_ollama_endpoint: + existing.endpoint_kind = "ollama" + if pinned_models: + existing.cached_models = json.dumps(pinned_models) + if supports_tools is not None: + existing.supports_tools = supports_tools + db.commit() + logger.info(f"Updated existing local model endpoint: {base_url}") + # Re-probe so cached_models matches what the server actually + # serves right now (the URL may have stayed the same but the + # model behind it changed across launches). + try: + if mlx_shim_model_id: + existing.cached_models = json.dumps([mlx_shim_model_id]) + existing.pinned_models = json.dumps([mlx_shim_model_id]) + db.commit() + else: + from routes.model_routes import _probe_endpoint + import json as _json2 + probed = _probe_endpoint(base_url, existing.api_key, timeout=5) + if probed: + existing.cached_models = _json2.dumps(probed) + db.commit() + except Exception as _pe: + logger.warning(f"Re-probe failed for {base_url}: {_pe!r}") + # Sweep stale dupes: other endpoints with the same display name + # at DIFFERENT URLs (likely failed earlier-attempt ports) get + # deleted so the picker doesn't show an offline ghost next to + # the working one. Only sweeps endpoints whose id starts with + # `local-` so we never touch a user's hand-added DeepSeek/OpenAI/ + # etc. entry with a coincidentally matching name. + stale = (db.query(ModelEndpoint) + .filter(ModelEndpoint.name == display_name) + .filter(ModelEndpoint.base_url != base_url) + .filter(ModelEndpoint.id.like("local-%")) + .all()) + for s in stale: + logger.info(f"Sweeping stale local endpoint {s.id} ({s.base_url})") + db.delete(s) + if stale: + db.commit() + return existing.id + + ep_id = f"local-{uuid.uuid4().hex[:8]}" + ep = ModelEndpoint( + id=ep_id, + name=display_name, + base_url=base_url, + api_key=None, + is_enabled=True, + model_type="llm", + endpoint_kind="ollama" if is_ollama_endpoint else "local", + model_refresh_mode="auto", + cached_models=json.dumps(pinned_models) if pinned_models else None, + pinned_models=json.dumps(pinned_models) if pinned_models else None, + supports_tools=supports_tools, + ) + db.add(ep) + db.commit() + logger.info(f"Auto-registered local model endpoint: {display_name} @ {base_url}") + # Same sweep on first-register path: drop any pre-existing local-* + # endpoints with this display name pointed elsewhere. + stale = (db.query(ModelEndpoint) + .filter(ModelEndpoint.name == display_name) + .filter(ModelEndpoint.id != ep_id) + .filter(ModelEndpoint.id.like("local-%")) + .all()) + for s in stale: + logger.info(f"Sweeping stale local endpoint {s.id} ({s.base_url})") + db.delete(s) + if stale: + db.commit() + # Probe /v1/models NOW and write cached_models so the chat + # picker actually shows the model on the next /api/models + # call. Without this immediate probe, the endpoint has empty + # cached_models until the next background refresh fires (up + # to a minute later) and the picker shows nothing — even + # though the endpoint is in the DB and the server is up. + try: + if mlx_shim_model_id: + ep.cached_models = json.dumps([mlx_shim_model_id]) + ep.pinned_models = json.dumps([mlx_shim_model_id]) + db.commit() + logger.info(f"Auto-register: pinned MLX DeepSeek-V4 shim model @ {base_url}") + else: + from routes.model_routes import _probe_endpoint + import json as _json2 + probed = _probe_endpoint(base_url, None, timeout=5) + if probed: + ep.cached_models = _json2.dumps(probed) + db.commit() + logger.info(f"Auto-register: probed {len(probed)} models @ {base_url}") + except Exception as _pe: + logger.warning(f"Auto-register: probe-after-create failed for {base_url}: {_pe!r}") + return ep_id + except Exception as e: + logger.error(f"Failed to auto-register local model endpoint: {e}") + db.rollback() + return None + finally: + db.close() + @router.post("/api/model/serve") async def model_serve(request: Request, req: ServeRequest): """Launch a model server in a tmux session (or PowerShell background process on Windows). `repo_id` is dual-purpose: a HuggingFace repo (`/`) for - model-serve commands, OR a bare pip package name when the cmd is a - `python -m pip install …`. We only enforce the strict HF format on - the model paths. + model-serve commands, a cached local-model id (the folder name reported + by `/api/model/cached`) for models scanned from a custom model dir, OR a + bare pip package name when the cmd is a `python -m pip install …`. We + keep strict validation, but serving local cached models must not require + a fake org/name wrapper. """ require_admin(request) # Defence-in-depth: reject values that could break out of shell contexts. - _validate_remote_host(req.remote_host) - req.ssh_port = _validate_ssh_port(req.ssh_port) + validate_remote_host(req.remote_host) + req.ssh_port = validate_ssh_port(req.ssh_port) req.gpus = _validate_gpus(req.gpus) req.hf_token = req.hf_token or _load_stored_hf_token() _validate_token(req.hf_token) - # Normalize away backslash-newline continuations (multi-line pasted - # serve commands) so the cleaned single-line command is what gets - # written into the runner script and used for engine auto-detection. - # `_validate_serve_cmd` returns None for empty input; coerce to "" so the - # many downstream `"engine" in req.cmd` membership checks can't hit - # `TypeError: argument of type 'NoneType'` (a 500 instead of a clean 400). - req.cmd = _validate_serve_cmd(req.cmd) or "" + # Cookbook emits two fixed Docker exec forms for its Ollama sidecars. + # Keep Docker out of the general allowlist: only these parsed shapes may + # proceed to the target-aware Docker availability/opt-in preflight. + if _is_generated_ollama_docker_exec_cmd(req.cmd): + req.cmd = req.cmd.strip() + else: + # Normalize away backslash-newline continuations (multi-line pasted + # serve commands) so the cleaned single-line command is what gets + # written into the runner script and used for engine auto-detection. + # `_validate_serve_cmd` returns None for empty input; coerce to "" so + # downstream `"engine" in req.cmd` checks cannot raise TypeError. + req.cmd = _validate_serve_cmd(req.cmd) or "" + req.cmd = _normalize_llama_cpp_python_cache_types(req.cmd) or "" + req.cmd = _normalize_minimax_m3_vllm_cmd(req.cmd) + req.cmd = _normalize_deepseek_v4_sglang_cmd(req.cmd) + req.cmd = _venv_safe_local_pip_install_cmd( + req.cmd, + local=not bool(req.remote_host), + in_venv=sys.prefix != sys.base_prefix, + ) is_pip_install = bool(req.cmd and "pip install" in req.cmd) if is_pip_install: + # Keep big dependency wheel builds (vLLM, …) off the home filesystem's + # pip cache so they don't fail mid-build with "No space left" (#1219) + # and leave the dep installed-but-unusable (#1459). + req.cmd = _pip_install_no_cache(req.cmd) + # Accept common aliases and enforce server extras for llama-cpp so + # `python -m llama_cpp.server` has all runtime dependencies. + # CRITICAL: the lookbehind / lookahead must also exclude `/` so + # the regex DOESN'T mangle a URL path like + # https://abetlen.github.io/llama-cpp-python/whl/cu124 + # The previous regex turned that URL into + # https://abetlen.github.io/llama-cpp-python[server]/whl/cu124 + # which pip then couldn't resolve → silent fallback to source + # build of the .tar.gz → CPU-only binary (because CMAKE_ARGS + # isn't set), defeating the entire purpose of the CUDA index. + req.cmd = re.sub(r"(?=!~,` for version specifiers. # v2 review HIGH-14: tightened from the previous regex which @@ -763,22 +1945,57 @@ async def model_serve(request: Request, req: ServeRequest): ): raise HTTPException(400, "Invalid pip package name") else: - _validate_repo_id(req.repo_id) + _validate_serve_model_id(req.repo_id) TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True) session_id = f"serve-{uuid.uuid4().hex[:8]}" remote = req.remote_host is_windows = req.platform == "windows" - if not is_windows and not await _binary_available("tmux", remote, req.ssh_port): + # Ollama: if the user didn't pin a port, resolve the actual port we'll + # bind to here (before runner construction) by probing the target host. + # Otherwise the runner script picks one at runtime and `_auto_register` + # below still registers the stale 11434 default — which on a host with + # a systemd ollama lands on the wrong (unreachable-from-docker) service. + # Match "ollama serve" as a phrase (with optional flags after), not + # any substring containing "ollama" — otherwise commands like + # `docker exec ollama-test ollama-import …` get wrapped as if they + # were native `ollama serve`, prepending OLLAMA_HOST=… and then + # running the ollama-not-found preflight which exits 127. + if re.search(r"\bollama\s+serve\b", req.cmd) and "OLLAMA_HOST=" not in req.cmd: + _ollama_bind_host = "0.0.0.0" if remote else "127.0.0.1" + _ollama_chosen_port = _pick_free_port_for_ollama( + remote, req.ssh_port, start_port=11434, max_offset=10, + ) + if _ollama_chosen_port: + req.cmd = f"OLLAMA_HOST={_ollama_bind_host}:{_ollama_chosen_port} {req.cmd}" + # LOCAL execution on a native-Windows host never uses tmux (detached + # process path below), regardless of the UI-supplied platform. + local_windows = IS_WINDOWS and not remote + if is_windows and remote and "diffusion_server.py" in req.cmd: + raise HTTPException( + 400, + "Remote Windows Diffusers serving is not supported yet; use local Windows or a Linux remote server.", + ) + + if not is_windows and not local_windows and not await _binary_available("tmux", remote, req.ssh_port): return { "ok": False, "error": _missing_binary_message("tmux", remote or "local server"), "session_id": session_id, } if _needs_binary(req.cmd, "docker") and not await _binary_available("docker", remote, req.ssh_port, windows=is_windows): + local_host_docker_blocked = ( + not remote + and running_in_container() + and not host_docker_access_enabled() + ) return { "ok": False, - "error": _missing_binary_message("docker", remote or "local server"), + "error": _missing_binary_message( + "docker", + remote or "local server", + local_host_docker_blocked=local_host_docker_blocked, + ), "session_id": session_id, } @@ -812,10 +2029,12 @@ async def model_serve(request: Request, req: ServeRequest): ps_lines.append('Write-Host "ERROR: vLLM is not supported on Windows. Use Ollama or llama.cpp instead."') ps_lines.append('exit 1') ps_lines.append(req.cmd) + if is_pip_install: + ps_lines.append('if ($LASTEXITCODE -eq 0) { Write-Host ""; Write-Host "DOWNLOAD_OK" }') ps_lines.append('Write-Host ""') ps_lines.append('Write-Host "=== Process exited with code $LASTEXITCODE ==="') runner_path = TMUX_LOG_DIR / f"{session_id}_run.ps1" - runner_path.write_text("\r\n".join(ps_lines) + "\r\n") + runner_path.write_text("\r\n".join(ps_lines) + "\r\n", encoding="utf-8") _port = req.ssh_port _Pf = f"-P {_port} " if _port and _port != "22" else "" @@ -834,7 +2053,31 @@ async def model_serve(request: Request, req: ServeRequest): else: # ── Linux/Termux: bash + tmux (existing flow) ── runner_lines = ["#!/bin/bash"] + # Mirror every line of stdout+stderr into a persistent log file + # on the host running the serve. This is the file tail_serve_output + # reads when the tmux pane has been overwritten by the post-crash + # bash prompt — without it, the agent's diagnostic tool sees the + # neofetch banner instead of the actual Python traceback. + # We save the original fds to 3/4 so we can RESTORE them before + # `exec ${SHELL}` at the end of the script. Without that restore, + # the post-crash interactive shell's neofetch banner ALSO gets + # teed into the log file and `tail -N` returns ONLY the banner — + # the actual traceback ends up earlier than the tail window. + runner_lines.append("mkdir -p /tmp/odysseus-tmux 2>/dev/null || true") + runner_lines.append("exec 3>&1 4>&2") + runner_lines.append( + f"exec > >(tee -a /tmp/odysseus-tmux/{session_id}.log) 2>&1" + ) runner_lines.extend(_user_shell_path_bootstrap()) + runner_lines.append('ODYSSEUS_PREFLIGHT_EXIT=""') + # Put Odysseus's own venv bin on PATH (local runs only) so the serve + # shell resolves the bundled python3/hf, mirroring the download flow. + if not remote: + runner_lines.append(_local_tooling_path_export(sys.executable)) + if local_windows: + # Detached Git Bash runs do not always inherit recently edited + # user PATH entries from the already-running Odysseus process. + runner_lines.append('export PATH="$HOME/bin:$HOME/llama.cpp/build-cuda/bin/Release:$HOME/llama.cpp/build/bin/Release:$HOME/llama.cpp/build/bin/Debug:$HOME/llama.cpp/build/bin:$PATH"') runner_lines.append("export FLASHINFER_DISABLE_VERSION_CHECK=1") if req.hf_token: runner_lines.append(f"export HF_TOKEN='{_bash_squote(req.hf_token)}'") @@ -844,63 +2087,458 @@ async def model_serve(request: Request, req: ServeRequest): runner_lines.append(_safe_env_prefix(req.env_prefix)) else: runner_lines.append("deactivate 2>/dev/null; hash -r") + _append_venv_nvidia_library_path_lines(runner_lines, cmd=req.cmd) + if "sglang.launch_server" in req.cmd or "mlx_lm.server" in req.cmd or re.search(r"\bvllm\s+serve\b", req.cmd or ""): + _append_openai_port_preflight_lines(runner_lines, cmd=req.cmd, expected_model=req.repo_id) # Show whether the HF token reached this server (masked) — a gated # model vLLM has to download will be denied without it. runner_lines.append(_HF_TOKEN_STATUS_SNIPPET) + handled_ollama_serve = False # Auto-install inference engine if missing - if "llama_cpp" in req.cmd or "llama-server" in req.cmd: + local_windows_llama_cmd = local_windows and ("llama_cpp" in req.cmd or "llama-server" in req.cmd) + if ("llama_cpp" in req.cmd or "llama-server" in req.cmd) and not local_windows_llama_cmd: # Prefer the NATIVE llama-server binary — its minja templating # renders modern GGUF chat templates that the Python bindings' # Jinja2 rejects (do_tojson ensure_ascii). Build it once from # source if missing; keep llama-cpp-python only as a fallback. runner_lines.append('# Ensure a llama.cpp server (prefer native llama-server)') - runner_lines.append('export PATH="$HOME/.local/bin:$HOME/bin:$HOME/llama.cpp/build/bin:$PATH"') + # Include the Homebrew bin dirs so a brew-installed llama-server / + # ollama is found (otherwise macOS falls back to a slow source build). + # /opt/homebrew = Apple Silicon, /usr/local = Intel; harmless on Linux. + runner_lines.append('export PATH="$HOME/.local/bin:$HOME/bin:$HOME/llama.cpp/build/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"') runner_lines.append('if [ -d /data/data/com.termux ]; then') runner_lines.append(' # Termux: no native build — use the Python bindings (CPU).') runner_lines.append(' if ! python3 -c "import llama_cpp" 2>/dev/null; then') runner_lines.append(' pkg install -y cmake 2>/dev/null') runner_lines.append(' pip install numpy diskcache jinja2 2>/dev/null') - runner_lines.append(' CMAKE_ARGS="-DGGML_BLAS=OFF -DGGML_LLAMAFILE=OFF" pip install llama-cpp-python --no-build-isolation --no-cache-dir 2>&1 || true') + runner_lines.append(' CMAKE_ARGS="-DGGML_BLAS=OFF -DGGML_LLAMAFILE=OFF" pip install \'llama-cpp-python[server]\' --no-build-isolation --no-cache-dir 2>&1 || true') runner_lines.append(' fi') runner_lines.append('elif ! command -v llama-server &>/dev/null; then') runner_lines.append(' echo "Native llama-server not found — building from source (one-time, may take a few minutes)..."') runner_lines.append(' mkdir -p ~/bin') runner_lines.append(' cd ~ && [ -d llama.cpp ] || git clone --depth 1 https://github.com/ggml-org/llama.cpp') - # GPU build if CUDA is present; fall back to a plain (CPU) build. - runner_lines.append(' cd ~/llama.cpp && { cmake -B build -DGGML_CUDA=ON 2>/dev/null || cmake -B build; } \\') - runner_lines.append(' && cmake --build build -j"$(nproc)" --target llama-server \\') - runner_lines.append(' && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server') + # Build with the right accelerator: Metal on macOS (llama.cpp + # enables it automatically, no flag), CUDA on Linux when present, + # else a plain CPU build. nproc is Linux-only — fall back to + # `sysctl hw.ncpu` on macOS. (Tip: `brew install llama.cpp` ships + # a prebuilt llama-server and skips this whole source build.) + runner_lines.append(' NPROC="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)"') + runner_lines.append(' if [ "$(uname -s)" = "Darwin" ]; then') + runner_lines.append(' command -v cmake >/dev/null 2>&1 || echo "WARNING: cmake not found — install it with: brew install cmake (or: brew install llama.cpp for a prebuilt llama-server)."') + # Start from a clean cache: a prior failed configure (e.g. a CUDA + # attempt) poisons build/CMakeCache.txt, so a plain `cmake -B build` + # would reuse the bad settings and fail again. CMAKE_BUILD_TYPE is + # explicit so the binary is optimized (Metal auto-enables on macOS). + runner_lines.append(' cd ~/llama.cpp && rm -rf build && cmake -B build -DCMAKE_BUILD_TYPE=Release \\') + runner_lines.append(' && cmake --build build -j"$NPROC" --target llama-server \\') + runner_lines.append(' && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server') + runner_lines.append(' else') + _append_llama_cpp_linux_accel_build_lines(runner_lines) + runner_lines.append(' fi') + # Source the env file the prebuilt-download path writes so + # LD_LIBRARY_PATH includes the directory holding libllama.so + # and friends. No-op when prebuilt wasn't used. + runner_lines.append(' [ -r ~/.config/odysseus-llama-cpp-env ] && . ~/.config/odysseus-llama-cpp-env') + # Auto-upgrade pip llama-cpp-python to the CUDA-enabled + # wheel when (a) NVIDIA hardware is present and (b) the + # currently-installed wheel is CPU-only. Without this the + # user gets the Python server happily running at 3 tok/s + # because pip's default index ships CPU-only wheels. + # Forward-compat: cu124 wheels work on driver/runtime + # 12.4+ including the cu13.x line. + runner_lines.append(' if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q "GPU " && python3 -c "import llama_cpp" 2>/dev/null; then') + runner_lines.append(' if ! python3 -c "import llama_cpp; import sys; sys.exit(0 if llama_cpp.llama_supports_gpu_offload() else 1)" 2>/dev/null; then') + runner_lines.append(' echo "[odysseus] NVIDIA detected but installed llama-cpp-python is CPU-only — reinstalling with CUDA wheel index for GPU offload..."') + runner_lines.append(' python3 -m pip install --user --break-system-packages --force-reinstall --no-cache-dir "llama-cpp-python[server]" --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124 2>&1 | tail -8 || echo "[odysseus] WARNING: CUDA wheel reinstall failed — Python server will stay CPU-only (slow). Manual fix: pip install --user --force-reinstall \'llama-cpp-python[server]\' --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124"') + runner_lines.append(' if python3 -c "import llama_cpp; import sys; sys.exit(0 if llama_cpp.llama_supports_gpu_offload() else 1)" 2>/dev/null; then') + runner_lines.append(' echo "[odysseus] llama-cpp-python now supports GPU offload."') + runner_lines.append(' fi') + runner_lines.append(' fi') + runner_lines.append(' fi') + # SHORT-CIRCUIT before the build/pip fallback: if the + # native binary is missing but llama_cpp Python is already + # installed, drop a wrapper at ~/bin/llama-server that + # translates llama-server CLI args to llama_cpp.server's + # underscore-style flags. The user's serve command stays + # `llama-server ...` and "just works" — no build, no cmake, + # no second install. This is the path that unblocks every + # remote where pip-installed llama-cpp-python is already + # working but Cookbook used to insist on a native binary. + runner_lines.append(' if ! command -v llama-server >/dev/null 2>&1 && python3 -c "import llama_cpp" 2>/dev/null; then') + runner_lines.append(' mkdir -p ~/bin') + runner_lines.append(' cat > ~/bin/llama-server <<\'_ODY_LLAMA_SHIM_EOF\'') + runner_lines.append('#!/usr/bin/env bash') + runner_lines.append('# Auto-generated by Odysseus Cookbook: a `llama-server` lookalike') + runner_lines.append('# that translates the native CLI to `python -m llama_cpp.server`.') + runner_lines.append('# Lets cookbook-generated launch commands run unchanged on hosts') + runner_lines.append('# where only the pip llama-cpp-python package is installed.') + runner_lines.append('ARGS=()') + runner_lines.append('while [ $# -gt 0 ]; do') + runner_lines.append(' case "$1" in') + runner_lines.append(' -ngl|--gpu-layers|--n-gpu-layers) ARGS+=(--n_gpu_layers "$2"); shift 2 ;;') + runner_lines.append(' -c|--ctx-size) ARGS+=(--n_ctx "$2"); shift 2 ;;') + runner_lines.append(' -b|--batch-size) ARGS+=(--n_batch "$2"); shift 2 ;;') + runner_lines.append(' -ub|--ubatch-size) shift 2 ;; # llama-cpp-python has no separate ubatch') + runner_lines.append(' --flash-attn) ARGS+=(--flash_attn true); shift 2 ;;') + runner_lines.append(' --cache-type-k) ARGS+=(--type_k "$2"); shift 2 ;;') + runner_lines.append(' --cache-type-v) ARGS+=(--type_v "$2"); shift 2 ;;') + runner_lines.append(' --n-cpu-moe) ARGS+=(--n_cpu_moe "$2"); shift 2 ;;') + runner_lines.append(' --mmproj) ARGS+=(--clip_model_path "$2"); shift 2 ;;') + runner_lines.append(' --image-max-tokens) shift 2 ;; # native-only') + runner_lines.append(' --no-mmap) ARGS+=(--no_mmap true); shift ;;') + runner_lines.append(' --no-warmup) shift ;; # native-only') + runner_lines.append(' --chat-template) ARGS+=(--chat_format "$2"); shift 2 ;;') + runner_lines.append(' --fit|--split-mode|--tensor-split|--main-gpu|--parallel) shift 2 ;; # native-only') + runner_lines.append(' --mlock) ARGS+=(--use_mlock true); shift ;;') + runner_lines.append(' *) ARGS+=("$1"); shift ;;') + runner_lines.append(' esac') + runner_lines.append('done') + runner_lines.append('exec python3 -m llama_cpp.server "${ARGS[@]}"') + runner_lines.append('_ODY_LLAMA_SHIM_EOF') + runner_lines.append(' chmod +x ~/bin/llama-server') + runner_lines.append(' echo "[odysseus] Created llama-server shim → python -m llama_cpp.server (no native binary needed)"') + runner_lines.append(' fi') runner_lines.append(' # If the native build failed, fall back to the Python bindings.') runner_lines.append(' if ! command -v llama-server &>/dev/null && ! python3 -c "import llama_cpp" 2>/dev/null; then') runner_lines.append(' echo "llama-server build failed — installing Python bindings as fallback..."') - runner_lines.append(' pip install --user --break-system-packages -q llama-cpp-python 2>/dev/null || pip install -q llama-cpp-python 2>/dev/null || true') + runner_lines.append(f" {_pip_install_fallback_chain('llama-cpp-python[server]', python_cmd='pip')} || true") + runner_lines.append(' fi') + runner_lines.append(' if ! command -v llama-server &>/dev/null && ! python3 -c "import llama_cpp" 2>/dev/null; then') + runner_lines.append(' echo "ERROR: llama.cpp serving is not available after install/build attempts."') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') runner_lines.append(' fi') runner_lines.append('fi') + elif re.search(r"\bollama\s+serve\b", req.cmd): + handled_ollama_serve = True + _ollama_default_host = "0.0.0.0" if remote else "127.0.0.1" + _ollama_host, _ollama_port = _ollama_bind_from_cmd( + req.cmd, + default_host=_ollama_default_host, + ) + # Always launch a fresh ollama under tmux so Stop reliably + # kills it. If the requested port is busy (e.g. a systemd + # ollama on 11434), scan upward for a free one rather than + # silently reattaching to an external service that Stop + # can't reach. + runner_lines.append(f'ODYSSEUS_OLLAMA_HOST={_bash_squote(_ollama_host)}') + runner_lines.append(f'ODYSSEUS_OLLAMA_PORT="{_ollama_port}"') + runner_lines.append('for _ody_off in 0 1 2 3 4 5 6 7 8 9; do') + runner_lines.append(' _ody_try_port=$((ODYSSEUS_OLLAMA_PORT + _ody_off))') + runner_lines.append(' if ! (exec 3<>/dev/tcp/127.0.0.1/$_ody_try_port) 2>/dev/null; then') + runner_lines.append(' exec 3<&-; exec 3>&-') + runner_lines.append(' ODYSSEUS_OLLAMA_PORT="$_ody_try_port"') + runner_lines.append(' break') + runner_lines.append(' fi') + runner_lines.append(' exec 3<&-; exec 3>&-') + runner_lines.append('done') + runner_lines.append('if ! command -v ollama &>/dev/null; then') + # Single-quoted on purpose: backticks inside a double-quoted + # echo are command substitution, and this line used to run the + # curl|sh installer on the target host instead of printing it. + runner_lines.append(f" echo '{_bash_squote(OLLAMA_MISSING_HINT)}'") + runner_lines.append(' echo') + runner_lines.append(' echo "=== Process exited with code 127 ==="') + runner_lines.append(' exec bash -i') + runner_lines.append('fi') + runner_lines.append('ODYSSEUS_OLLAMA_URL="http://${ODYSSEUS_OLLAMA_HOST}:${ODYSSEUS_OLLAMA_PORT}"') + if remote and _ollama_host in ("0.0.0.0", "::"): + runner_lines.append('echo "[odysseus] WARNING: remote Ollama will bind to ${ODYSSEUS_OLLAMA_HOST}:${ODYSSEUS_OLLAMA_PORT} so Odysseus can reach it from this host."') + runner_lines.append('echo "[odysseus] Ollama has no built-in authentication; expose this only on a trusted LAN/VPN or provide an explicit OLLAMA_HOST with your own access controls."') + runner_lines.append('echo "Starting ollama server on ${ODYSSEUS_OLLAMA_HOST}:${ODYSSEUS_OLLAMA_PORT}..."') + runner_lines.append('OLLAMA_HOST="${ODYSSEUS_OLLAMA_HOST}:${ODYSSEUS_OLLAMA_PORT}" ollama serve') + runner_lines.append('_ody_exit=$?') + runner_lines.append('echo') + runner_lines.append('echo "=== Process exited with code ${_ody_exit} ==="') + runner_lines.append('exec bash -i') elif "vllm serve" in req.cmd: + # vLLM is CUDA/ROCm-only and does not run on macOS at all. + runner_lines.append('if [ "$(uname -s)" = "Darwin" ]; then') + runner_lines.append(' echo "ERROR: vLLM does not run on macOS. Use Ollama or llama.cpp (Metal) instead."') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=1') + runner_lines.append('fi') # Put ~/.local/bin on PATH first — without a venv, vllm installs # there via --user and the non-login serve shell otherwise can't # find the `vllm` CLI ("command not found"). Mirrors llama.cpp above. runner_lines.append('export PATH="$HOME/.local/bin:$PATH"') runner_lines.append('if ! command -v vllm &>/dev/null; then') - runner_lines.append(' echo "ERROR: vLLM is not installed. Open Cookbook -> Dependencies and install vllm on this server, then launch again."') - runner_lines.append(' exit 127') + runner_lines.append(' echo "ERROR: vLLM is not installed."') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') + runner_lines.append('fi') + runner_lines.append(f"ODYSSEUS_SERVE_CMD='{_bash_squote(req.cmd)}'") + runner_lines.append('if [ -z "$ODYSSEUS_PREFLIGHT_EXIT" ]; then') + runner_lines.append(' ODYSSEUS_VLLM_HELP_CMD="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'') + runner_lines.append('import shlex, sys') + runner_lines.append('parts = shlex.split(sys.argv[1])') + runner_lines.append('try:') + runner_lines.append(' serve_i = parts.index("serve")') + runner_lines.append('except ValueError:') + runner_lines.append(' print("vllm serve --help")') + runner_lines.append('else:') + runner_lines.append(' print(shlex.join(parts[:serve_i + 1] + ["--help"]))') + runner_lines.append('PY') + runner_lines.append(')"') + runner_lines.append(' ODYSSEUS_VLLM_SUPPORTS_SWAP=0') + runner_lines.append(' if eval "$ODYSSEUS_VLLM_HELP_CMD" 2>&1 | grep -q -- "--swap-space"; then ODYSSEUS_VLLM_SUPPORTS_SWAP=1; fi') + runner_lines.append('fi') + runner_lines.append('if [ -z "$ODYSSEUS_PREFLIGHT_EXIT" ] && [ "${ODYSSEUS_VLLM_SUPPORTS_SWAP:-0}" = "1" ] && ! printf "%s" "$ODYSSEUS_SERVE_CMD" | grep -q -- "--swap-space"; then') + runner_lines.append(' echo "[odysseus] Setting vLLM --swap-space 0 so the runtime does not reserve CPU swap per GPU."') + runner_lines.append(' ODYSSEUS_SERVE_CMD="${ODYSSEUS_SERVE_CMD} --swap-space 0"') + runner_lines.append('fi') + runner_lines.append('if [ -z "$ODYSSEUS_PREFLIGHT_EXIT" ] && [ "${ODYSSEUS_VLLM_SUPPORTS_SWAP:-0}" != "1" ]; then') + runner_lines.append(' if printf "%s" "$ODYSSEUS_SERVE_CMD" | grep -q -- "--swap-space"; then') + runner_lines.append(' echo "[odysseus] vLLM serve does not expose --swap-space; removing the flag and patching the runtime default to 0."') + runner_lines.append(' ODYSSEUS_SERVE_CMD="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'') + runner_lines.append('import shlex, sys') + runner_lines.append('parts = shlex.split(sys.argv[1])') + runner_lines.append('out = []') + runner_lines.append('skip = False') + runner_lines.append('for part in parts:') + runner_lines.append(' if skip:') + runner_lines.append(' skip = False') + runner_lines.append(' continue') + runner_lines.append(' if part == "--swap-space":') + runner_lines.append(' skip = True') + runner_lines.append(' continue') + runner_lines.append(' if part.startswith("--swap-space="):') + runner_lines.append(' continue') + runner_lines.append(' out.append(part)') + runner_lines.append('print(shlex.join(out))') + runner_lines.append('PY') + runner_lines.append(')"') + runner_lines.append(' fi') + runner_lines.append(' ODYSSEUS_SERVE_CMD="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'') + runner_lines.append('import shlex, sys') + runner_lines.append('parts = shlex.split(sys.argv[1])') + runner_lines.append('patch = r"""import inspect, sys') + runner_lines.append('from vllm.engine.arg_utils import EngineArgs, AsyncEngineArgs') + runner_lines.append('def _odysseus_swap0(cls):') + runner_lines.append(' params = list(inspect.signature(cls).parameters)') + runner_lines.append(' if "swap_space" not in params:') + runner_lines.append(' return') + runner_lines.append(' idx = params.index("swap_space")') + runner_lines.append(' defaults = list(cls.__init__.__defaults__ or ())') + runner_lines.append(' if idx < len(defaults):') + runner_lines.append(' defaults[idx] = 0') + runner_lines.append(' cls.__init__.__defaults__ = tuple(defaults)') + runner_lines.append(' fields = getattr(cls, "__dataclass_fields__", {})') + runner_lines.append(' if "swap_space" in fields:') + runner_lines.append(' fields["swap_space"].default = 0') + runner_lines.append('_odysseus_swap0(EngineArgs)') + runner_lines.append('_odysseus_swap0(AsyncEngineArgs)') + runner_lines.append('try:') + runner_lines.append(' from vllm.config import CacheConfig') + runner_lines.append(' CacheConfig.swap_space = 0') + runner_lines.append('except Exception:') + runner_lines.append(' pass') + runner_lines.append('_orig_create_engine_config = EngineArgs.create_engine_config') + runner_lines.append('def _odysseus_create_engine_config(self, *args, **kwargs):') + runner_lines.append(' self.swap_space = 0') + runner_lines.append(' return _orig_create_engine_config(self, *args, **kwargs)') + runner_lines.append('EngineArgs.create_engine_config = _odysseus_create_engine_config') + runner_lines.append('AsyncEngineArgs.create_engine_config = _odysseus_create_engine_config') + runner_lines.append('from vllm.entrypoints.cli.main import main') + runner_lines.append('sys.exit(main())"""') + runner_lines.append('try:') + runner_lines.append(' serve_i = parts.index("serve")') + runner_lines.append('except ValueError:') + runner_lines.append(' print(shlex.join(parts))') + runner_lines.append('else:') + runner_lines.append(' exe_i = serve_i - 1') + runner_lines.append(' exe = parts[exe_i] if exe_i >= 0 else "vllm"') + runner_lines.append(' py = "python3"') + runner_lines.append(' if exe.endswith("/bin/vllm"):') + runner_lines.append(' py = exe[:-len("/bin/vllm")] + "/bin/python"') + runner_lines.append(' parts[exe_i:serve_i] = [py, "-c", patch]') + runner_lines.append(' print(shlex.join(parts))') + runner_lines.append('PY') + runner_lines.append(')"') + runner_lines.append(' echo "[odysseus] Patched vLLM internal swap_space default to 0 for this runtime."') runner_lines.append('fi') elif "sglang.launch_server" in req.cmd: runner_lines.append('export PATH="$HOME/.local/bin:$PATH"') - runner_lines.append('if ! python3 -c "import sglang" 2>/dev/null; then') - runner_lines.append(' echo "ERROR: SGLang is not installed. Open Cookbook -> Dependencies and install sglang on this server, then launch again."') - runner_lines.append(' exit 127') + runner_lines.append('if ! command -v sglang &>/dev/null; then') + runner_lines.append(' echo "ERROR: SGLang is not installed."') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') + runner_lines.append('elif ! ODYSSEUS_SGLANG_IMPORT_ERROR="$(python3 -c "import sglang" 2>&1)"; then') + runner_lines.append(' echo "ERROR: SGLang is installed but failed to import."') + runner_lines.append(' printf "%s\\n" "$ODYSSEUS_SGLANG_IMPORT_ERROR"') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') + runner_lines.append('fi') + elif "mlx_lm.server" in req.cmd: + runner_lines.append('export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"') + runner_lines.append('if ! ODYSSEUS_MLX_IMPORT_ERROR="$(python3 -c "import mlx_lm" 2>&1)"; then') + runner_lines.append(' echo "ERROR: MLX LM is not installed."') + runner_lines.append(' printf "%s\\n" "$ODYSSEUS_MLX_IMPORT_ERROR"') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') + runner_lines.append('fi') + runner_lines.append(f"ODYSSEUS_SERVE_CMD='{_bash_squote(req.cmd)}'") + runner_lines.append('if [ -z "$ODYSSEUS_PREFLIGHT_EXIT" ]; then') + runner_lines.append(' ODYSSEUS_MLX_CMD_PY="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'') + runner_lines.append('import shlex, sys') + runner_lines.append('parts = shlex.split(sys.argv[1])') + runner_lines.append('py = "python3"') + runner_lines.append('for i, part in enumerate(parts):') + runner_lines.append(' if part.endswith("/bin/python") or part.endswith("/bin/python3") or "/bin/python3." in part:') + runner_lines.append(' py = part') + runner_lines.append(' break') + runner_lines.append('print(py)') + runner_lines.append('PY') + runner_lines.append(')"') + runner_lines.append(' ODYSSEUS_SERVE_CMD="$("$ODYSSEUS_MLX_CMD_PY" - "$ODYSSEUS_SERVE_CMD" <<\'PY\'') + runner_lines.append('import json, os, shlex, sys') + runner_lines.append('from pathlib import Path') + runner_lines.append('parts = shlex.split(sys.argv[1])') + runner_lines.append('try:') + runner_lines.append(' i = parts.index("--model")') + runner_lines.append(' model = parts[i + 1]') + runner_lines.append('except Exception:') + runner_lines.append(' print(shlex.join(parts)); raise SystemExit') + runner_lines.append('if "/" in model and not model.startswith("/") and model.startswith("mlx-community/"):') + runner_lines.append(' roots = []') + runner_lines.append(' def add(p):') + runner_lines.append(' if not p: return') + runner_lines.append(' p = os.path.expanduser(p)') + runner_lines.append(' if p not in roots: roots.append(p)') + runner_lines.append(' add(os.environ.get("HUGGINGFACE_HUB_CACHE"))') + runner_lines.append(' hf_home = os.environ.get("HF_HOME")') + runner_lines.append(' if hf_home: add(os.path.join(hf_home, "hub"))') + runner_lines.append(' add("~/.cache/huggingface/hub")') + runner_lines.append(' cache_name = "models--" + model.replace("/", "--")') + runner_lines.append(' best = ""') + runner_lines.append(' best_mtime = -1.0') + runner_lines.append(' for root in roots:') + runner_lines.append(' snap_root = os.path.join(root, cache_name, "snapshots")') + runner_lines.append(' if not os.path.isdir(snap_root): continue') + runner_lines.append(' for name in os.listdir(snap_root):') + runner_lines.append(' path = os.path.join(snap_root, name)') + runner_lines.append(' if not os.path.isdir(path): continue') + runner_lines.append(' if not os.path.exists(os.path.join(path, "config.json")): continue') + runner_lines.append(' try: mtime = os.path.getmtime(path)') + runner_lines.append(' except OSError: mtime = 0') + runner_lines.append(' if mtime > best_mtime:') + runner_lines.append(' best, best_mtime = path, mtime') + runner_lines.append(' if best:') + runner_lines.append(' print("[odysseus] MLX using cached snapshot:", best, file=sys.stderr)') + runner_lines.append(' launch_model = best') + runner_lines.append(' if "deepseek-v4" in model.lower():') + runner_lines.append(' try:') + runner_lines.append(' import mlx_lm.models.deepseek_v4 as dsv4') + runner_lines.append(' import mlx_lm.utils as mlx_utils') + runner_lines.append(' utils_path = Path(mlx_utils.__file__)') + runner_lines.append(' utils_text = utils_path.read_text()') + runner_lines.append(' utils_needle = \' def class_predicate(p, m):\\n # Handle custom per layer quantizations\\n if p in config["quantization"]:\\n return config["quantization"][p]\\n if not hasattr(m, "to_quantized"):\\n return False\\n return f"{p}.scales" in weights\\n\'') + runner_lines.append(' utils_repl = \' def class_predicate(p, m):\\n # Odysseus: DeepSeek-V4 MXFP4 switch layers may already be quantized.\\n if type(m).__name__ == "QuantizedSwitchLinear":\\n return False\\n # Handle custom per layer quantizations\\n if p in config["quantization"]:\\n return config["quantization"][p]\\n if not hasattr(m, "to_quantized"):\\n return False\\n return f"{p}.scales" in weights\\n\'') + runner_lines.append(' if utils_repl not in utils_text and utils_needle in utils_text:') + runner_lines.append(' bak = utils_path.with_suffix(utils_path.suffix + ".odysseus_bak")') + runner_lines.append(' if not bak.exists(): bak.write_text(utils_text)') + runner_lines.append(' utils_path.write_text(utils_text.replace(utils_needle, utils_repl))') + runner_lines.append(' print("[odysseus] Patched MLX-LM QuantizedSwitchLinear double-quantization guard.", file=sys.stderr)') + runner_lines.append(' dsv4_path = Path(dsv4.__file__)') + runner_lines.append(' dsv4_text = dsv4_path.read_text()') + runner_lines.append(' dsv4_needle = \' for sub in ("attn", "ffn"):\\n for p in ("fn", "base", "scale"):\\n nk = nk.replace(f".hc_{sub}_{p}", f".hc_{sub}.{p}")\\n for wo, wn in w_remap.items():\\n\'') + runner_lines.append(' dsv4_repl = \' for sub in ("attn", "ffn"):\\n for p in ("fn", "base", "scale"):\\n nk = nk.replace(f".hc_{sub}_{p}", f".hc_{sub}.{p}")\\n # Odysseus: normalize alternate hyper-connection key aliases.\\n nk = nk.replace(".attn_hc.", ".hc_attn.")\\n nk = nk.replace(".ffn_hc.", ".hc_ffn.")\\n for wo, wn in w_remap.items():\\n\'') + runner_lines.append(' if dsv4_repl not in dsv4_text and dsv4_needle in dsv4_text:') + runner_lines.append(' bak = dsv4_path.with_suffix(dsv4_path.suffix + ".odysseus_bak")') + runner_lines.append(' if not bak.exists(): bak.write_text(dsv4_text)') + runner_lines.append(' dsv4_path.write_text(dsv4_text.replace(dsv4_needle, dsv4_repl))') + runner_lines.append(' print("[odysseus] Patched MLX-LM DeepSeek-V4 hyper-connection key aliases.", file=sys.stderr)') + runner_lines.append(' except Exception as e:') + runner_lines.append(' print("[odysseus] WARNING: failed to apply MLX DeepSeek-V4 compatibility patch:", e, file=sys.stderr)') + runner_lines.append(' try:') + runner_lines.append(' src = Path(best)') + runner_lines.append(' shim = Path.home() / ".cache" / "odysseus" / "mlx-shims" / src.name') + runner_lines.append(' if len(src.name) > 20:') + runner_lines.append(' shim = Path.home() / ".cache" / "odysseus" / "mlx-shims" / model.split("/")[-1]') + runner_lines.append(' shim.mkdir(parents=True, exist_ok=True)') + runner_lines.append(' for child in src.iterdir():') + runner_lines.append(' target = shim / child.name') + runner_lines.append(' if child.name == "tokenizer_config.json":') + runner_lines.append(' continue') + runner_lines.append(' if target.exists() or target.is_symlink():') + runner_lines.append(' continue') + runner_lines.append(' target.symlink_to(child)') + runner_lines.append(' tc_path = src / "tokenizer_config.json"') + runner_lines.append(' if tc_path.exists():') + runner_lines.append(' tc = json.loads(tc_path.read_text())') + runner_lines.append(' if tc.get("tool_parser_type") == "deepseek_v4":') + runner_lines.append(' tc.pop("tool_parser_type", None)') + runner_lines.append(' (shim / "tokenizer_config.json").write_text(json.dumps(tc, indent=2, ensure_ascii=False))') + runner_lines.append(' launch_model = str(shim)') + runner_lines.append(' print("[odysseus] MLX DeepSeek-V4 using sanitized shim:", launch_model, file=sys.stderr)') + runner_lines.append(' except Exception as e:') + runner_lines.append(' print("[odysseus] WARNING: failed to create MLX DeepSeek-V4 shim:", e, file=sys.stderr)') + runner_lines.append(' parts[i + 1] = launch_model') + runner_lines.append(' else:') + runner_lines.append(' print("[odysseus] MLX cached snapshot not found for:", model, file=sys.stderr)') + runner_lines.append('print(shlex.join(parts))') + runner_lines.append('PY') + runner_lines.append(')"') + runner_lines.append('fi') + elif "scripts/diffusion_server.py" in req.cmd or ".diffusion_server.py" in req.cmd: + runner_lines.append('export PATH="$HOME/.local/bin:$PATH"') + runner_lines.append('if ! ODYSSEUS_DIFFUSION_IMPORT_ERROR="$(python3 -c "import torch, diffusers" 2>&1)"; then') + runner_lines.append(' echo "ERROR: Diffusion serving requires PyTorch + diffusers."') + runner_lines.append(' printf "%s\\n" "$ODYSSEUS_DIFFUSION_IMPORT_ERROR"') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') runner_lines.append('fi') - runner_lines.append(req.cmd) - # Keep shell open after exit so user can see errors - runner_lines.append('echo ""; echo "=== Process exited with code $? ==="; exec "${SHELL:-/bin/bash}"') + handled_ollama_sidecar_probe = False + if (not handled_ollama_serve + and re.search(r"\bdocker\s+exec\s+(?:ollama-rocm|ollama-test)\s+ollama\s+show\b", req.cmd or "")): + handled_ollama_sidecar_probe = True + _append_serve_preflight_exit_lines( + runner_lines, + keep_shell_open=not local_windows, + ) + runner_lines.append(req.cmd) + runner_lines.append('_ody_exit=$?') + runner_lines.append('echo') + runner_lines.append('echo "=== Process exited with code ${_ody_exit} ==="') + runner_lines.append('if [ "$_ody_exit" -eq 0 ]; then') + runner_lines.append(' echo "[odysseus] Ollama sidecar model is available; keeping Cookbook task attached to the persistent Ollama daemon."') + runner_lines.append(' while true; do sleep 3600; done') + runner_lines.append('fi') + runner_lines.append('exec bash -i') + + if not handled_ollama_serve and not handled_ollama_sidecar_probe: + _append_serve_preflight_exit_lines( + runner_lines, + keep_shell_open=not local_windows, + ) + if "vllm serve" in req.cmd or "mlx_lm.server" in req.cmd: + runner_lines.append('eval "$ODYSSEUS_SERVE_CMD"') + elif is_pip_install: + if not is_windows and (req.platform or "").lower() in {"darwin", "macos"}: + req.cmd = _pip_install_command_without_break_system_packages(req.cmd) + _append_pip_install_runner_lines(runner_lines, req.cmd) + else: + runner_lines.append(req.cmd) + if local_windows: + # Detached background process — no interactive shell to keep open. + # Print the exit marker the status poller looks for, then stop. + _append_serve_exit_code_lines( + runner_lines, + keep_shell_open=False, + is_pip_install=is_pip_install, + ) + else: + # Keep shell open after exit so user can see errors + _append_serve_exit_code_lines( + runner_lines, + keep_shell_open=True, + is_pip_install=is_pip_install, + ) runner_path = TMUX_LOG_DIR / f"{session_id}_run.sh" - runner_path.write_text("\n".join(runner_lines) + "\n") - runner_path.chmod(0o755) + runner_path.write_text("\n".join(runner_lines) + "\n", encoding="utf-8") + # chmod is a no-op on Windows; bash on Windows runs the script + # regardless of the executable bit. + safe_chmod(runner_path, 0o755) - if remote: + if local_windows: + # LOCAL Windows: launch the bash runner detached (tmux replacement). + setup_cmd = None + elif remote: remote_runner = f".{session_id}_run.sh" # If command references scripts/, scp those too scp_extras = "" @@ -913,34 +2551,68 @@ async def model_serve(request: Request, req: ServeRequest): if diff_script.exists(): scp_extras = f"scp -O {_Pf}-q '{diff_script}' {remote}:.diffusion_server.py && " runner_path.write_text( - runner_path.read_text().replace( + runner_path.read_text(encoding="utf-8").replace( "scripts/diffusion_server.py", ".diffusion_server.py" - ) + ), + encoding="utf-8", ) setup_cmd = ( f"{scp_extras}" f"scp -O {_Pf}-q '{runner_path}' {remote}:{remote_runner} && " - f"ssh {_pf}{remote} 'chmod +x {remote_runner} && tmux new-session -d -s {session_id} \"./{remote_runner}\"'" + f"ssh {_pf}{remote} {shlex.quote(_remote_tmux_launch_command(session_id, remote_runner))}" ) else: - setup_cmd = f"tmux new-session -d -s {session_id} {shlex.quote(str(runner_path))}" + setup_cmd = f"tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} {shlex.quote(str(runner_path))}" - proc = await asyncio.create_subprocess_shell( - setup_cmd, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - await proc.wait() + if setup_cmd is None: + # LOCAL Windows: launch the bash runner detached; no tmux setup_cmd. + try: + _launch_local_detached(session_id, runner_lines) + except Exception as e: + logger.error(f"Local detached serve launch failed: {e}") + return {"ok": False, "error": str(e), "session_id": session_id} + else: + proc = await asyncio.create_subprocess_shell( + setup_cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + await proc.wait() - if proc.returncode != 0: - stderr = (await proc.stderr.read()).decode(errors="replace") - return {"ok": False, "error": stderr, "session_id": session_id} + if proc.returncode != 0: + stderr = (await proc.stderr.read()).decode(errors="replace") + return {"ok": False, "error": stderr, "session_id": session_id} - # Auto-register as model endpoint if serving a diffusion model + # Auto-register a model endpoint so the served model shows up in the model + # picker with no manual /setup step. Diffusion models get an image + # endpoint; any other real model serve (i.e. not a pip-install task) gets + # a local LLM endpoint pointed at its /v1. endpoint_id = None is_diffusion = "diffusion_server.py" in req.cmd if is_diffusion: endpoint_id = _auto_register_image_endpoint(req, remote) + elif not is_pip_install: + endpoint_id = _auto_register_llm_endpoint(req, remote) + + # Crash watchdog: the auto-register above writes the endpoint row + # IMMEDIATELY (before the server has even bound its port) so the + # picker shows the model as it warms up. When the serve process + # crashes right at startup (missing module, bad cmd, port collision, + # ModuleNotFoundError on llama_cpp, etc.), the endpoint is left + # dangling — every subsequent chat returns 503 or an empty response. + # Schedule a background task to read the tmux output for the + # "=== Process exited with code N ===" marker the runner emits; + # if N != 0 within the watch window, delete the endpoint we just + # created. Skipped for diffusion (different image-endpoint cleanup + # path) and pip-install tasks (no endpoint to drop). + if endpoint_id and not is_diffusion and not is_pip_install: + asyncio.create_task(_serve_crash_watchdog( + endpoint_id=endpoint_id, + session_id=session_id, + remote=remote, + ssh_port=req.ssh_port, + is_windows=is_windows, + )) # Log to assistant try: @@ -969,12 +2641,11 @@ class SetupRequest(BaseModel): async def server_setup(request: Request, req: SetupRequest): """Install required dependencies on a remote server via SSH.""" require_admin(request) - host = _validate_remote_host(req.host) + host = validate_remote_host(req.host) if not host: raise HTTPException(400, "host is required") port = req.ssh_port - if port is not None and port != "" and not re.fullmatch(r"\d{1,5}", port): - raise HTTPException(400, "Invalid ssh_port") + port = validate_ssh_port(port) pf = f"-p {port} " if port and port != "22" else "" # Detect platform: Windows first (echo %OS% → Windows_NT), then Termux, then Linux @@ -1145,6 +2816,25 @@ async def _probe_amd_sysfs(host: str | None, ssh_port: str | None) -> list[dict] out, err = await _run_gpu_shell("ls -1 /sys/class/drm 2>/dev/null", host, ssh_port, timeout=4) if err is not None or not out: return [] + # Pick the runtime label up-front so each GPU dict gets the + # right `backend`. AMD silicon can be driven by ROCm/HIP (native) + # OR Vulkan (mesa RADV). Reporting "rocm" on a host where no + # ROCm toolchain is installed misleads the frontend env-var + # prefix logic — it would emit `HIP_VISIBLE_DEVICES=` for a + # Vulkan-only stack, which is a silent no-op at best. + rt_out, _ = await _run_gpu_shell( + 'command -v rocminfo >/dev/null 2>&1 && echo rocm ' + '|| (command -v hipconfig >/dev/null 2>&1 && echo rocm) ' + '|| (command -v vulkaninfo >/dev/null 2>&1 && echo vulkan) ' + '|| echo unknown', + host, ssh_port, timeout=4, + ) + _amd_runtime = (rt_out or "").strip().splitlines()[-1:][0].strip() if rt_out else "rocm" + if _amd_runtime not in ("rocm", "vulkan"): + # Default to rocm so existing ROCm-installed hosts keep + # working; "unknown" only happens when neither toolchain is + # detected (e.g. minimal sysfs read on a fresh box). + _amd_runtime = "rocm" gpus = [] for entry in out.split(): if not entry.startswith("card") or "-" in entry: @@ -1177,11 +2867,18 @@ async def _probe_amd_sysfs(host: str | None, ssh_port: str | None) -> list[dict] total_mb = max(0, int(total_bytes / (1024 * 1024))) used_mb = max(0, min(total_mb, int(used_bytes / (1024 * 1024)))) free_mb = max(0, total_mb - used_mb) + # GTT = the system-RAM pool the GPU pages into when VRAM is full. + # On a discrete card a large gtt_used means the model spilled past + # VRAM into RAM over PCIe — much slower. Surface it so the UI can + # warn "spilling to RAM" instead of the user wondering why it's slow. + gtt_used_raw = await _gpu_read_file(f"{base}/mem_info_gtt_used", host, ssh_port) + gtt_used_mb = max(0, int(int(gtt_used_raw) / (1024 * 1024))) if (gtt_used_raw and gtt_used_raw.isdigit()) else 0 gpus.append({ "index": len(gpus), "name": name, "uuid": entry, "free_mb": free_mb, "total_mb": total_mb, "used_mb": used_mb, + "gtt_used_mb": gtt_used_mb, "util_pct": 0, "busy": bool(total_mb and (free_mb / total_mb) < 0.85), - "processes": [], "backend": "rocm", "source": "amd-sysfs", + "processes": [], "backend": _amd_runtime, "source": "amd-sysfs", "unified_memory": unified, }) if gpus: @@ -1191,6 +2888,59 @@ async def _probe_amd_sysfs(host: str | None, ssh_port: str | None) -> list[dict] gpus[0]["busy"] = True return gpus + async def _probe_apple_unified_memory(host: str | None, ssh_port: str | None) -> dict | None: + """Best-effort Apple Silicon unified-memory probe, local or over SSH.""" + cmd = ( + "uname -s; uname -m; " + "sysctl -n hw.memsize 2>/dev/null || true; " + "vm_stat 2>/dev/null | awk '" + "/page size of/ {gsub(/[^0-9]/, \"\", $8); page=$8} " + "/Pages free/ {gsub(/[^0-9]/, \"\", $3); free=$3} " + "/Pages inactive/ {gsub(/[^0-9]/, \"\", $3); inactive=$3} " + "/Pages speculative/ {gsub(/[^0-9]/, \"\", $3); speculative=$3} " + "/Pages purgeable/ {gsub(/[^0-9]/, \"\", $3); purgeable=$3} " + "END {if (!page) page=16384; print page, free+inactive+speculative+purgeable}'" + ) + out, err = await _run_gpu_shell(cmd, host, ssh_port, timeout=6) + if err is not None or not out: + return None + lines = [ln.strip() for ln in out.splitlines() if ln.strip()] + if len(lines) < 4: + return None + if lines[0] != "Darwin" or lines[1] not in {"arm64", "arm64e"}: + return None + try: + total_bytes = int(lines[2]) + page_parts = lines[3].split() + page_size = int(page_parts[0]) + available_pages = int(page_parts[1]) + except (ValueError, IndexError): + return None + if total_bytes <= 0: + return None + total_mb = int(total_bytes / (1024 * 1024)) + free_mb = max(0, min(total_mb, int((page_size * available_pages) / (1024 * 1024)))) + used_mb = max(0, total_mb - free_mb) + return { + "ok": True, + "gpus": [{ + "index": 0, + "name": "Apple Silicon unified memory", + "uuid": "apple-metal-0", + "free_mb": free_mb, + "total_mb": total_mb, + "used_mb": used_mb, + "util_pct": 0, + "busy": bool(total_mb and (free_mb / total_mb) < 0.2), + "processes": [], + "backend": "metal", + "source": "apple-vm-stat", + "unified_memory": True, + }], + "backend": "metal", + "source": "apple-vm-stat", + } + @router.get("/api/cookbook/gpus") async def list_gpus(request: Request, host: str | None = None, ssh_port: str | None = None): """Probe GPU memory/process state locally or via SSH. @@ -1211,9 +2961,8 @@ async def list_gpus(request: Request, host: str | None = None, ssh_port: str | N `busy` is True when free_mb/total_mb < 0.5. """ require_admin(request) - host = _validate_remote_host(host) - if ssh_port is not None and ssh_port != "" and not _SSH_PORT_RE.fullmatch(ssh_port): - raise HTTPException(400, "Invalid ssh_port") + host = validate_remote_host(host) + ssh_port = validate_ssh_port(ssh_port) gpu_query = "nvidia-smi --query-gpu=index,name,memory.free,memory.total,memory.used,utilization.gpu,uuid --format=csv,noheader,nounits" nvidia_error = None try: @@ -1281,12 +3030,63 @@ async def list_gpus(request: Request, host: str | None = None, ssh_port: str | N if gpus: return {"ok": True, "gpus": gpus, "backend": "cuda", "source": "nvidia-smi"} + # Local Apple Silicon / Metal fallback. macOS has no nvidia-smi and no + # Linux /sys/class/drm tree, but services.hwfit.hardware already knows + # how to size the shared unified-memory GPU budget. Keep this route in + # sync so Cookbook's GPU picker doesn't show "nvidia-smi not found" on + # native Mac launches. + if not host and sys.platform == "darwin": + try: + from services.hwfit.hardware import detect_system + info = detect_system(fresh=True) + backend = str(info.get("backend") or "").lower() + if backend in {"metal", "mps", "apple"} and info.get("gpu_count", 0) > 0: + total_mb = int(float(info.get("gpu_vram_gb") or info.get("total_ram_gb") or 0) * 1024) + free_mb = int(float(info.get("available_ram_gb") or 0) * 1024) + if total_mb and (free_mb <= 0 or free_mb > total_mb): + free_mb = total_mb + used_mb = max(0, total_mb - max(0, free_mb)) + return { + "ok": True, + "gpus": [{ + "index": 0, + "name": info.get("gpu_name") or info.get("cpu_name") or "Apple Silicon GPU", + "uuid": "apple-metal-0", + "free_mb": max(0, free_mb), + "total_mb": max(0, total_mb), + "used_mb": used_mb, + "util_pct": 0, + "busy": bool(total_mb and (free_mb / total_mb) < 0.5), + "processes": [], + "backend": "metal", + "source": "apple-metal", + "unified_memory": True, + }], + "backend": "metal", + "source": "apple-metal", + "fallback_from": "nvidia-smi", + "nvidia_error": nvidia_error, + } + except Exception as e: + logger.warning("Apple Metal GPU fallback failed: %s", e) + + apple_gpus = await _probe_apple_unified_memory(host, ssh_port) + if apple_gpus: + apple_gpus["fallback_from"] = "nvidia-smi" + apple_gpus["nvidia_error"] = nvidia_error + return apple_gpus + amd_gpus = await _probe_amd_sysfs(host, ssh_port) if amd_gpus: + # The per-GPU dict already carries the runtime label picked by + # _probe_amd_sysfs (rocm vs vulkan); mirror that into the + # wrapper so the frontend can read `data.backend` directly + # without scanning the list. + _amd_wrap_backend = str(amd_gpus[0].get("backend") or "rocm") return { "ok": True, "gpus": amd_gpus, - "backend": "rocm", + "backend": _amd_wrap_backend, "source": "amd-sysfs", "fallback_from": "nvidia-smi", "nvidia_error": nvidia_error, @@ -1330,9 +3130,8 @@ async def kill_pid(request: Request, req: KillPidRequest): sig = (req.signal or "TERM").upper() if sig not in ("TERM", "KILL", "INT"): raise HTTPException(400, "signal must be TERM, KILL, or INT") - host = _validate_remote_host(req.host) - if req.ssh_port and not _SSH_PORT_RE.fullmatch(req.ssh_port): - raise HTTPException(400, "Invalid ssh_port") + host = validate_remote_host(req.host) + req.ssh_port = validate_ssh_port(req.ssh_port) kill_cmd = f"kill -{sig} {req.pid}" try: if host: @@ -1341,6 +3140,16 @@ async def kill_pid(request: Request, req: KillPidRequest): proc = await asyncio.create_subprocess_shell( cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE ) + elif IS_WINDOWS: + # No `kill` binary / POSIX signals on Windows. taskkill /F /T tears + # down the PID and its children. There's no graceful-vs-force + # distinction, so TERM/KILL/INT all map to the same forced kill. + # NB: never use os.kill(pid, 0) to probe here — on Windows that + # routes to TerminateProcess and would kill the process. + if not pid_alive(req.pid): + return {"ok": False, "error": f"PID {req.pid} is not running"} + await asyncio.to_thread(kill_process_tree, req.pid) + return {"ok": True, "pid": req.pid, "signal": sig} else: proc = await asyncio.create_subprocess_exec( "kill", f"-{sig}", str(req.pid), @@ -1362,12 +3171,29 @@ async def kill_pid(request: Request, req: KillPidRequest): async def get_cookbook_state(request: Request): """Load saved cookbook state (tasks, servers, presets, settings).""" require_admin(request) + now = time.monotonic() + try: + mtime = _cookbook_state_path.stat().st_mtime if _cookbook_state_path.exists() else 0.0 + except Exception: + mtime = 0.0 + cached = _state_get_cache.get("value") + if cached is not None and _state_get_cache.get("mtime") == mtime and now - float(_state_get_cache.get("ts") or 0) < 1.5: + return cached if _cookbook_state_path.exists(): try: - return _state_for_client(json.loads(_cookbook_state_path.read_text())) + state = json.loads(_cookbook_state_path.read_text(encoding="utf-8")) + saved_tasks = state.get("tasks", []) + tasks = saved_tasks if isinstance(saved_tasks, list) else list(saved_tasks.values()) if isinstance(saved_tasks, dict) else [] + client_state = _state_for_client(state) + _state_get_cache.update({"ts": now, "mtime": mtime, "value": client_state}) + return client_state except Exception: - return {} - return {} + client_state = _state_for_client({}) + _state_get_cache.update({"ts": now, "mtime": mtime, "value": client_state}) + return client_state + client_state = _state_for_client({}) + _state_get_cache.update({"ts": now, "mtime": mtime, "value": client_state}) + return client_state @router.post("/api/cookbook/state") async def save_cookbook_state(request: Request): @@ -1393,7 +3219,7 @@ async def save_cookbook_state(request: Request): data = {} try: if _cookbook_state_path.exists(): - on_disk = json.loads(_cookbook_state_path.read_text()) + on_disk = json.loads(_cookbook_state_path.read_text(encoding="utf-8")) else: on_disk = {} except Exception: @@ -1417,6 +3243,44 @@ async def save_cookbook_state(request: Request): disk_tasks = on_disk.get("tasks") or [] if isinstance(on_disk, dict) else [] incoming_tasks = data.get("tasks") if isinstance(data.get("tasks"), list) else [] + incoming_removed = data.get("removedTasks") if isinstance(data.get("removedTasks"), dict) else {} + disk_removed = on_disk.get("removedTasks") if isinstance(on_disk, dict) and isinstance(on_disk.get("removedTasks"), dict) else {} + removed_tasks = {**disk_removed, **incoming_removed} + data["removedTasks"] = removed_tasks + removed_ids = set(removed_tasks.keys()) + if removed_ids: + incoming_tasks = [ + t for t in incoming_tasks + if not (isinstance(t, dict) and t.get("sessionId") in removed_ids) + ] + data["tasks"] = incoming_tasks + # Anti-poisoning guard: a stale browser tab can keep POSTing a + # download task as status='done' from before the strict-finish + # fix landed, undoing any server-side correction. For each + # incoming "done" download, override to "running" if the last + # shard pattern says N _completed: + logger.info(f"cookbook state POST: rejecting stale done for {_it.get('sessionId')} " + f"({_completed}/{_starts} files complete, no DOWNLOAD_OK)") + _it["status"] = "running" incoming_ids = {t.get("sessionId") for t in incoming_tasks if isinstance(t, dict) and t.get("sessionId")} import time as _t now_ms = int(_t.time() * 1000) @@ -1427,6 +3291,8 @@ async def save_cookbook_state(request: Request): sid = t.get("sessionId") if not sid or sid in incoming_ids: continue # client's version wins + if sid in removed_ids: + continue # intentional cross-device clear/remove ts = t.get("ts") or 0 if isinstance(ts, (int, float)) and (now_ms - ts) <= RACE_WINDOW_MS: preserved.append(t) @@ -1435,7 +3301,19 @@ async def save_cookbook_state(request: Request): f"not in incoming body (race guard): " f"{[t.get('sessionId') for t in preserved]}") data["tasks"] = incoming_tasks + preserved - atomic_write_json(str(_cookbook_state_path), _state_for_storage(data, on_disk), indent=2) + storage_state = _state_for_storage(data, on_disk) + if storage_state == on_disk: + return {"ok": True, "preserved": len(preserved), "unchanged": True} + atomic_write_json(str(_cookbook_state_path), storage_state, indent=2) + try: + mtime = _cookbook_state_path.stat().st_mtime + _state_get_cache.update({ + "ts": time.monotonic(), + "mtime": mtime, + "value": _state_for_client(storage_state), + }) + except Exception: + pass return {"ok": True, "preserved": len(preserved)} except Exception as e: return {"ok": False, "error": str(e)} @@ -1533,12 +3411,14 @@ def _is_excluded(repo_id: str, tags: list) -> bool: # Add 30% headroom for KV cache, activations, etc. needed_vram = (est_vram * 1.3) if est_vram else None - if vram_gb > 0 and needed_vram is not None and needed_vram > vram_gb: - continue - # Skip if no size info — without a size we can't tell if it's a real - # full-weight model or a tiny adapter, so we'd rather drop it - if est_vram is None: - continue + if vram_gb > 0: + if needed_vram is None: + # The "trending models that fit" list must be conservative: + # if we cannot estimate size from the repo id/tags, do not + # present it as runnable on this hardware. + continue + if needed_vram > vram_gb: + continue out.append({ "repo_id": repo_id, @@ -1555,6 +3435,567 @@ def _is_excluded(repo_id: str, tags: list) -> bool: return {"models": out} + # Rate-limit for the orphan-tmux adoption sweep. Five-minute interval so SSH + # work is genuinely sparse even on an actively-polled cookbook page. + _last_orphan_sweep_ts = [0.0] + _ORPHAN_SWEEP_MIN_INTERVAL_S = 300.0 + # Concurrency guard so two requests racing don't both spawn a sweep. + _orphan_sweep_inflight = [False] + + def _maybe_sweep_orphans(tasks: list, state: dict) -> None: + """Scan each configured cookbook server for `serve-*` tmux sessions + the cookbook doesn't know about and adopt them into state.tasks. + + Heavy SSH work runs in a background thread via asyncio.to_thread so + it never blocks the request that triggered it. Was previously + disabled because the sync implementation pegged uvicorn CPU during + active cookbook polling — re-enabled now with the work pushed off + the event loop and a slower (60s) cadence. + """ + import time as _time + now = _time.monotonic() + if _orphan_sweep_inflight[0]: + return + if now - _last_orphan_sweep_ts[0] < _ORPHAN_SWEEP_MIN_INTERVAL_S: + return + _last_orphan_sweep_ts[0] = now + _orphan_sweep_inflight[0] = True + # Snapshot inputs so the worker doesn't race with state mutations. + try: + tasks_snap = list(tasks or []) + except Exception: + tasks_snap = [] + state_snap = state if isinstance(state, dict) else {} + + # Caller is _cookbook_tasks_status_sync (sync context, no event + # loop). Use a plain background thread — no asyncio needed. + import threading + def _run_sweep() -> None: + try: + _sync_sweep_orphans(tasks_snap, state_snap) + except Exception as _e: + logger.warning(f"orphan sweep thread failed: {_e!r}") + finally: + _orphan_sweep_inflight[0] = False + try: + threading.Thread(target=_run_sweep, daemon=True, name="orphan-sweep").start() + except Exception as _e: + logger.warning(f"orphan sweep thread spawn failed: {_e!r}") + _orphan_sweep_inflight[0] = False + return + + def _sync_sweep_orphans(tasks: list, state: dict) -> None: + """The actual sync sweep — never call this on the event loop.""" + import subprocess + env = state.get("env") if isinstance(state, dict) else {} + servers = env.get("servers") if isinstance(env, dict) else [] + logger.info(f"orphan sweep starting: {len(servers) if isinstance(servers, list) else 0} server(s), known_sids={len([t for t in tasks if isinstance(t, dict) and t.get('sessionId')])}") + if not isinstance(servers, list): + return + + known_sids = { + t.get("sessionId") for t in tasks + if isinstance(t, dict) and t.get("sessionId") + } + + adopted_any = False + for srv in servers: + if not isinstance(srv, dict): + continue + host = (srv.get("host") or "").strip() + if not host: + continue # local-only entry; the /proc scan handles it + try: + host = validate_remote_host(host) + except HTTPException: + continue + sport = str(srv.get("port") or "").strip() + ssh_base = ["ssh", "-o", "ConnectTimeout=4", "-o", "StrictHostKeyChecking=no"] + if sport and sport != "22": + try: + sport = validate_ssh_port(sport) + except HTTPException: + continue + if sport != "22": + ssh_base.extend(["-p", sport]) + + try: + ls = subprocess.run( + ssh_base + [host, _remote_tmux_command("ls")], + timeout=6, capture_output=True, text=True, + ) + except Exception: + continue + for line in (ls.stdout or "").splitlines(): + sid = line.split(":", 1)[0].strip() + if not sid or not _SESSION_ID_RE.match(sid): + continue + if sid in known_sids: + continue + try: + cap = subprocess.run( + ssh_base + [host, _remote_tmux_command("capture-pane", "-t", sid, "-p", "-S", "-300")], + timeout=6, capture_output=True, text=True, + ) + pane = cap.stdout or "" + except Exception: + pane = "" + + if sid.startswith("cookbook-"): + repo_id = "" + try: + script = subprocess.run( + ssh_base + [host, "cat", f".{sid}_run.sh"], + timeout=6, capture_output=True, text=True, + ) + script_text = script.stdout or "" + except Exception: + script_text = "" + m_repo = re.search(r"repo_id\s*=\s*['\"]([^'\"]+/[^'\"]+)['\"]", script_text) + if not m_repo: + m_repo = re.search(r"snapshot_download\(\s*repo_id\s*=\s*['\"]([^'\"]+/[^'\"]+)['\"]", script_text) + if not m_repo: + m_repo = re.search(r"(?:https://huggingface\.co/)?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", script_text) + if not m_repo and pane: + m_cache = re.search(r"models--([A-Za-z0-9_.-]+)--([A-Za-z0-9_.-]+)", pane) + if m_cache: + repo_id = f"{m_cache.group(1)}/{m_cache.group(2)}" + repo_id = m_repo.group(1) if m_repo else f"adopted:{sid}" + adopted_download_done = "DOWNLOAD_OK" in pane + if repo_id.startswith("adopted:") and pane: + m_cache = re.search(r"models--([A-Za-z0-9_.-]+)--([A-Za-z0-9_.-]+)", pane) + if m_cache: + repo_id = f"{m_cache.group(1)}/{m_cache.group(2)}" + import time as _t2 + tasks.append({ + "id": sid, + "sessionId": sid, + "name": repo_id.split("/")[-1] if "/" in repo_id else repo_id, + "type": "download", + "status": "completed" if adopted_download_done else "running", + "progress": "Download complete" if adopted_download_done else "", + "output": (pane or f"Auto-adopted from orphan tmux download session on {host}.")[-5000:], + "ts": int(_t2.time() * 1000), + "completedAt": int(_t2.time() * 1000) if adopted_download_done else None, + "payload": { + "repo_id": repo_id, + "remote_host": host, + "_cmd": "(orphan tmux download - original launch cmd recovered from tmux/session only)", + }, + "remoteHost": host, + "sshPort": sport, + "platform": "linux", + "_cacheComplete": adopted_download_done, + "_adoptedExternally": True, + }) + known_sids.add(sid) + adopted_any = True + logger.info(f"auto-adopted orphan download tmux session {sid!r} on {host}") + continue + # Adopt any session whose pane is currently running a + # known model-server process (checked below). The earlier + # prefix gate (serve-/cookbook-) dropped legitimate + # serves whenever tmux fell back to numeric IDs, leaving + # them invisible in the Cookbook UI — so the user could + # neither see nor stop them. + # Skip zombie / idle-shell sessions. A tmux session left + # over from a crashed vllm just shows a bash prompt — + # adopting it would pollute the UI with "running" tasks + # that aren't actually serving anything. pane_current_command + # is the foreground process in the pane right now; only + # real model serves leave a python/vllm/etc. process there. + try: + pc = subprocess.run( + ssh_base + [host, "tmux", "list-panes", "-t", sid, + "-F", "#{pane_current_command}"], + timeout=4, capture_output=True, text=True, + ) + cur = (pc.stdout or "").strip().splitlines() + except Exception: + cur = [] + LIVE_PROCS = {"python", "python3", "vllm", "llama-server", + "llama_cpp_main", "sglang", "mlx_lm", "lmdeploy", + "ollama", "node", "uvicorn"} + if not any(c in LIVE_PROCS for c in cur): + continue + # Try to recover a plausible repo_id + port from the + # pane buffer. Cheap heuristic — if we can't, register + # with placeholder fields; the UI still shows it. + import re as _re_orphan + # vLLM banner: "model /path/...". Falls back to the + # raw vllm-serve command if the banner already scrolled. + m_model = _re_orphan.search(r"model\s+(\S+)", pane) + model = m_model.group(1) if m_model else "" + if not model: + m_serve = _re_orphan.search(r"vllm\s+serve\s+(\S+)", pane) + model = m_serve.group(1) if m_serve else f"adopted:{sid}" + m_port = _re_orphan.search(r"--port\s+(\d+)", pane) + port = int(m_port.group(1)) if m_port else 0 + + import time as _t2 + tasks.append({ + "id": sid, + "sessionId": sid, + "name": model.split("/")[-1] if "/" in model else model, + "type": "serve", + "status": "running", + "output": f"Auto-adopted from orphan tmux session on {host}. " + "Open the task to see live output.", + "ts": int(_t2.time() * 1000), + "payload": { + "repo_id": model, + "remote_host": host, + "_cmd": "(orphan tmux session — original launch cmd unknown)", + "port": port, + }, + "remoteHost": host, + "sshPort": sport, + "platform": "linux", + "_serveReady": False, + "_endpointAdded": False, + "_adoptedExternally": True, + }) + known_sids.add(sid) + adopted_any = True + logger.info(f"auto-adopted orphan tmux session {sid!r} on {host}") + + if adopted_any: + try: + from core.atomic_io import atomic_write_json + state["tasks"] = tasks + atomic_write_json(_cookbook_state_path, state) + except Exception as e: + logger.warning(f"orphan sweep: state write failed: {e}") + + @router.get("/api/cookbook/hf-gguf-files") + async def hf_gguf_files(repo_id: str, owner: str = Depends(require_user)): + """List GGUF files in a HuggingFace repo for the direct-download picker.""" + import httpx + + repo_id = _validate_repo_id(repo_id) + url = f"https://huggingface.co/api/models/{repo_id}" + try: + headers = {} + token = _load_stored_hf_token() + if token: + headers["Authorization"] = f"Bearer {token}" + async with httpx.AsyncClient(timeout=15, follow_redirects=True) as client: + resp = await client.get(url, headers=headers) + if resp.status_code != 200: + return {"ok": False, "files": [], "error": f"HF API HTTP {resp.status_code}"} + data = resp.json() + except Exception: + logger.exception("HF GGUF file scan failed for %s", repo) + return {"ok": False, "files": [], "error": "HF API request failed"} + files = [ + str(s.get("rfilename") or "") + for s in data.get("siblings", []) + if str(s.get("rfilename") or "").lower().endswith(".gguf") + ] + return {"ok": True, "repo_id": repo_id, "files": files} + + # In-memory cache for the Ollama library scrape. ollama.com is a public + # site, but it doesn't expose a stable JSON listing — we fetch the HTML + # search page and regex out the model cards. Cached for 1 h so a busy + # cookbook view doesn't hammer the site on every render. + _ollama_library_cache: dict = {"models": [], "fetched_at": 0.0, "error": None} + + _OLLAMA_FALLBACK_LIBRARY = [ + {"name": "qwen2.5", "description": "Qwen2.5 series — strong general/coding model from Alibaba.", "sizes": ["0.5b", "1.5b", "3b", "7b", "14b", "32b", "72b"]}, + {"name": "qwen2.5-coder", "description": "Code-specialized Qwen2.5 family.", "sizes": ["0.5b", "1.5b", "3b", "7b", "14b", "32b"]}, + {"name": "qwen3", "description": "Qwen3 — newer Alibaba family with hybrid reasoning.", "sizes": ["0.6b", "1.7b", "4b", "8b", "14b", "32b"]}, + {"name": "llama3.2", "description": "Meta Llama 3.2 instruct (and tiny / vision variants).", "sizes": ["1b", "3b", "11b", "90b"]}, + {"name": "llama3.1", "description": "Meta Llama 3.1 instruct.", "sizes": ["8b", "70b", "405b"]}, + {"name": "llama3.3", "description": "Meta Llama 3.3 70B instruct.", "sizes": ["70b"]}, + {"name": "gemma3", "description": "Google Gemma 3 — multimodal capable open-weights.", "sizes": ["1b", "4b", "12b", "27b"]}, + {"name": "gemma2", "description": "Google Gemma 2 instruct.", "sizes": ["2b", "9b", "27b"]}, + {"name": "mistral", "description": "Mistral 7B instruct — small, fast generalist.", "sizes": ["7b"]}, + {"name": "mistral-nemo", "description": "Mistral NeMo 12B instruct.", "sizes": ["12b"]}, + {"name": "mistral-small", "description": "Mistral Small 22B / 24B instruct.", "sizes": ["22b", "24b"]}, + {"name": "mixtral", "description": "Mistral MoE 8x7B / 8x22B.", "sizes": ["8x7b", "8x22b"]}, + {"name": "phi3", "description": "Microsoft Phi-3 small / medium.", "sizes": ["mini", "medium"]}, + {"name": "phi4", "description": "Microsoft Phi-4 14B.", "sizes": ["14b"]}, + {"name": "deepseek-r1", "description": "DeepSeek R1 reasoning model (distilled variants).", "sizes": ["1.5b", "7b", "8b", "14b", "32b", "70b"]}, + {"name": "deepseek-v3", "description": "DeepSeek V3 MoE 671B (huge — needs serious VRAM).", "sizes": ["671b"]}, + {"name": "codellama", "description": "Meta Code Llama instruct family.", "sizes": ["7b", "13b", "34b", "70b"]}, + {"name": "starcoder2", "description": "BigCode StarCoder2 — code completion.", "sizes": ["3b", "7b", "15b"]}, + {"name": "deepseek-coder-v2", "description": "DeepSeek Coder V2 — code MoE.", "sizes": ["16b", "236b"]}, + {"name": "nomic-embed-text", "description": "Embedding model — text vector encoder.", "sizes": ["latest"]}, + {"name": "mxbai-embed-large", "description": "Embedding model — Mixedbread large.", "sizes": ["latest"]}, + {"name": "llava", "description": "LLaVA multimodal vision-language model.", "sizes": ["7b", "13b", "34b"]}, + {"name": "minicpm-v", "description": "MiniCPM-V multimodal.", "sizes": ["8b"]}, + {"name": "command-r", "description": "Cohere Command R — RAG-oriented.", "sizes": ["35b"]}, + {"name": "command-r-plus", "description": "Cohere Command R+ — larger RAG model.", "sizes": ["104b"]}, + {"name": "qwq", "description": "Qwen QwQ reasoning preview.", "sizes": ["32b"]}, + {"name": "smollm2", "description": "HuggingFaceTB SmolLM2 — tiny capable models.", "sizes": ["135m", "360m", "1.7b"]}, + {"name": "granite3.1-dense", "description": "IBM Granite 3.1 dense instruct.", "sizes": ["2b", "8b"]}, + {"name": "nemotron", "description": "NVIDIA Nemotron 70B.", "sizes": ["70b"]}, + {"name": "olmo2", "description": "AI2 OLMo 2 open-weights.", "sizes": ["7b", "13b"]}, + ] + + @router.get("/api/cookbook/ollama/library") + async def ollama_library(refresh: int = 0, request: Request = None, owner: str = Depends(require_user)): + """List popular Ollama library models for the Browse picker. + + Tries a 1-hour-cached fetch of ollama.com/library, falls back to a + curated hard-coded list so the picker always renders something.""" + import time as _time + import httpx as _httpx + TTL = 3600.0 + now = _time.time() + if refresh or (now - _ollama_library_cache["fetched_at"]) > TTL or not _ollama_library_cache["models"]: + models: list[dict] = [] + err = None + try: + async with _httpx.AsyncClient(timeout=8, follow_redirects=True) as client: + resp = await client.get( + "https://ollama.com/search?sort=popular", + headers={"User-Agent": "odysseus-cookbook/1.0"}, + ) + if resp.status_code == 200: + html = resp.text + # ollama.com renders each model card as a single anchor: + # + # The description + sizes live inside that anchor. Pull + # the whole block then extract pieces individually. + block_re = re.compile( + r']*href="/library/([A-Za-z0-9._-]+)"[^>]*>(.*?)', + re.DOTALL, + ) + desc_re = re.compile(r']*>([^<]{4,400})

', re.DOTALL) + # Size tags on ollama.com cards look like "0.5b", "14b", + # "8x7b", "27b". Pulled from short -wrapped chips. + size_re = re.compile(r'>\s*(\d+(?:\.\d+)?(?:x\d+)?[bBmM])\s*<') + seen: set[str] = set() + for bm in block_re.finditer(html): + name = bm.group(1).strip() + if name in seen: + continue + seen.add(name) + body = bm.group(2) + dm = desc_re.search(body) + desc = (dm.group(1).strip() if dm else "").replace("\n", " ") + sizes_raw = size_re.findall(body) + # Dedup sizes preserving order + sizes: list[str] = [] + for s in sizes_raw: + s_low = s.lower() + if s_low not in sizes: + sizes.append(s_low) + models.append({"name": name, "description": desc, "sizes": sizes}) + if len(models) >= 80: + break + else: + err = f"HTTP {resp.status_code}" + except Exception as e: + err = str(e)[:160] + # Merge curated fallback so classics (qwen2.5, llama3, deepseek-r1, + # …) stay reachable even when ollama.com's front page is dominated + # by brand-new releases the user might not be looking for. + live_names = {m["name"] for m in models} + for fb in _OLLAMA_FALLBACK_LIBRARY: + if fb["name"] not in live_names: + models.append(fb) + if not models: + models = list(_OLLAMA_FALLBACK_LIBRARY) + if err is None: + err = "parsed 0 results — using fallback list" + _ollama_library_cache["models"] = models + _ollama_library_cache["fetched_at"] = now + _ollama_library_cache["error"] = err + return { + "models": _ollama_library_cache["models"], + "fetched_at": _ollama_library_cache["fetched_at"], + "error": _ollama_library_cache["error"], + } + + # ── vLLM recipe scraper ───────────────────────────────────────────── + # Fetches the official YAML recipe for a model from vllm-project/recipes + # and normalizes it into a small JSON the frontend can consume. Cached + # per-repo so the GitHub raw endpoint isn't hammered. + _vllm_recipe_cache: dict[str, tuple[float, dict | None]] = {} + # Manifest of all / ids that have a recipe in the upstream + # repo. Cheap to fetch (one Git Tree API call), so we cache the whole + # set for ~12h. Per-row "does this model have a recipe?" lookups hit + # this set instead of doing 912 individual recipe fetches. + _vllm_recipe_manifest: dict = {"fetched_at": 0.0, "models": set(), "error": ""} + + @router.get("/api/cookbook/vllm-recipe-manifest") + async def vllm_recipe_manifest(refresh: int = 0): + """Return the set of / ids known to have a vLLM recipe. + One GitHub Tree API call, 12h cache. The frontend uses this to badge + rows in the model list before the user expands them.""" + import time as _time + import httpx as _httpx + TTL = 12 * 3600.0 + now = _time.time() + if ( + refresh + or (now - _vllm_recipe_manifest["fetched_at"]) > TTL + or not _vllm_recipe_manifest["models"] + ): + url = ( + "https://api.github.com/repos/vllm-project/recipes/" + "git/trees/main?recursive=1" + ) + def _fetch_sync() -> tuple[int, dict | None, str]: + try: + headers = {"Accept": "application/vnd.github+json"} + with _httpx.Client(timeout=10.0, follow_redirects=True) as client: + r = client.get(url, headers=headers) + if r.status_code != 200: + return r.status_code, None, r.text[:200] + return 200, r.json(), "" + except Exception as e: + return 0, None, f"fetch error: {e}" + status, data, err = await asyncio.to_thread(_fetch_sync) + if status == 200 and isinstance(data, dict): + models: set[str] = set() + for entry in data.get("tree") or []: + path = (entry or {}).get("path") or "" + if not path.startswith("models/") or not path.endswith(".yaml"): + continue + # path = "models//.yaml" → "/" + body = path[len("models/"):-len(".yaml")] + if "/" in body: + models.add(body) + _vllm_recipe_manifest["models"] = models + _vllm_recipe_manifest["fetched_at"] = now + _vllm_recipe_manifest["error"] = "" + else: + _vllm_recipe_manifest["error"] = ( + f"HTTP {status}: {err}" if status else err + ) + # Don't clobber a stale-but-usable list on transient failures. + if not _vllm_recipe_manifest["models"]: + return { + "models": [], + "count": 0, + "error": _vllm_recipe_manifest["error"], + } + return { + "models": sorted(_vllm_recipe_manifest["models"]), + "count": len(_vllm_recipe_manifest["models"]), + "fetched_at": _vllm_recipe_manifest["fetched_at"], + "error": _vllm_recipe_manifest["error"], + } + + @router.get("/api/cookbook/vllm-recipe") + async def vllm_recipe(repo: str, refresh: int = 0): + """Return the vLLM official recipe for a HuggingFace repo, if one + exists at vllm-project/recipes. `repo` is the full HF id like + 'MiniMaxAI/MiniMax-M2'. Cached 6h.""" + import time as _time + import httpx as _httpx + import yaml as _yaml + + TTL = 6 * 3600.0 + now = _time.time() + repo = (repo or "").strip().strip("/") + if "/" not in repo: + return {"exists": False, "error": "repo must be /"} + + cached = _vllm_recipe_cache.get(repo) + if cached and not refresh and (now - cached[0]) < TTL: + return cached[1] or {"exists": False, "cached": True} + + url = ( + f"https://raw.githubusercontent.com/vllm-project/recipes/" + f"main/models/{repo}.yaml" + ) + + def _fetch_sync() -> tuple[int, str]: + try: + with _httpx.Client(timeout=8.0, follow_redirects=True) as client: + r = client.get(url) + return r.status_code, r.text + except Exception as e: + return 0, f"fetch error: {e}" + + status, text = await asyncio.to_thread(_fetch_sync) + if status == 404: + _vllm_recipe_cache[repo] = (now, {"exists": False}) + return {"exists": False} + if status != 200: + return {"exists": False, "error": f"HTTP {status}", "transient": True} + + try: + doc = _yaml.safe_load(text) or {} + except Exception as e: + return {"exists": False, "error": f"yaml parse: {e}"} + + meta = doc.get("meta") or {} + model = doc.get("model") or {} + features = doc.get("features") or {} + deps = doc.get("dependencies") or [] + variants = doc.get("variants") or {} + hw_overrides = doc.get("hardware_overrides") or {} + strat_overrides = doc.get("strategy_overrides") or {} + + # Tool-call + reasoning parsers, as flat arg arrays, so the frontend + # can drop them straight into the launch command. + tool_calling = features.get("tool_calling") or {} + reasoning = features.get("reasoning") or {} + + normalized = { + "exists": True, + "source_url": url, + "title": meta.get("title") or "", + "provider": meta.get("provider") or "", + "description": meta.get("description") or "", + "date_updated": str(meta.get("date_updated") or ""), + "hardware_support": meta.get("hardware") or {}, + "model_id": model.get("model_id") or repo, + "min_vllm_version": model.get("min_vllm_version") or "", + "architecture": model.get("architecture") or "", + "parameter_count": model.get("parameter_count") or "", + "active_parameters": model.get("active_parameters") or "", + "context_length": model.get("context_length") or 0, + "base_args": list(model.get("base_args") or []), + "base_env": dict(model.get("base_env") or {}), + "tool_calling": { + "description": tool_calling.get("description") or "", + "args": list(tool_calling.get("args") or []), + } if tool_calling else None, + "reasoning": { + "description": reasoning.get("description") or "", + "args": list(reasoning.get("args") or []), + } if reasoning else None, + "dependencies": [ + { + "note": (d.get("note") or "").strip(), + "command": (d.get("command") or "").strip(), + "optional": bool(d.get("optional", False)), + } + for d in deps if isinstance(d, dict) + ], + "variants": { + k: { + "model_id": v.get("model_id") or model.get("model_id") or repo, + "precision": v.get("precision") or "", + "vram_minimum_gb": v.get("vram_minimum_gb") or 0, + "description": v.get("description") or "", + "extra_args": list(v.get("extra_args") or []), + "extra_env": dict(v.get("extra_env") or {}), + } + for k, v in variants.items() if isinstance(v, dict) + }, + "hardware_overrides": { + hw: { + "extra_args": list((ov or {}).get("extra_args") or []), + "extra_env": dict((ov or {}).get("extra_env") or {}), + } + for hw, ov in hw_overrides.items() if isinstance(ov, dict) + }, + "strategy_overrides": { + strat: dict(ov or {}) + for strat, ov in strat_overrides.items() if isinstance(ov, dict) + }, + "compatible_strategies": list(doc.get("compatible_strategies") or []), + } + _vllm_recipe_cache[repo] = (now, normalized) + return normalized + @router.get("/api/cookbook/tasks/status") async def cookbook_tasks_status(request: Request): """Check status of all active cookbook tmux sessions. @@ -1564,16 +4005,108 @@ async def cookbook_tasks_status(request: Request): event loop. Now the whole body runs in a worker thread via asyncio.to_thread so other requests stay responsive.""" require_admin(request) - return await asyncio.to_thread(_cookbook_tasks_status_sync) + now = time.monotonic() + cached = _tasks_status_cache.get("value") + if cached is not None and now - float(_tasks_status_cache.get("ts") or 0) < 2.0: + return cached + inflight = _tasks_status_inflight.get("task") + if inflight and not inflight.done(): + return await inflight + + async def _compute(): + data = await asyncio.to_thread(_cookbook_tasks_status_sync) + _tasks_status_cache.update({"ts": time.monotonic(), "value": data}) + return data + + task = asyncio.create_task(_compute()) + _tasks_status_inflight["task"] = task + try: + return await task + finally: + if _tasks_status_inflight.get("task") is task: + _tasks_status_inflight["task"] = None def _cookbook_tasks_status_sync(): import subprocess + def _pick_download_progress(lines: list[str]) -> str: + """Pick the most useful live HF progress line from a tmux pane.""" + if not lines: + return "" + downloading_lines = [l for l in lines if l.startswith("Downloading")] + if downloading_lines: + return downloading_lines[-1] + progress_lines = [ + l for l in lines + if re.search(r"\b(?:100|[1-9]?\d)%", l) + and ( + "<" in l + or "it/s" in l + or "B/s" in l + or "safetensors" in l + or ".gguf" in l.lower() + ) + ] + if progress_lines: + return progress_lines[-1] + return lines[-1] + + def _download_cache_complete(repo_id: str, remote_host: str = "", ssh_port: str = "", cache_root: str = "") -> bool: + """Best-effort check for a completed HF cache entry. + + tmux output can stop at a stale progress line if the pane/session + disappears before Cookbook captures the final DOWNLOAD_OK marker. + In that case, trust the cache shape: a snapshot directory with files + and no *.incomplete blobs means HuggingFace finished materializing the + model. cache_root is the task's custom download dir — the runner + pointed HF_HOME there, so the cache lives under /hub, + not wherever this probe's environment says. + """ + if not repo_id or "/" not in repo_id: + return False + cmd = ["python3", "-c", HF_CACHE_COMPLETE_PROBE, repo_id, cache_root or ""] + try: + if remote_host: + ssh_base = ["ssh"] + if ssh_port and ssh_port != "22": + ssh_base.extend(["-p", str(ssh_port)]) + shell_cmd = " ".join(shlex.quote(x) for x in cmd) + proc = subprocess.run(ssh_base + [remote_host, shell_cmd], timeout=12, capture_output=True) + else: + proc = subprocess.run(cmd, timeout=12, capture_output=True) + return proc.returncode == 0 + except Exception: + return False + + def _download_cache_incomplete(repo_id: str, remote_host: str = "", ssh_port: str = "", cache_root: str = "") -> bool: + """Best-effort check for resumable HF partial blobs. + + A lost SSH/tmux session can leave a real download still incomplete. + Treat any *.incomplete blob as stronger evidence than stale + "100%" lines in the captured pane output. + """ + if not repo_id or "/" not in repo_id: + return False + cmd = ["python3", "-c", HF_CACHE_INCOMPLETE_PROBE, repo_id, cache_root or ""] + try: + if remote_host: + ssh_base = ["ssh"] + if ssh_port and ssh_port != "22": + ssh_base.extend(["-p", str(ssh_port)]) + shell_cmd = " ".join(shlex.quote(x) for x in cmd) + proc = subprocess.run(ssh_base + [remote_host, shell_cmd], timeout=12, capture_output=True) + else: + proc = subprocess.run(cmd, timeout=12, capture_output=True) + return proc.returncode == 0 + except Exception: + return False + # Load saved tasks from cookbook state tasks = [] + state = {} if _cookbook_state_path.exists(): try: - state = json.loads(_cookbook_state_path.read_text()) + state = json.loads(_cookbook_state_path.read_text(encoding="utf-8")) saved_tasks = state.get("tasks", []) if isinstance(saved_tasks, list): tasks = saved_tasks @@ -1582,6 +4115,21 @@ def _cookbook_tasks_status_sync(): except Exception: pass + # Orphan-tmux auto-adoption sweep. When the agent (or anyone) + # SSH-launches a `serve-*` tmux session — usually because + # serve_model rejected `source ... && vllm ...` or because of a + # manual relaunch via tmux send-keys — that session is invisible + # to the cookbook UI even though it's a live model server. The + # sweep finds those orphans on each configured remote host and + # writes them into state.tasks with _adoptedExternally=True, so + # they show up in the UI on the next poll without anyone having + # to remember to call adopt_served_model. Rate-limited via the + # module-level _last_orphan_sweep so we don't SSH every 3s. + try: + _maybe_sweep_orphans(tasks, state) + except Exception as _sweep_e: + logger.warning(f"orphan sweep failed (non-fatal): {_sweep_e!r}") + results = [] for task in tasks: session_id = task.get("sessionId", "") @@ -1611,12 +4159,18 @@ def _cookbook_tasks_status_sync(): if not _SESSION_ID_RE.match(session_id): logger.warning(f"Skipping task with unsafe session_id: {session_id!r}") continue - if remote and not _REMOTE_HOST_RE.match(remote): - logger.warning(f"Skipping task with unsafe remoteHost: {remote!r}") - continue - if _tport and not _SSH_PORT_RE.match(str(_tport)): - logger.warning(f"Skipping task with unsafe sshPort: {_tport!r}") - continue + if remote: + try: + remote = validate_remote_host(remote) + except HTTPException: + logger.warning(f"Skipping task with unsafe remoteHost: {remote!r}") + continue + if _tport: + try: + _tport = validate_ssh_port(str(_tport)) + except HTTPException: + logger.warning(f"Skipping task with unsafe sshPort: {_tport!r}") + continue if task_platform == "windows" and remote: # Windows: check PID file + Get-Process, read log tail sd = "$env:TEMP\\odysseus-sessions" @@ -1640,72 +4194,175 @@ def _cookbook_tasks_status_sync(): ssh_base = ["ssh"] if _tport and _tport != "22": ssh_base.extend(["-p", str(_tport)]) - check_cmd = ssh_base + [remote, "tmux", "has-session", "-t", session_id] - capture_cmd = ssh_base + [remote, "tmux", "capture-pane", "-t", session_id, "-p", "-S", "-50"] + check_cmd = ssh_base + [remote, _remote_tmux_command("has-session", "-t", session_id)] + # Capture 500 lines (was 50) so a Python traceback survives + # the post-crash neofetch banner + bash prompt that otherwise + # fills the visible tail. Without this, output_tail ends up + # as just "Locale: C / Ubuntu_Odysseus ❯" and the agent + # can't diagnose the actual error. + capture_cmd = ssh_base + [remote, _remote_tmux_command("capture-pane", "-t", session_id, "-p", "-S", "-500")] + elif IS_WINDOWS: + # LOCAL Windows task: launched as a detached process (no tmux). + # Liveness comes from the .pid file, output from the + # .log file the wrapper redirects into. No subprocess. + check_cmd = None + capture_cmd = None else: check_cmd = ["tmux", "has-session", "-t", session_id] - capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-50"] + capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-500"] - try: - alive = subprocess.run(check_cmd, timeout=10, capture_output=True) - is_alive = alive.returncode == 0 - except Exception: - is_alive = False + local_win_task = (not remote) and IS_WINDOWS - # Capture last lines for progress. Prefer the "Downloading" line - # (real aggregate bytes) over "Fetching N files" (whole-file count that - # lags with hf_transfer). Falls back to the true last line otherwise. progress_text = "" - full_snapshot = "" - if is_alive: + full_snapshot = (task.get("output") or "")[-12000:] if task_type == "serve" else "" + + if local_win_task: + # File-based liveness + output for the detached-process model. + pid_path = TMUX_LOG_DIR / f"{session_id}.pid" + log_path = TMUX_LOG_DIR / f"{session_id}.log" + task_pid = None + try: + task_pid = int(pid_path.read_text(encoding="utf-8").strip()) + except Exception: + task_pid = None + is_alive = pid_alive(task_pid) try: - cap = subprocess.run(capture_cmd, timeout=10, capture_output=True, text=True) - if cap.returncode == 0: - full_snapshot = cap.stdout.strip() + if log_path.exists(): + full_snapshot = log_path.read_text( + encoding="utf-8", errors="replace" + ).strip()[-12000:] lines = [l.strip() for l in full_snapshot.split('\n') if l.strip()] - downloading_lines = [l for l in lines if l.startswith("Downloading")] - if downloading_lines: - progress_text = downloading_lines[-1] - elif lines: - progress_text = lines[-1] + progress_text = _pick_download_progress(lines) except Exception: pass + else: + # Skip the live SSH check entirely for tasks already in a + # terminal state — they won't change, and 10s timeouts + # stacked per task were the dominant cost of this whole + # status endpoint (3+ minute stalls with ~8 accumulated + # stopped tasks). The agent's `list_served_models` call + # was blocking the chat stream every time. + _task_status = (task.get("status") or "").lower() + _persisted_serve_ready = ( + task_type == "serve" + and bool(full_snapshot) + and _parse_serve_phase(full_snapshot, task_type).get("status") == "ready" + ) + if _task_status in {"stopped", "done", "completed", + "crashed", "error", "failed", + "ended", "killed"} and not _persisted_serve_ready: + is_alive = False + # Keep the persisted output_tail for the UI — it's + # what the agent uses to diagnose past failures. + full_snapshot = (task.get("output") or "")[-12000:] + else: + try: + alive = subprocess.run(check_cmd, timeout=4, capture_output=True) + is_alive = alive.returncode == 0 + except Exception: + is_alive = False - # Determine status + # Capture last lines for progress. Prefer the "Downloading" line + # (real aggregate bytes) over "Fetching N files" (whole-file count that + # lags with hf_transfer). Falls back to the true last line otherwise. + if is_alive: + try: + cap = subprocess.run(capture_cmd, timeout=4, capture_output=True, text=True) + if cap.returncode == 0: + full_snapshot = cap.stdout.strip() + lines = [l.strip() for l in full_snapshot.split('\n') if l.strip()] + progress_text = _pick_download_progress(lines) + except Exception: + pass + + # Determine status. For the local-Windows detached model the log file + # persists after the process exits, so a finished download still has a + # snapshot to classify (DOWNLOAD_OK / exit marker) — evaluate it even + # when the PID is gone instead of blindly reporting "stopped". + download_zero_files = False + exit_code = None status = "unknown" - if is_alive: + download_has_ok = task_type == "download" and "DOWNLOAD_OK" in full_snapshot + download_has_failed = task_type == "download" and "DOWNLOAD_FAILED" in full_snapshot + download_has_incomplete_evidence = ( + task_type == "download" + and ( + ".incomplete" in full_snapshot + or bool(re.search(r'model-\d+-of-\d+\.[A-Za-z0-9_.-]+:\s+(?:[0-9]|[1-8][0-9])%', full_snapshot)) + or _download_cache_incomplete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "") + ) + ) + if is_alive or (local_win_task and full_snapshot): lower = full_snapshot.lower() - has_exit = "=== process exited with code" in lower + exit_match = re.search(r"=== process exited with code\s+(-?\d+)", full_snapshot, re.I) + has_exit = exit_match is not None + exit_code = int(exit_match.group(1)) if exit_match else None has_error = "error" in lower or "failed" in lower or "traceback" in lower if has_exit and task_type == "serve": # Serve tasks that exit are always errors — they should run indefinitely status = "error" + elif has_exit and task_type == "download": + # Dependency installs are tracked as download tasks but only + # emit the generic runner exit marker, not HF download markers. + if download_has_incomplete_evidence and not download_has_ok: + status = "running" if is_alive else "stopped" + else: + status = "completed" if exit_code == 0 else "error" elif has_exit and "unrecognized arguments" in lower: status = "error" elif has_error and not ("application startup complete" in lower): status = "error" - elif task_type == "download" and ("100%" in full_snapshot or "DOWNLOAD_OK" in full_snapshot): - # Only download tasks treat 100% as "completed". - # Serve tasks log 100%|██████| during inference progress - # (diffusion sampling, etc.) — that's "running", not done. - status = "completed" + elif task_type == "download" and download_has_ok: + if re.search(r"Fetching\s+0\s+files", full_snapshot, re.IGNORECASE): + status = "error" + download_zero_files = True + else: + status = "completed" + elif task_type == "download" and download_has_failed: + status = "error" + elif task_type == "download" and download_has_incomplete_evidence: + status = "running" if is_alive else "stopped" elif "application startup complete" in lower: status = "ready" + elif not is_alive: + # local-Windows: process gone, log has no success/ready marker. + status = "stopped" else: status = "running" else: - # Session is dead — check if it completed or crashed - status = "stopped" + # Session is dead — check if it completed or crashed. The + # runner markers in the retained output are conclusive + # (DOWNLOAD_OK only prints after exit 0), so check them before + # the cache probe, which can't see ollama pulls at all. + marker = classify_dead_download(full_snapshot) if task_type == "download" else None + if marker is not None: + status, download_zero_files = marker + if status == "completed" and not progress_text: + progress_text = "Download complete" + elif ( + task_type == "download" + and not download_has_incomplete_evidence + and _download_cache_complete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "") + ): + status = "completed" + if not progress_text: + progress_text = "Download complete" + if not full_snapshot: + full_snapshot = "DOWNLOAD_OK" + else: + status = "stopped" # Parse structured phase info — single source of truth for the UI - phase_info = _parse_serve_phase(full_snapshot, task_type) if (task_type == "serve" and status == "running" and full_snapshot) else {} - if phase_info.get("status") == "ready": + phase_info = _parse_serve_phase(full_snapshot, task_type) if (task_type == "serve" and full_snapshot) else {} + if phase_info.get("status") == "ready" and is_alive: status = "ready" serve_phase = phase_info.get("phase", "") diagnosis = _diagnose_serve_output(full_snapshot) if task_type == "serve" and full_snapshot else None - if diagnosis and status in {"running", "unknown", "stopped"}: + if diagnosis and status in {"running", "unknown", "stopped"} and phase_info.get("status") != "ready": status = "error" - output_tail = "\n".join(full_snapshot.splitlines()[-12:]) if full_snapshot else "" + if download_zero_files: + diagnosis = {"message": "No matching files were downloaded. The model repo or filename/quant pattern may be wrong (for example a ':Q4_K_M' tag that does not exist in the repo). Check the repo and the include/quant pattern."} + output_tail = error_aware_output_tail(full_snapshot, status) results.append({ "session_id": session_id, @@ -1716,6 +4373,7 @@ def _cookbook_tasks_status_sync(): "phase": serve_phase, "diagnosis": diagnosis, "output_tail": output_tail, + "exit_code": exit_code, "cmd": _payload.get("_cmd") or "", "tps": phase_info.get("tps"), "reqs": phase_info.get("reqs"), diff --git a/routes/copilot_routes.py b/routes/copilot_routes.py new file mode 100644 index 000000000..1d8be52ce --- /dev/null +++ b/routes/copilot_routes.py @@ -0,0 +1,173 @@ +# routes/copilot_routes.py +"""GitHub Copilot device-flow login. + +Drives the GitHub OAuth *device flow* and, on success, creates (or refreshes) +an owner-scoped ``ModelEndpoint`` pointing at the Copilot API with the +device-flow access token stored as its (encrypted) ``api_key``. After that the +endpoint behaves like any other OpenAI-compatible provider — the Copilot- +specific request headers are injected centrally by ``build_headers`` / +``_provider_headers`` (see :mod:`src.copilot`). + +Flow: + 1. ``POST /api/copilot/device/start`` → returns a ``poll_id`` plus the + ``user_code`` + ``verification_uri`` to show the user. The secret + ``device_code`` is kept server-side, never sent to the browser. + 2. The browser polls ``POST /api/copilot/device/poll`` with ``poll_id``. + While pending it returns ``{status: "pending"}``; once the user authorises + it provisions the endpoint and returns ``{status: "authorized", ...}``. + +All routes are admin-gated (endpoint/provider management is an admin action). +""" + +import json +import uuid +import logging +from typing import Dict, Optional + +import httpx +from fastapi import HTTPException, Request + +from core.database import SessionLocal, ModelEndpoint +from routes.device_flow import ( + DeviceFlowPoll, + DeviceFlowStart, + PendingDeviceFlowStore, + create_device_flow_router, +) +from src.auth_helpers import get_current_user +from src import copilot + +logger = logging.getLogger(__name__) + +_DEVICE_FLOW_STORE = PendingDeviceFlowStore() + + +def _provision_endpoint(token: str, base: str, owner: Optional[str]) -> Dict: + """Create or update the owner's Copilot endpoint with a fresh token.""" + try: + models = copilot.fetch_models(base, token) + except Exception as e: + logger.warning(f"Copilot model fetch failed during provisioning: {e}") + models = [] + model_ids = [m["id"] for m in models] + # Copilot picker models support OpenAI-style tool calling; mark the endpoint + # tool-capable so the agent loop sends native tool schemas. + # Tool-capable if any picker model advertises tool_calls. When the model + # fetch failed (empty list) default to True, since Copilot picker models + # support OpenAI-style tool calling. + supports_tools = bool(not models or any(m.get("tool_calls") for m in models)) + + db = SessionLocal() + try: + ep = ( + db.query(ModelEndpoint) + .filter(ModelEndpoint.base_url == base) + .filter((ModelEndpoint.owner.is_(None)) | (ModelEndpoint.owner == owner)) + .order_by(ModelEndpoint.owner.desc()) + .first() + ) + if ep is None: + ep = ModelEndpoint( + id=str(uuid.uuid4())[:8], + name="GitHub Copilot", + base_url=base, + model_type="llm", + owner=owner, + ) + db.add(ep) + ep.api_key = token + ep.is_enabled = True + ep.supports_tools = supports_tools + if model_ids: + ep.cached_models = json.dumps(model_ids) + db.commit() + result = { + "id": ep.id, + "name": ep.name, + "base_url": ep.base_url, + "models": model_ids, + } + finally: + db.close() + + # Best-effort: refresh the model cache so the new endpoint shows up. + try: + from routes.model_routes import _invalidate_models_cache + _invalidate_models_cache() + except Exception: + pass + return result + + +def _start_device_flow(request: Request, form) -> DeviceFlowStart: + host = copilot.GITHUB_HOST + ent = str(form.get("enterprise_url") or "").strip() + if ent: + host = copilot.normalize_domain(ent) + try: + data = copilot.request_device_code(host) + except httpx.HTTPStatusError as e: + status = e.response.status_code if e.response is not None else "unknown" + raise HTTPException(502, f"GitHub device-code request failed (HTTP {status})") + except Exception as e: + raise HTTPException(502, f"GitHub device-code request failed: {e}") + + device_code = data.get("device_code") + if not device_code: + raise HTTPException(502, "GitHub did not return a device code") + + # verification_uri_complete embeds the user code, so the browser tab we + # open lands the user straight on GitHub's "Authorize" screen with the + # code pre-filled — one click, no manual code entry. + return DeviceFlowStart( + pending={ + "device_code": device_code, + "host": host, + "enterprise_url": ent, + "owner": get_current_user(request) or None, + }, + response={ + "user_code": data.get("user_code"), + "verification_uri": data.get("verification_uri"), + "verification_uri_complete": data.get("verification_uri_complete"), + }, + interval=int(data.get("interval") or 5), + expires_in=int(data.get("expires_in") or 900), + ) + + +def _poll_device_flow(_request: Request, pending: Dict) -> DeviceFlowPoll: + try: + data = copilot.poll_access_token(pending["host"], pending["device_code"]) + except Exception as e: + return DeviceFlowPoll.pending(f"poll error: {e}") + + token = data.get("access_token") + if token: + base = copilot.enterprise_base(pending["enterprise_url"]) if pending["enterprise_url"] else copilot.COPILOT_BASE + try: + result = _provision_endpoint(token, base, pending["owner"]) + except Exception as e: + logger.exception("Copilot endpoint provisioning failed") + raise HTTPException(500, f"Login succeeded but provisioning failed: {e}") + return DeviceFlowPoll.authorized(result) + + err = data.get("error") + if err == "authorization_pending": + return DeviceFlowPoll.pending() + if err == "slow_down": + return DeviceFlowPoll.slow_down(int(data.get("interval") or 0) or None) + if err in ("expired_token", "access_denied"): + return DeviceFlowPoll.failed(err) + # Unknown error — surface but keep the session for another try. + return DeviceFlowPoll.pending(err or "unknown") + + +def setup_copilot_routes(): + return create_device_flow_router( + prefix="/api/copilot", + tags=["copilot"], + store=_DEVICE_FLOW_STORE, + start_flow=_start_device_flow, + poll_flow=_poll_device_flow, + ) diff --git a/routes/device_flow.py b/routes/device_flow.py new file mode 100644 index 000000000..8b8ab4ac8 --- /dev/null +++ b/routes/device_flow.py @@ -0,0 +1,193 @@ +"""Shared OAuth/device-flow route scaffolding for provider setup.""" + +from __future__ import annotations + +import inspect +import threading +import time +import uuid +from dataclasses import dataclass +from typing import Any, Callable, Iterable, Mapping, Optional + +from fastapi import APIRouter, Form, HTTPException, Request + +from core.middleware import require_admin + + +@dataclass(frozen=True) +class DeviceFlowStart: + """Provider-specific start result consumed by the shared route wrapper.""" + + pending: Mapping[str, Any] + response: Mapping[str, Any] + interval: int = 5 + expires_in: int = 900 + + +@dataclass(frozen=True) +class DeviceFlowPoll: + """Normalized provider poll outcome.""" + + status: str + endpoint: Optional[Mapping[str, Any]] = None + error: Optional[str] = None + detail: Optional[str] = None + interval: Optional[int] = None + + @classmethod + def pending(cls, detail: Optional[str] = None) -> "DeviceFlowPoll": + return cls(status="pending", detail=detail) + + @classmethod + def slow_down(cls, interval: Optional[int] = None, detail: Optional[str] = None) -> "DeviceFlowPoll": + return cls(status="slow_down", interval=interval, detail=detail) + + @classmethod + def authorized(cls, endpoint: Mapping[str, Any]) -> "DeviceFlowPoll": + return cls(status="authorized", endpoint=endpoint) + + @classmethod + def failed(cls, error: str) -> "DeviceFlowPoll": + return cls(status="failed", error=error) + + +class PendingDeviceFlowStore: + """Thread-safe in-memory pending device-flow store. + + Device codes and provider-side secrets stay inside this process. Each entry + stores provider payload separately from poll metadata so provider callbacks + only receive the fields they created. + """ + + def __init__(self, *, time_func: Callable[[], float] = time.time): + self._pending: dict[str, dict[str, Any]] = {} + self._lock = threading.Lock() + self._time = time_func + + def _now(self) -> float: + return float(self._time()) + + def prune_expired(self) -> None: + now = self._now() + with self._lock: + for key in [k for k, v in self._pending.items() if v.get("expires_at", 0) < now]: + self._pending.pop(key, None) + + def add(self, payload: Mapping[str, Any], *, interval: int, expires_in: int) -> str: + self.prune_expired() + poll_id = uuid.uuid4().hex + with self._lock: + self._pending[poll_id] = { + "payload": dict(payload), + "interval": max(int(interval or 5), 1), + "expires_at": self._now() + max(int(expires_in or 900), 1), + "next_poll_at": 0.0, + } + return poll_id + + def get_payload(self, poll_id: str) -> Optional[dict[str, Any]]: + self.prune_expired() + with self._lock: + entry = self._pending.get(poll_id) + if entry is None: + return None + return dict(entry.get("payload") or {}) + + def is_throttled(self, poll_id: str) -> bool: + with self._lock: + entry = self._pending.get(poll_id) + return bool(entry and self._now() < float(entry.get("next_poll_at") or 0)) + + def schedule_next(self, poll_id: str) -> None: + now = self._now() + with self._lock: + entry = self._pending.get(poll_id) + if entry is not None: + entry["next_poll_at"] = now + int(entry.get("interval") or 5) + + def slow_down(self, poll_id: str, interval: Optional[int] = None) -> None: + now = self._now() + with self._lock: + entry = self._pending.get(poll_id) + if entry is not None: + new_interval = int(interval or (int(entry.get("interval") or 5) + 5)) + entry["interval"] = max(new_interval, 1) + entry["next_poll_at"] = now + entry["interval"] + + def pop(self, poll_id: str) -> None: + with self._lock: + self._pending.pop(poll_id, None) + + +async def _maybe_await(value: Any) -> Any: + if inspect.isawaitable(value): + return await value + return value + + +def _pending_response(detail: Optional[str] = None) -> dict[str, Any]: + response: dict[str, Any] = {"status": "pending"} + if detail: + response["detail"] = detail + return response + + +def create_device_flow_router( + *, + prefix: str, + tags: Iterable[str], + store: PendingDeviceFlowStore, + start_flow: Callable[[Request, Mapping[str, Any]], DeviceFlowStart], + poll_flow: Callable[[Request, Mapping[str, Any]], DeviceFlowPoll], +) -> APIRouter: + """Create standard `/device/start|poll|cancel` routes for a provider.""" + + router = APIRouter(prefix=prefix, tags=list(tags)) + + @router.post("/device/start") + async def device_start(request: Request): + require_admin(request) + form = await request.form() + start = await _maybe_await(start_flow(request, form)) + interval = int(start.interval or 5) + expires_in = int(start.expires_in or 900) + poll_id = store.add(start.pending, interval=interval, expires_in=expires_in) + response = dict(start.response) + response.update({"poll_id": poll_id, "interval": interval, "expires_in": expires_in}) + return response + + @router.post("/device/poll") + async def device_poll(request: Request, poll_id: str = Form(...)): + require_admin(request) + payload = store.get_payload(poll_id) + if payload is None: + raise HTTPException(404, "Unknown or expired login session") + if store.is_throttled(poll_id): + return {"status": "pending"} + + try: + outcome = await _maybe_await(poll_flow(request, payload)) + except Exception: + store.pop(poll_id) + raise + + if outcome.status == "authorized": + store.pop(poll_id) + return {"status": "authorized", "endpoint": dict(outcome.endpoint or {})} + if outcome.status == "failed": + store.pop(poll_id) + return {"status": "failed", "error": outcome.error or "denied"} + if outcome.status == "slow_down": + store.slow_down(poll_id, outcome.interval) + return _pending_response(outcome.detail) + + store.schedule_next(poll_id) + return _pending_response(outcome.detail) + + @router.post("/device/cancel") + def device_cancel(request: Request, poll_id: str = Form(...)): + require_admin(request) + store.pop(poll_id) + return {"status": "cancelled"} + + return router diff --git a/routes/diagnostics_routes.py b/routes/diagnostics_routes.py index 8f3a915c2..e6167a80f 100644 --- a/routes/diagnostics_routes.py +++ b/routes/diagnostics_routes.py @@ -1,12 +1,14 @@ """Diagnostics routes — /api/db/stats, /api/rag/stats, /api/test/youtube, /api/test-research.""" import logging +import os from typing import Dict, Any -from fastapi import APIRouter, HTTPException, Form +from fastapi import APIRouter, HTTPException, Form, Request from services.youtube.youtube_handler import extract_youtube_id, extract_transcript_async -from core.constants import DEFAULT_HOST +from core.constants import DEFAULT_HOST, DATA_DIR +from core.middleware import require_admin logger = logging.getLogger(__name__) @@ -15,11 +17,45 @@ def setup_diagnostics_routes( rag_manager, rag_available: bool, research_handler, + memory_vector=None, ) -> APIRouter: router = APIRouter(tags=["diagnostics"]) + @router.get("/api/diagnostics/services") + async def get_service_health(request: Request) -> Dict[str, Any]: + """Consolidated degraded-state report for ChromaDB, SearXNG, email, + ntfy, and provider endpoints. Non-intrusive probes — safe to poll.""" + require_admin(request) + from src.service_health import collect_service_health + return await collect_service_health(rag_manager, memory_vector) + + @router.get("/api/diagnostics/logs") + async def get_diagnostics_logs(request: Request, limit: int = 200) -> Dict[str, Any]: + require_admin(request) + limit = max(1, min(limit, 1000)) + try: + log_file = os.path.join(DATA_DIR, "logs", "app.log") + if not os.path.exists(log_file): + return {"status": "success", "logs": []} + + # Safe tail read of the log file (max 5MB via rotation) + with open(log_file, "r", encoding="utf-8", errors="ignore") as f: + lines = f.readlines() + + tail_lines = lines[-limit:] if len(lines) > limit else lines + tail_lines = [line.rstrip('\r\n') for line in tail_lines] + + return { + "status": "success", + "logs": tail_lines + } + except Exception as e: + logger.error(f"Diagnostics logs retrieval error: {e}") + raise HTTPException(500, f"Failed to retrieve logs: {str(e)}") + @router.get("/api/db/stats") - async def get_database_stats() -> Dict[str, Any]: + async def get_database_stats(request: Request) -> Dict[str, Any]: + require_admin(request) try: from core.database import get_detailed_stats return get_detailed_stats() @@ -28,13 +64,15 @@ async def get_database_stats() -> Dict[str, Any]: raise HTTPException(500, "Failed to retrieve database statistics") @router.get("/api/rag/stats") - async def get_rag_stats() -> Dict[str, Any]: + async def get_rag_stats(request: Request) -> Dict[str, Any]: + require_admin(request) if rag_available and rag_manager: return rag_manager.get_stats() return {"error": "RAG system not available"} @router.get("/api/test/youtube") - async def test_youtube(url: str) -> Dict[str, Any]: + async def test_youtube(request: Request, url: str) -> Dict[str, Any]: + require_admin(request) try: video_id = extract_youtube_id(url) if not video_id: @@ -54,7 +92,8 @@ async def test_youtube(url: str) -> Dict[str, Any]: return {"error": str(e)} @router.post("/api/test-research") - async def test_research(query: str = Form("What is machine learning?")) -> Dict[str, Any]: + async def test_research(request: Request, query: str = Form("What is machine learning?")) -> Dict[str, Any]: + require_admin(request) try: endpoint = f"http://{DEFAULT_HOST}:8000/v1/chat/completions" model = "gpt-oss-120b" diff --git a/routes/document_helpers.py b/routes/document_helpers.py index b60ad9456..a0c2d08eb 100644 --- a/routes/document_helpers.py +++ b/routes/document_helpers.py @@ -3,13 +3,17 @@ """Document routes — CRUD for living documents with version history.""" import logging -from typing import Dict, Any, Optional +import os +import re +from typing import Any, Dict, Optional -from fastapi import HTTPException +from fastapi import HTTPException, Request from pydantic import BaseModel from core.database import Document, DocumentVersion from core.database import Session as DbSession +from src.auth_helpers import _auth_disabled +from src.upload_handler import UploadHandler logger = logging.getLogger(__name__) @@ -25,6 +29,7 @@ class DocumentCreate(BaseModel): class DocumentUpdate(BaseModel): content: str summary: Optional[str] = None + force_version: bool = False class DocumentPatch(BaseModel): title: Optional[str] = None @@ -75,6 +80,8 @@ def _verify_doc_owner(db, doc: Document, user: str): the session join for any not-yet-backfilled legacy row. """ if user is None: + if _auth_disabled(): + return # Single-user / no-auth mode: allow access raise HTTPException(403, "Authentication required") if doc.owner is not None: if doc.owner != user: @@ -99,8 +106,10 @@ def _owner_session_filter(q, user): The owner backfill runs in init_db before the app serves requests, so by the time this filter is live there are no NULL-owner rows to leak; - we therefore match the owner strictly.""" - if user is None: + we therefore match the owner strictly for authenticated callers.""" + if not user: + if user == "" or _auth_disabled(): + return q return q.filter(False) return q.filter(Document.owner == user) @@ -126,46 +135,82 @@ def _slug(name: str) -> str: _PDF_RENDER_SCALE = 2.0 -def _locate_upload(upload_dir: str, file_id: str): - """Find an upload by its filename ID. - - Lookup order: - 1. Direct hit at `upload_dir/file_id` (very small deployments). - 2. The `uploads.json` index that `UploadHandler.save_upload` maintains — - maps file_hash → metadata containing the full path. O(1) once loaded. - 3. Fallback: `os.walk` the date-bucketed tree. Slow on large stores; - only triggers for legacy uploads recorded before the index existed. - - `followlinks=False` keeps a stray symlink loop in `data/uploads/` from - spinning the walker into infinite recursion. - """ - import os - import json as _json - direct = os.path.join(upload_dir, file_id) - if os.path.exists(direct): - return direct - # O(1) via uploads.json +def _upload_path_inside(upload_dir: str, path: str) -> bool: + base = os.path.realpath(upload_dir) + p = os.path.realpath(path) try: - idx_path = os.path.join(upload_dir, "uploads.json") - if os.path.exists(idx_path): - with open(idx_path, "r") as f: - idx = _json.load(f) - for meta in (idx.values() if isinstance(idx, dict) else []): - if meta.get("id") == file_id: - p = meta.get("path") - if p and os.path.exists(p): - return p + return os.path.commonpath([base, p]) == base except Exception: - pass - for root, _dirs, files in os.walk(upload_dir, followlinks=False): - if file_id in files: - return os.path.join(root, file_id) - return None + return False + + +def _resolve_user_upload_path( + upload_handler: Any, + upload_id: str, + owner: Optional[str], + auth_manager=None, +) -> Optional[str]: + """Resolve an upload id to a filesystem path the caller may read.""" + if upload_handler is None: + return None + resolved = upload_handler.resolve_upload( + upload_id, + owner=owner, + auth_manager=auth_manager, + ) + if not isinstance(resolved, dict) or not resolved: + return None + path = resolved.get("path") + upload_dir = getattr(upload_handler, "upload_dir", None) + if path and upload_dir and not _upload_path_inside(upload_dir, path): + logger.warning("Upload path outside upload directory: %s", path) + return None + return path + + +def _locate_upload( + upload_dir: str, + file_id: str, + owner: Optional[str] = None, + auth_manager=None, + upload_handler: Any = None, +): + """Find an upload by its filename ID via UploadHandler.resolve_upload.""" + if upload_handler is None: + from src.upload_handler import UploadHandler + + base_dir = os.path.dirname(os.path.abspath(upload_dir)) + upload_handler = UploadHandler(base_dir, upload_dir) + return _resolve_user_upload_path(upload_handler, file_id, owner, auth_manager) + + +def _assert_pdf_marker_upload_owned( + request: Request, + content: str, + user: Optional[str], + upload_handler: Any, +) -> None: + """Reject document content whose pdf_source marker points at another user's upload.""" + if upload_handler is None: + return + from src.pdf_form_doc import find_source_upload_id + + upload_id = find_source_upload_id(content or "") + if not upload_id: + return + auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None) + if not _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager): + raise HTTPException( + 400, + "Document PDF marker references an upload you do not own", + ) def _derive_title(content: str) -> str: """Derive a title from document content.""" import re + if not isinstance(content, str): + return "Untitled" text = content.strip() if not text: return "Untitled" diff --git a/routes/document_routes.py b/routes/document_routes.py index 94b331dda..dae8b09fa 100644 --- a/routes/document_routes.py +++ b/routes/document_routes.py @@ -7,26 +7,100 @@ from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Form -from sqlalchemy import func +from sqlalchemy import case, func, or_ from core.database import SessionLocal, Document, DocumentVersion from core.database import Session as DbSession -from src.auth_helpers import get_current_user +from src.auth_helpers import get_current_user, _auth_disabled +from src.constants import MAIL_ATTACHMENTS_DIR +from src.upload_handler import reserve_upload_references logger = logging.getLogger(__name__) +def _get_session_or_404(db, session_id: str, user: Optional[str]): + session = db.query(DbSession).filter(DbSession.id == session_id).first() + if not session: + raise HTTPException(404, "Session not found") + if user and session.owner != user: + raise HTTPException(404, "Session not found") + return session + + +def _aggregate_language_facets(lang_rows): + """Sum document counts per display language for the library facet. + + NULL-language and explicit "text" rows share the "text" bucket (the + language filter treats them as one), so they must be ADDED. The old dict + comprehension keyed both to "text", silently overwriting one group and + undercounting the facet versus what the filter actually returns. + """ + out = {} + for lang, cnt in lang_rows: + key = lang or "text" + out[key] = out.get(key, 0) + cnt + return out + + +def _library_language_for_document(doc: Document) -> str: + """Return the display language used by the document library. + + PDF documents are stored as markdown wrappers so the editor can preserve + extracted text, form fields, and annotations. The library should still + identify them as PDFs instead of exposing that internal wrapper format. + """ + from src.pdf_form_doc import find_source_upload_id + + if find_source_upload_id(doc.current_content or ""): + return "pdf" + return doc.language or "text" + + +def _email_source_key(content: str) -> tuple[str, str]: + """Return the source email identity embedded in an email draft document.""" + import re + + text = content or "" + uid_m = re.search(r"(?im)^X-Source-UID:\s*(.+?)\s*$", text) + folder_m = re.search(r"(?im)^X-Source-Folder:\s*(.+?)\s*$", text) + uid = (uid_m.group(1).strip() if uid_m else "") + folder = (folder_m.group(1).strip() if folder_m else "INBOX") + return uid, folder + from routes.document_helpers import ( DocumentCreate, DocumentUpdate, DocumentPatch, _doc_to_dict, _version_to_dict, _verify_doc_owner, _owner_session_filter, - _slug, _locate_upload, _derive_title, + _slug, _resolve_user_upload_path, _assert_pdf_marker_upload_owned, _derive_title, _PDF_RENDER_SCALE, ) + def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: router = APIRouter(tags=["documents"]) + def _reserve_document_uploads(user: Optional[str], content: str) -> None: + missing_id = reserve_upload_references(upload_handler, user, content) + if missing_id: + raise HTTPException( + 409, + f"Referenced upload is no longer available: {missing_id}", + ) + + def _locate_current_user_upload(request: Request, upload_id: str, user: Optional[str]): + if upload_handler is None: + return None + auth_manager = getattr(getattr(request.app, "state", None), "auth_manager", None) + return _resolve_user_upload_path(upload_handler, upload_id, user, auth_manager) + + def _load_pdf_viewer_fitz(): + from src.pdf_runtime import load_pymupdf_for_pdf_viewer + + try: + return load_pymupdf_for_pdf_viewer() + except RuntimeError as exc: + raise HTTPException(503, str(exc)) from exc + # ---- POST /api/document ---- @router.post("/api/document") async def create_document(request: Request, req: DocumentCreate) -> Dict[str, Any]: @@ -39,20 +113,12 @@ async def create_document(request: Request, req: DocumentCreate) -> Dict[str, An # the doc is owner-stamped, so it lives in the library on its own. session = None if req.session_id: - session = db.query(DbSession).filter(DbSession.id == req.session_id).first() - if not session: - raise HTTPException(404, "Session not found") # Match the lenient ownership model the rest of the app uses # (see _owner_filter): only block when an AUTHENTICATED user is # writing into a DIFFERENT user's session. In single-user / - # unconfigured / localhost-bypass mode the middleware leaves - # current_user unset (None), and those sessions are already - # served freely everywhere else. - if user and session.owner and session.owner != user: - raise HTTPException(403, "Cannot create document in another user's session") - - doc_id = str(uuid.uuid4()) - ver_id = str(uuid.uuid4()) + # unconfigured / localhost-bypass mode, falsey users preserve + # the existing lenient path. + session = _get_session_or_404(db, req.session_id, user) # If no language was supplied (e.g. cloning a doc whose language # was never set), detect it from the content rather than storing @@ -60,13 +126,57 @@ async def create_document(request: Request, req: DocumentCreate) -> Dict[str, An # to markdown for prose. language = req.language if not language: - from src.tool_implementations import _looks_like_email_document, _sniff_doc_language + from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language, _coerce_email_document_content language = _sniff_doc_language(req.content) else: - from src.tool_implementations import _looks_like_email_document + from src.agent_tools.document_tools import _looks_like_email_document, _coerce_email_document_content if _looks_like_email_document(req.content, req.title): language = "email" + _reserve_document_uploads(user, req.content) + _assert_pdf_marker_upload_owned(request, req.content, user, upload_handler) + + # Reply drafts are keyed to the source email. If a UI/tool path tries + # to create a second draft for the same email in the same chat, + # update the existing draft instead so quoted thread history stays + # attached to the visible document. + if language == "email" and req.session_id: + source_uid, source_folder = _email_source_key(req.content) + if source_uid: + candidates = ( + db.query(Document) + .filter(Document.session_id == req.session_id) + .filter(Document.is_active == True) + .filter(Document.language == "email") + .order_by(Document.updated_at.desc()) + .limit(25) + .all() + ) + for existing in candidates: + old_uid, old_folder = _email_source_key(existing.current_content or "") + if old_uid != source_uid or old_folder != source_folder: + continue + merged = _coerce_email_document_content(existing.current_content or "", req.content) + if existing.current_content != merged: + new_ver = (existing.version_count or 1) + 1 + existing.current_content = merged + existing.title = req.title or existing.title + existing.version_count = new_ver + db.add(DocumentVersion( + id=str(uuid.uuid4()), + document_id=existing.id, + version_number=new_ver, + content=merged, + summary="Updated existing email draft", + source="user", + )) + db.commit() + db.refresh(existing) + return _doc_to_dict(existing) + + doc_id = str(uuid.uuid4()) + ver_id = str(uuid.uuid4()) + doc = Document( id=doc_id, session_id=req.session_id, @@ -121,17 +231,17 @@ async def import_pdf( with a `pdf_source` marker so the viewer renders the pages without overlays. """ - from src.constants import UPLOAD_DIR from src.pdf_forms import has_form_fields, extract_fields from src.pdf_form_doc import ( save_field_sidecar, create_form_markdown_document, create_plain_pdf_document, ) - from src.document_processor import _process_pdf + from src.document_processor import _process_pdf, strip_pdf_content_marker import os - user = get_current_user(request) + from src.auth_helpers import require_privilege + user = require_privilege(request, "can_use_documents") # session_id is optional — a library import isn't tied to a chat. When # given, validate it; otherwise the PDF becomes a session-less library @@ -139,11 +249,7 @@ async def import_pdf( if session_id: db = SessionLocal() try: - sess = db.query(DbSession).filter(DbSession.id == session_id).first() - if not sess: - raise HTTPException(404, "Session not found") - if user and sess.owner and sess.owner != user: - raise HTTPException(403, "Cannot import into another user's session") + _get_session_or_404(db, session_id, user) finally: db.close() @@ -160,13 +266,13 @@ async def import_pdf( raise HTTPException(500, f"Upload failed: {e}") upload_id = meta["id"] - pdf_path = _locate_upload(UPLOAD_DIR, upload_id) + pdf_path = _locate_current_user_upload(request, upload_id, user) if not pdf_path: raise HTTPException(500, "Saved PDF could not be located") title = os.path.splitext(meta.get("original_name") or meta.get("name") or upload_id)[0] try: - body_text = _process_pdf(pdf_path).lstrip("\n[PDF content]:").strip() + body_text = strip_pdf_content_marker(_process_pdf(pdf_path, owner=user)) except Exception: body_text = None @@ -228,19 +334,30 @@ async def documents_library( db = SessionLocal() try: from sqlalchemy import or_ + pdf_marker_cond = or_( + Document.current_content.like('% marker is found and stripped even + # when the result doesn't carry a "results" or "stdout" key. + _src_text = result.get("output") or result.get("results") or result.get("stdout") or "" if block.tool_type == "web_search" and _src_text: _src_marker = "\s*\n*', '', text) + text = re.sub(r'\s*', '', text) + text = re.sub(r'\s*', '', text) + return text.strip() + + +def _create_pdf_text_derivative(db, *, source_doc, content: str, owner: Optional[str], summary: str) -> dict: + import uuid + from src.database import Document, DocumentVersion + + clean = _strip_pdf_editor_markers(content) + title_base = (getattr(source_doc, "title", None) or "PDF").strip() + title = title_base if title_base.lower().endswith("edited") else f"{title_base} edited" + doc_id = str(uuid.uuid4()) + ver_id = str(uuid.uuid4()) + new_doc = Document( + id=doc_id, + session_id=getattr(source_doc, "session_id", None), + title=title, + language="markdown", + current_content=clean, + version_count=1, + is_active=True, + owner=owner if owner is not None else getattr(source_doc, "owner", None), + ) + ver = DocumentVersion( + id=ver_id, + document_id=doc_id, + version_number=1, + content=clean, + summary=summary, + source="ai", + ) + db.add(new_doc) + db.add(ver) + db.commit() + set_active_document(doc_id) + return { + "action": "create", + "doc_id": doc_id, + "title": title, + "language": "markdown", + "content": clean, + "version": 1, + "source_doc_id": getattr(source_doc, "id", None), + } + + +class CreateDocumentTool: + async def execute(self, content: str, ctx: dict) -> dict: + """Create a new document. Supports two formats: + 1) Line-based: line 1 = title, line 2 (optional) = language, rest = content + 2) XML-like tags: ......... + Some models mix them — strip any XML-style tags and fall back to line parsing.""" + import uuid, re as _re + from src.database import SessionLocal, Document, DocumentVersion, Session as DbSession + + raw = content or "" + session_id = ctx.get("session_id") + owner = ctx.get("owner") + + # Known languages the editor understands (match the

Long-term facts the AI remembers across chats — recall, edit, or curate.

- - - +
+ + + +
+ +
- +
@@ -293,45 +307,54 @@

-

Add Memory

+

Add Memory

- Import a .txt, .md, .pdf, .csv, .log, .json, .py, .js, or .html file — the AI reads it and suggests candidate memories you can approve. Needs an open chat session (it uses that session's model). + Import a .txt, .md, .pdf, .csv, .log, .json, .py, .js, or .html file — the AI reads it and suggests candidate memories you can approve.

- + Add a memory — e.g. 'I prefer concise replies'
+
-

Add Skill

+

Add Skill

-

Create a skill by hand — title, what it solves, and an approach.

+

Import a skill from GitHub or skills.sh (folder with SKILL.md and optional templates).

+
+
+ + Import URL — e.g. GitHub tree link to a skill folder +
+ +
+

Or create a skill by hand — title, what it solves, and an approach.

- + Title — short name, e.g. “build-vllm-wheel”
- + When to use — what problem does this skill solve?
- + How — the approach, steps, commands, or rules to follow
- + Tags — comma-separated, e.g. python, build, vllm
- +

@@ -339,8 +362,9 @@

-

Skills

+

Skills

+

Reusable procedures the AI can call via /skill — sort by confidence to surface the proven ones.

@@ -365,10 +389,10 @@

Skills Confidence ≤ 70% - - + +

- +