diff --git a/.dockerignore b/.dockerignore index 609a1b746..384ba4b99 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,16 @@ +.git +node_modules +.venv env +__pycache__ +*.pyc +*.pyo enferno/media enferno/imports logs +backups +.env +.env.* +.DS_Store +*.md +docs/node_modules diff --git a/.gitignore b/.gitignore index fea7c8a77..f63a36744 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,9 @@ backups/* cookies.txt *.egg-info/ + +# Release signing: NEVER commit secret keys. The pinned public key is baked +# into the installer/updater, not stored as a loose file in the repo. +*.key +bayanat-release.key +bayanat-release.pub diff --git a/SAFE_EXPUNGING.md b/SAFE_EXPUNGING.md new file mode 100644 index 000000000..cdc2eb497 --- /dev/null +++ b/SAFE_EXPUNGING.md @@ -0,0 +1,43 @@ +# Safe Expunging Process + +This document describes when and how history-altering operations are permitted on the Bayanat source repository, satisfying the SLSA v1.2 Source track "Safe Expunging Process" requirement. + +## Scope + +Applies to the public repository `sjacorg/bayanat` and the private release repository `sjacorg/bayanat.prod`, specifically to operations that remove or rewrite committed history on protected references (`main`, release tags matching `v*`). + +## Default + +History on protected references is append-only. Force-push, branch deletion, tag deletion, and retagging are blocked by repository rulesets. + +## Permitted Reasons to Expunge + +Expunging may be approved only for one of the following reasons: + +1. **Secret leak.** An unredacted credential, private key, or access token was committed. +2. **Personal data leak.** Non-public personal data of an identifiable individual was committed. +3. **Legal or safety order.** A verified order from counsel or a credible safety concern requires removal of specific content. +4. **Malicious injection.** Attacker-introduced code or data must be removed as part of incident response. + +Bug fixes, style corrections, and cleanup are never valid reasons. + +## Approval + +Both maintainers must approve in writing, recorded in the security advisory created for the incident. + +## Procedure + +1. File a private security advisory at https://github.com/sjacorg/bayanat/security/advisories with the reason, affected commits, and proposed action. +2. Record both maintainer approvals in the advisory. +3. If the reason involves a secret, rotate it before rewriting. +4. Rewrite with `git filter-repo` (not `filter-branch`), preserving commit signatures where possible. +5. Temporarily bypass branch protection, force-update the protected reference, then re-enable protection. +6. Invalidate and regenerate any affected release tags. Old tags are not reused. + +## Consumer Notification + +After any expunging action, publish a public security advisory that includes: + +- What was removed and why (redacted as needed). +- New commit hashes and release tags that replace the expunged revisions. +- Operator guidance (re-clone, re-verify signatures, check deployed commit against the new history). diff --git a/bayanat b/bayanat index a89ac1d9f..181be75bd 100755 --- a/bayanat +++ b/bayanat @@ -5,7 +5,7 @@ # set -euo pipefail -readonly SCRIPT_VERSION="1.2.0" +readonly SCRIPT_VERSION="1.3.0" readonly BAYANAT_ROOT="/opt/bayanat" readonly RELEASES_DIR="$BAYANAT_ROOT/releases" readonly SHARED_DIR="$BAYANAT_ROOT/shared" @@ -13,7 +13,27 @@ readonly LOGS_DIR="$BAYANAT_ROOT/logs" readonly CURRENT_LINK="$BAYANAT_ROOT/current" readonly DEFAULT_REPO="sjacorg/bayanat" readonly GIT_URL="https://github.com/${BAYANAT_REPO:-$DEFAULT_REPO}.git" +# Pinned SJAC release-signing key (BAY-01-017). Release tarballs are verified +# against this minisign public key before install; an unsigned or mismatched +# tarball is refused. Baked into this root-owned script so it cannot be swapped +# at update time. Rotation needs a fresh install or a documented manual swap +# (minisign has no revocation); see docs/deployment/release-signing.md. +readonly RELEASE_PUBKEY="RWS7XvDVF0InHWTCh/86K8sXGcHU/PmzCl4uH9GUDjNnNzHhcX1BvGqZ" +# Service accounts (BAY-01-032): web and worker run as separate system users. +# $APP_USER remains the deploy/database identity (PG role name, peer auth for +# admin operations) and the shared group that grants read access to releases. readonly APP_USER="bayanat" +readonly APP_GROUP="bayanat" +readonly WEB_USER="bayanat-web" +readonly CELERY_USER="bayanat-celery" +# App-writable runtime state shared between web and worker (config.json, +# reload/restart sentinels, celery beat schedule). Kept separate from +# $SHARED_DIR itself so .env never lives in a service-writable directory. +readonly RUNTIME_DIR="$SHARED_DIR/runtime" +# uv-managed Python interpreters go in a root-owned, world-readable location +# instead of /root, so the venv stays usable by the service accounts +# (releases are built by root, BAY-01-032). +export UV_PYTHON_INSTALL_DIR="/usr/local/share/uv/python" # --- Logging --- @@ -71,10 +91,13 @@ current_version() { } flask_run() { + # --no-sync: releases and their .venv are root-owned (BAY-01-032); deps + # are installed by _install_deps as root, so uv must not try to re-sync + # the environment while running as $APP_USER. local version="$1"; shift sudo -u "$APP_USER" \ - env FLASK_APP=run.py \ - /usr/local/bin/uv run \ + env FLASK_APP=run.py UV_PYTHON_INSTALL_DIR="$UV_PYTHON_INSTALL_DIR" \ + /usr/local/bin/uv run --no-sync \ --directory "$RELEASES_DIR/$version" \ flask "$@" } @@ -85,6 +108,487 @@ swap_symlink() { mv -Tf "$CURRENT_LINK.tmp" "$CURRENT_LINK" } +# --- Snapshot helpers --- + +SNAPSHOT_RETENTION_DAYS="${BAYANAT_SNAPSHOT_RETENTION_DAYS:-30}" +readonly SNAPSHOT_RETENTION_COUNT=5 + +_pg_load() { + # Loads POSTGRES_* / REDIS_PASSWORD from shared/.env into the current + # shell env WITHOUT eval. Must be invoked inside a subshell so the + # exports do not leak. + # + # Why no eval: a malicious .env value (writable by the bayanat user) + # could smuggle shell commands that would run as root under `bayanat + # update`. `export "$key=$val"` treats $val as a literal string, no + # interpretation. + local env_file="$SHARED_DIR/.env" + [[ -f "$env_file" ]] || die "missing $env_file" + local key val + while IFS='=' read -r key val; do + case "$key" in + POSTGRES_HOST|POSTGRES_PORT|POSTGRES_USER|POSTGRES_PASSWORD|POSTGRES_DB|REDIS_PASSWORD) + # Strip matching surrounding quotes (common .env convention). + if [[ ${#val} -ge 2 ]]; then + if [[ "${val:0:1}" == '"' && "${val: -1}" == '"' ]] \ + || [[ "${val:0:1}" == "'" && "${val: -1}" == "'" ]]; then + val="${val:1:${#val}-2}" + fi + fi + export "$key=$val" + ;; + esac + done < "$env_file" +} + +snapshot_pg_dump() { + # $1 = previous tag, $2 = target tag + # Writes shared/backups/pre--to--.dump via .partial rename. + # Mirrors enferno/utils/backup_utils.py:pg_dump logic: localhost + no + # password -> socket peer auth; else TCP with PGPASSWORD. + local prev="$1" target="$2" + local ts name partial final + ts=$(date -u +%Y%m%d-%H%M) + name="pre-${prev}-to-${target}-${ts}.dump" + partial="$SHARED_DIR/backups/${name}.partial" + final="$SHARED_DIR/backups/${name}" + # Log goes to stderr so `$(snapshot_pg_dump ...)` captures only the + # basename from the trailing `echo "$name"`. + log "Taking pre-update snapshot: $name" >&2 + ( + _pg_load + local user="${POSTGRES_USER:-$APP_USER}" + local db="${POSTGRES_DB:-$APP_USER}" + local host="${POSTGRES_HOST:-localhost}" + if [[ "$host" == "localhost" && -z "${POSTGRES_PASSWORD:-}" ]]; then + sudo -u "$user" pg_dump -Fc -f "$partial" "$db" + else + PGPASSWORD="${POSTGRES_PASSWORD:-}" \ + pg_dump -Fc \ + -h "$host" \ + -p "${POSTGRES_PORT:-5432}" \ + -U "$user" \ + -f "$partial" \ + "$db" + fi + ) + mv -Tf "$partial" "$final" + echo "$name" +} + +prune_snapshots() { + # Keep last $SNAPSHOT_RETENTION_COUNT snapshots OR last + # $SNAPSHOT_RETENTION_DAYS days, whichever is greater. + local backups="$SHARED_DIR/backups" + [[ -d "$backups" ]] || return 0 + # Sweep any leaked .partial snapshot files from interrupted dumps + rm -f "$backups"/*.dump.partial 2>/dev/null || true + local cutoff_epoch + cutoff_epoch=$(date -d "-${SNAPSHOT_RETENTION_DAYS} days" +%s 2>/dev/null \ + || date -v-"${SNAPSHOT_RETENTION_DAYS}"d +%s) + local idx=0 name path mtime + while IFS= read -r path; do + idx=$((idx + 1)) + name=$(basename "$path") + mtime=$(stat -c %Y "$path" 2>/dev/null || stat -f %m "$path") + if [[ "$idx" -le "$SNAPSHOT_RETENTION_COUNT" ]]; then + continue + fi + if [[ "$mtime" -lt "$cutoff_epoch" ]]; then + log "Pruning snapshot: $name" + rm -f "$path" + fi + done < <(ls -1t "$backups"/pre-*.dump 2>/dev/null || true) +} + +list_snapshots() { + local backups="$SHARED_DIR/backups" + [[ -d "$backups" ]] || { echo "No snapshots directory."; return; } + printf '%-60s %10s %20s\n' "NAME" "SIZE" "AGE" + local name size mtime age + while IFS= read -r path; do + name=$(basename "$path") + size=$(du -h "$path" | cut -f1) + mtime=$(stat -c %Y "$path" 2>/dev/null || stat -f %m "$path") + age="$(( ($(date +%s) - mtime) / 3600 ))h" + printf '%-60s %10s %20s\n' "$name" "$size" "$age" + done < <(ls -1t "$backups"/pre-*.dump 2>/dev/null || true) +} + +restore_pg() { + # $1 = snapshot basename (no path). Stops services, restores, starts services. + local name="$1" + local path="$SHARED_DIR/backups/$name" + [[ -f "$path" ]] || die "snapshot not found: $path" + log "Restoring $name (this will DROP and RECREATE tables)" + read -r -p "Type 'yes' to confirm: " reply + [[ "$reply" == "yes" ]] || die "aborted" + systemctl stop bayanat bayanat-celery + if ! ( + _pg_load + local user="${POSTGRES_USER:-$APP_USER}" + local db="${POSTGRES_DB:-$APP_USER}" + local host="${POSTGRES_HOST:-localhost}" + if [[ "$host" == "localhost" && -z "${POSTGRES_PASSWORD:-}" ]]; then + sudo -u "$user" pg_restore --clean --if-exists -d "$db" "$path" + else + PGPASSWORD="${POSTGRES_PASSWORD:-}" \ + pg_restore --clean --if-exists \ + -h "$host" \ + -p "${POSTGRES_PORT:-5432}" \ + -U "$user" \ + -d "$db" \ + "$path" + fi + ); then + systemctl start bayanat bayanat-celery + die "pg_restore failed; services restarted on existing DB" + fi + systemctl start bayanat bayanat-celery + log "Restore complete" +} + +# --- Update state + lock --- + +readonly STATE_DIR="$BAYANAT_ROOT/state" +readonly STATE_FILE="$STATE_DIR/update.json" +readonly LOCK_FILE="$STATE_DIR/update.lock" + +_now_iso() { date -u +%Y-%m-%dT%H:%M:%SZ; } + +write_state() { + # write_state + # Reads STATE_TARGET / STATE_PREVIOUS / STATE_SNAPSHOT / + # STATE_STARTED_AT / STATE_PROGRESS / STATE_ERROR_JSON from environment. + local phase="$1" label="$2" + mkdir -p "$STATE_DIR" + local tmp="$STATE_FILE.tmp" + cat > "$tmp" </dev/null || echo IDLE +} + +read_field() { + # $1 = field name + [[ -f "$STATE_FILE" ]] || return 1 + python3 -c "import json,sys; print(json.load(open('$STATE_FILE')).get('$1',''))" 2>/dev/null +} + +acquire_lock() { + mkdir -p "$STATE_DIR" + if [[ -f "$LOCK_FILE" ]]; then + local pid + pid=$(cat "$LOCK_FILE" 2>/dev/null || echo 0) + if [[ "$pid" -gt 0 ]] && kill -0 "$pid" 2>/dev/null; then + die "another update is running (pid $pid)" + fi + log "Removing stale lock (pid $pid)" + rm -f "$LOCK_FILE" + fi + echo $$ > "$LOCK_FILE" +} + +release_lock() { + rm -f "$LOCK_FILE" +} + +# --- Recovery dispatch --- + +recover_state() { + local phase + phase=$(read_phase) + case "$phase" in + IDLE) + return 0 + ;; + PREPARE_DONE) + log "Recovery: PREPARE_DONE — cleaning partial release and clearing state" + local target + target=$(read_field target || true) + if [[ -n "$target" ]]; then + rm -rf "$RELEASES_DIR/$target.partial" "$RELEASES_DIR/$target" + fi + clear_state + release_lock + ;; + MIGRATE_DONE) + log "Recovery: MIGRATE_DONE — starting services on previous release" + systemctl start bayanat bayanat-celery + if _wait_healthy 60; then + log "Recovery OK; operator should re-run update to finish" + clear_state + release_lock + else + STATE_ERROR_JSON='"MIGRATE_DONE recovery: previous release unhealthy"' + write_state NEEDS_INTERVENTION "Services unhealthy after recovery; restore snapshot manually" + exit 2 + fi + ;; + SWITCH_DONE) + log "Recovery: SWITCH_DONE — running code rollback" + rollback_code + ;; + SUCCESS|ROLLED_BACK) + clear_state + release_lock + ;; + NEEDS_INTERVENTION) + local snap prev + snap=$(read_field snapshot || true) + prev=$(read_field previous || true) + die "Operator intervention required. Snapshot: $snap Previous tag: $prev See: bayanat snapshots" + ;; + MIGRATE|SWITCH|ROLLBACK) + log "Recovery: $phase — attempting to restart services" + systemctl start bayanat bayanat-celery 2>/dev/null || true + if _wait_healthy 60; then + warn "Services healthy but update was interrupted mid-$phase" + warn "DB schema state is uncertain; re-run 'bayanat update' to finish" + clear_state + release_lock + else + export STATE_ERROR_JSON="\"interrupted during $phase; services unhealthy\"" + write_state NEEDS_INTERVENTION "Update interrupted mid-$phase; manual recovery required" + exit 2 + fi + ;; + *) + warn "Unknown phase '$phase' — clearing and starting fresh" + clear_state + release_lock + ;; + esac +} + +# --- Health probe --- + +_socket_path() { + # Hardened layout binds in /run/bayanat (releases are read-only, + # BAY-01-032); fall back to the legacy in-release socket for installs + # that predate it. + if [[ -S /run/bayanat/bayanat.sock ]]; then + echo /run/bayanat/bayanat.sock + else + echo "$CURRENT_LINK/bayanat.sock" + fi +} + +_socket_health() { + # curl the Flask /health over the unix socket. Returns 0 on HTTP 200. + curl -s --unix-socket "$(_socket_path)" \ + --max-time 3 -o /dev/null -w '%{http_code}' \ + http://localhost/health 2>/dev/null | grep -q '^200$' +} + +_db_ping() { + ( + _pg_load + local user="${POSTGRES_USER:-$APP_USER}" + local db="${POSTGRES_DB:-$APP_USER}" + local host="${POSTGRES_HOST:-localhost}" + if [[ "$host" == "localhost" && -z "${POSTGRES_PASSWORD:-}" ]]; then + sudo -u "$user" psql -d "$db" -c 'SELECT 1' -t >/dev/null 2>&1 + else + PGPASSWORD="${POSTGRES_PASSWORD:-}" \ + psql -h "$host" -U "$user" -d "$db" \ + -c 'SELECT 1' -t >/dev/null 2>&1 + fi + ) +} + +_redis_ping() { + # Redis requires auth on hardened installs (BAY-01-027). REDISCLI_AUTH + # keeps the password out of the process list, unlike -a. + ( + _pg_load + if [[ -n "${REDIS_PASSWORD:-}" ]]; then + REDISCLI_AUTH="$REDIS_PASSWORD" redis-cli ping 2>/dev/null | grep -q '^PONG$' + else + redis-cli ping 2>/dev/null | grep -q '^PONG$' + fi + ) +} + +_wait_healthy() { + # $1 = deadline in seconds (default 60) + local deadline=$(( $(date +%s) + ${1:-60} )) + while (( $(date +%s) < deadline )); do + if _socket_health && _db_ping && _redis_ping; then + return 0 + fi + sleep 1 + done + return 1 +} + +# --- Update pipeline --- + +update_preflight() { + preflight_checks + _verify_service_health + command -v pg_dump >/dev/null || die "pg_dump not in PATH" + command -v pg_restore >/dev/null || die "pg_restore not in PATH" + # pg_dump major version must match Postgres server major version + local client_major server_major + client_major=$(pg_dump --version | awk '{print $3}' | cut -d. -f1) + server_major=$( + _pg_load + local user="${POSTGRES_USER:-$APP_USER}" + local db="${POSTGRES_DB:-$APP_USER}" + local host="${POSTGRES_HOST:-localhost}" + if [[ "$host" == "localhost" && -z "${POSTGRES_PASSWORD:-}" ]]; then + sudo -u "$user" psql -d "$db" \ + -c 'SHOW server_version_num' -t 2>/dev/null + else + PGPASSWORD="${POSTGRES_PASSWORD:-}" \ + psql -h "$host" -U "$user" -d "$db" \ + -c 'SHOW server_version_num' -t 2>/dev/null + fi | tr -d ' ' | cut -c1-2 + ) + [[ -n "$client_major" && -n "$server_major" ]] \ + || die "could not determine pg_dump/server versions" + [[ "$client_major" == "$server_major" ]] \ + || die "pg_dump major ($client_major) != server major ($server_major)" + # Disk in backups (need >= 2 GB for snapshot headroom) + local backups_free_kb + backups_free_kb=$(df --output=avail "$SHARED_DIR/backups" | tail -1 | tr -d ' ') + [[ "$backups_free_kb" -ge 2097152 ]] \ + || die "need >= 2GB free in $SHARED_DIR/backups for snapshot" + # Schema aligned with models + flask_run "$(current_version)" check-db-alignment >/dev/null \ + || die "schema drift detected; run 'flask check-db-alignment' for details" + # Flask doctor + flask_run "$(current_version)" doctor >/dev/null \ + || die "flask doctor failed" +} + +do_prepare() { + # $1 = target tag + local target="$1" + local current + current=$(current_version || echo 0) + [[ "$target" != "$current" ]] || die "already on $target" + log "PREPARE: fetching $target" + update_preflight + _fetch_release "$target" + _install_deps "$target" + _link_shared "$target" + acquire_lock + export STATE_TARGET="$target" + export STATE_PREVIOUS="$current" + export STATE_STARTED_AT + STATE_STARTED_AT="$(_now_iso)" + export STATE_PROGRESS="Fetched $target, ready to migrate" + write_state PREPARE_DONE "Prepared $target, ready to migrate" +} + +do_migrate() { + local target="$STATE_TARGET" + local prev="$STATE_PREVIOUS" + log "MIGRATE: stopping services" + export STATE_PROGRESS="Stopping services" + write_state MIGRATE "Stopping services for maintenance window" + systemctl stop bayanat bayanat-celery + log "MIGRATE: pruning old snapshots" + prune_snapshots + export STATE_PROGRESS="Taking pre-update snapshot" + write_state MIGRATE "Taking pre-update snapshot" + export STATE_SNAPSHOT + if ! STATE_SNAPSHOT=$(snapshot_pg_dump "$prev" "$target"); then + export STATE_ERROR_JSON='"snapshot failed"' + write_state NEEDS_INTERVENTION "Pre-update snapshot failed; check backups disk and pg_dump" + systemctl start bayanat bayanat-celery || true + release_lock + exit 2 + fi + log "MIGRATE: running migrations" + export STATE_PROGRESS="Running migrations" + write_state MIGRATE "Running migrations" + if ! flask_run "$target" db upgrade; then + export STATE_ERROR_JSON='"db upgrade failed"' + write_state NEEDS_INTERVENTION "Migration failed; previous release remains linked" + systemctl start bayanat bayanat-celery || true + release_lock + exit 2 + fi + export STATE_PROGRESS="Migration complete" + write_state MIGRATE_DONE "Migration complete, switching to new release" +} + +do_switch_verify() { + local target="$STATE_TARGET" + export STATE_PROGRESS="Swapping to $target" + write_state SWITCH "Swapping current -> $target" + swap_symlink "$RELEASES_DIR/$target" + systemctl start bayanat bayanat-celery + write_state SWITCH_DONE "Verifying new release" + export STATE_PROGRESS="Waiting for health probe" + if _wait_healthy 60; then + # Refresh /usr/local/bin/bayanat from the now-active release so the + # system CLI stays in lockstep with the deployed code. Done AFTER the + # health probe so a bad release never overwrites a working CLI. + _install_self "$target" + export STATE_PROGRESS="Update successful" + write_state SUCCESS "Update to $target complete" + clear_state + release_lock + log "SUCCESS: running $target" + return 0 + else + warn "Health probe failed; rolling back code" + rollback_code + return 1 + fi +} + +rollback_code() { + local prev="$STATE_PREVIOUS" + if [[ -z "$prev" ]]; then + export STATE_ERROR_JSON='"cannot roll back: no previous tag recorded"' + write_state NEEDS_INTERVENTION "Cannot roll back: no previous tag recorded" + exit 2 + fi + write_state ROLLBACK "Reverting symlink to $prev" + systemctl stop bayanat bayanat-celery || true + swap_symlink "$RELEASES_DIR/$prev" + systemctl start bayanat bayanat-celery + if _wait_healthy 60; then + export STATE_PROGRESS="Rolled back to $prev" + write_state ROLLED_BACK "Rolled back to $prev; snapshot retained" + clear_state + release_lock + log "ROLLED_BACK: on $prev; snapshot: ${STATE_SNAPSHOT:-none}" + exit 1 + else + export STATE_ERROR_JSON='"code rollback failed health probe"' + write_state NEEDS_INTERVENTION "Rollback to $prev unhealthy; restore snapshot ${STATE_SNAPSHOT:-} manually" + exit 2 + fi +} + # --- Step functions (each is idempotent) --- _install_system_packages() { @@ -96,7 +600,8 @@ _install_system_packages() { git postgresql postgresql-contrib postgis redis-server \ python3 python3-dev build-essential \ libpq-dev libxml2-dev libxslt1-dev libssl-dev libffi-dev \ - libjpeg-dev libzip-dev libimage-exiftool-perl ffmpeg curl wget jq + libjpeg-dev libzip-dev libimage-exiftool-perl ffmpeg curl wget jq \ + minisign } _install_uv() { @@ -138,59 +643,207 @@ _setup_app_user() { fi log "Created user $APP_USER" fi - usermod -aG "$APP_USER" caddy 2>/dev/null || true + # Per-service accounts (BAY-01-032): uWSGI and Celery run as separate + # nologin users so a compromised worker cannot act as the web app. Both + # share $APP_GROUP for read access to releases and shared dirs. + local svc_user + for svc_user in "$WEB_USER" "$CELERY_USER"; do + if ! id "$svc_user" &>/dev/null; then + useradd --system --no-create-home -g "$APP_GROUP" \ + -s /usr/sbin/nologin "$svc_user" + log "Created service user $svc_user" + fi + done + usermod -aG "$APP_GROUP" caddy 2>/dev/null || true } _setup_database() { systemctl enable --now postgresql redis-server log "Setting up database..." - sudo -u postgres createuser -s "$APP_USER" 2>/dev/null || true + # Role is a plain owner with no instance-level privileges (BAY-01-031). + # Extensions are created by the postgres superuser below so the app role + # doesn't need that privilege. The role keeps table ownership because the + # dynamic-fields feature issues ALTER TABLE at runtime; a DML-only + # runtime role would break that product feature. + sudo -u postgres createuser "$APP_USER" 2>/dev/null || true + # Idempotent for upgrades: strip any instance-level privilege a previous + # install may have granted. + sudo -u postgres psql -c \ + "ALTER ROLE \"$APP_USER\" NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION;" \ + 2>/dev/null || true sudo -u postgres createdb bayanat -O "$APP_USER" 2>/dev/null || true - - # Configure pg_hba trust auth for app user - local pg_hba + sudo -u postgres psql -d bayanat \ + -c "CREATE EXTENSION IF NOT EXISTS pg_trgm;" \ + -c "CREATE EXTENSION IF NOT EXISTS postgis;" >/dev/null + # Only the app role (and superuser) may connect to the app database. + sudo -u postgres psql \ + -c "REVOKE CONNECT ON DATABASE bayanat FROM PUBLIC;" \ + -c "GRANT CONNECT ON DATABASE bayanat TO \"$APP_USER\";" >/dev/null + + # Configure pg_hba peer auth for app user. Peer auth maps the OS user to + # the PG role over the local socket, so only mapped processes can connect + # as $APP_USER. Replaces the previous 'trust' rule which let any local OS + # user connect as the app role. The ident map admits the per-service + # accounts (BAY-01-032) plus $APP_USER itself for admin operations. + local pg_hba pg_ident pg_hba=$(find /etc/postgresql -name pg_hba.conf 2>/dev/null | head -1) [[ -n "$pg_hba" ]] || die "Cannot find pg_hba.conf" + pg_ident=$(find /etc/postgresql -name pg_ident.conf 2>/dev/null | head -1) + [[ -n "$pg_ident" ]] || die "Cannot find pg_ident.conf" - local rule="local all $APP_USER trust" + local os_user + for os_user in "$APP_USER" "$WEB_USER" "$CELERY_USER"; do + if ! grep -qE "^bayanat[[:space:]]+${os_user}[[:space:]]" "$pg_ident"; then + printf '%-15s %-23s %s\n' "bayanat" "$os_user" "$APP_USER" >> "$pg_ident" + fi + done + + local rule="local all $APP_USER peer map=bayanat" if grep -qF "$rule" "$pg_hba"; then log "pg_hba.conf already configured" else log "Configuring pg_hba.conf for $APP_USER" + # Remove any prior rule for this user from a previous install + sed -i "/^local[[:space:]]\+all[[:space:]]\+${APP_USER}[[:space:]]/d" "$pg_hba" # Insert before the "local all all" catch-all, or append if grep -q "^local.*all.*all" "$pg_hba"; then sed -i "/^local.*all.*all/i $rule" "$pg_hba" else echo "$rule" >> "$pg_hba" fi - systemctl reload postgresql fi + systemctl reload postgresql +} + +REDIS_PW="" + +_setup_redis() { + # Require password auth on the loopback Redis listener (BAY-01-027). + # The password is generated once, stored in shared/.env (read by the app + # via REDIS_PASSWORD), and asserted into redis.conf on every run. + local pw="" + if [[ -f "$SHARED_DIR/.env" ]]; then + pw=$(grep -E '^REDIS_PASSWORD=' "$SHARED_DIR/.env" | head -1 \ + | cut -d= -f2- | tr -d "'\"") || true + fi + if [[ -z "$pw" ]]; then + pw=$(openssl rand -hex 24) + if [[ -f "$SHARED_DIR/.env" ]]; then + printf "REDIS_PASSWORD='%s'\n" "$pw" >> "$SHARED_DIR/.env" + fi + fi + REDIS_PW="$pw" + + local conf="/etc/redis/redis.conf" + [[ -f "$conf" ]] || die "Cannot find $conf" + if grep -qE '^[[:space:]]*requirepass ' "$conf"; then + sed -i -E "s|^[[:space:]]*requirepass .*|requirepass $pw|" "$conf" + else + printf '\n# Managed by bayanat installer (BAY-01-027)\nrequirepass %s\n' "$pw" >> "$conf" + fi + systemctl restart redis-server } _create_directories() { - mkdir -p "$RELEASES_DIR" "$SHARED_DIR/media" "$SHARED_DIR/backups" \ - "$LOGS_DIR" + mkdir -p "$RELEASES_DIR" "$SHARED_DIR/media" "$SHARED_DIR/exports" \ + "$SHARED_DIR/imports" "$SHARED_DIR/backups" "$RUNTIME_DIR" \ + "$LOGS_DIR" "$STATE_DIR" + # STATE_DIR stays root-owned (BAY-01-013): the updater writes it as root and + # the app only reads it. Keeping it out of the service user's reach removes + # the symlink-race chown primitive into root-owned files. + _secure_state_dir +} + +_secure_state_dir() { + # Root-owned updater state dir. World-readable so the app (service user) + # can read update.json for the status display, but not writable by it. + chown -R root:root "$STATE_DIR" + chmod 755 "$STATE_DIR" + [[ -f "$STATE_FILE" ]] && chmod 644 "$STATE_FILE" + return 0 +} + +_set_permissions() { + # Least-privilege install layout (BAY-01-030/032). Root owns the tree; + # service users reach it only through $APP_GROUP, and "other" is locked + # out everywhere. Top-level dirs deny traversal, so files inside need no + # recursive sweep (kept non-recursive for media, which can be large). + chown root:"$APP_GROUP" "$BAYANAT_ROOT" "$RELEASES_DIR" "$SHARED_DIR" + chmod 750 "$BAYANAT_ROOT" "$RELEASES_DIR" "$SHARED_DIR" + + # App-writable shared areas: setgid keeps new files in $APP_GROUP so the + # web and worker accounts can read each other's output. + local d + for d in "$SHARED_DIR/media" "$SHARED_DIR/exports" "$SHARED_DIR/imports" \ + "$SHARED_DIR/backups" "$RUNTIME_DIR" "$LOGS_DIR"; do + chown root:"$APP_GROUP" "$d" + chmod 2770 "$d" + done + + # Secrets: group-readable only (the services read them), never writable + # by the services and never world-readable. + local f + for f in "$SHARED_DIR/.env" "$SHARED_DIR/uwsgi-prod.ini"; do + if [[ -f "$f" ]]; then + chown root:"$APP_GROUP" "$f" + chmod 640 "$f" + fi + done + + # Log files created by admin CLI runs (umask 022) must stay appendable + # by both service accounts. + find "$LOGS_DIR" -type f -exec chgrp "$APP_GROUP" {} + -exec chmod 660 {} + 2>/dev/null || true + + _secure_state_dir } -_clone_release() { +# Fetch a release as a signed tarball and verify it before install (BAY-01-017). +# Replaces the old unsigned `git clone`: an unsigned or tampered release is +# refused, so a reachable update trigger can at worst install authentic SJAC +# code. SJAC attaches `bayanat-.tar.gz` + `.minisig` to each GitHub +# release; the signing procedure is in docs/deployment/release-signing.md. +_fetch_release() { local tag="$1" local dest="$RELEASES_DIR/$tag" + local asset="bayanat-$tag.tar.gz" + local url="https://github.com/${BAYANAT_REPO:-$DEFAULT_REPO}/releases/download/$tag/$asset" - if [[ -d "$dest/.git" ]]; then - if sudo -u "$APP_USER" git -C "$dest" rev-parse HEAD &>/dev/null; then - log "Release $tag already cloned" - return 0 - fi - warn "Partial clone detected, removing $dest" - rm -rf "$dest" - elif [[ -d "$dest" ]]; then - warn "Invalid release directory, removing $dest" - rm -rf "$dest" + # A complete release is marked done; anything else is a partial fetch. + if [[ -f "$dest/.bayanat-release" ]]; then + log "Release $tag already fetched" + return 0 fi - - log "Cloning $tag..." - git clone --depth 1 --branch "$tag" "$GIT_URL" "$dest" + rm -rf "$dest" + + local tmp + tmp=$(mktemp -d) + + log "Downloading $tag..." + curl -fsSL --max-time 300 -o "$tmp/$asset" "$url" \ + || die "Could not download $asset from the $tag release" \ + "Confirm the release exists and ships a signed tarball asset." + curl -fsSL --max-time 60 -o "$tmp/$asset.minisig" "$url.minisig" \ + || die "Release $tag is unsigned (no $asset.minisig)" \ + "Refusing to install unverified code. Update manually only if you trust this release." + + log "Verifying signature against the SJAC release key..." + minisign -Vm "$tmp/$asset" -P "$RELEASE_PUBKEY" \ + || die "Signature verification FAILED for $tag" \ + "The tarball does not match SJAC's signing key. Refusing to install." + + log "Extracting $tag..." + mkdir -p "$dest" + tar -xzf "$tmp/$asset" --strip-components=1 -C "$dest" \ + || die "Failed to extract $asset" + touch "$dest/.bayanat-release" + rm -rf "$tmp" + + # Releases are root-owned and read-only for the service accounts + # (BAY-01-030/032): group gets read+exec via $APP_GROUP, no write, and + # "other" is locked out. + chown -R root:"$APP_GROUP" "$dest" + chmod -R o-rwx,g-w "$dest" } _generate_env() { @@ -215,6 +868,15 @@ FORCE_HTTPS=$force_https LOG_DIR=$LOGS_DIR BACKUPS_LOCAL_PATH=$SHARED_DIR/backups +BAYANAT_CONFIG_FILE=$RUNTIME_DIR/config.json + +POSTGRES_HOST=localhost +POSTGRES_PORT=5432 +POSTGRES_USER=$APP_USER +POSTGRES_PASSWORD= +POSTGRES_DB=$APP_USER + +REDIS_PASSWORD='$REDIS_PW' EOF } @@ -224,18 +886,40 @@ _link_shared() { [[ -f "$SHARED_DIR/.env" ]] && ln -sfn "$SHARED_DIR/.env" "$release/.env" - if [[ -d "$release/enferno/media" ]] && [[ ! -L "$release/enferno/media" ]]; then - rm -rf "$release/enferno/media" - fi - ln -sfn "$SHARED_DIR/media" "$release/enferno/media" + # Releases are read-only for the services (BAY-01-032), so every + # app-written path is redirected into shared/. + local d + for d in media exports imports; do + if [[ -d "$release/enferno/$d" ]] && [[ ! -L "$release/enferno/$d" ]]; then + rm -rf "$release/enferno/$d" + fi + ln -sfn "$SHARED_DIR/$d" "$release/enferno/$d" + done + + # Sentinels the app touches to request a web reload / worker restart. + # uWSGI watches reload.ini (touch-reload); a systemd path unit watches + # restart-celery. Both live in the app-writable runtime dir. + mkdir -p "$RUNTIME_DIR" + local s + for s in reload.ini restart-celery; do + touch "$RUNTIME_DIR/$s" + chown root:"$APP_GROUP" "$RUNTIME_DIR/$s" + chmod 660 "$RUNTIME_DIR/$s" + ln -sfn "$RUNTIME_DIR/$s" "$release/$s" + done } _install_deps() { + # Runs as root: releases (including .venv) are read-only for the service + # accounts (BAY-01-032). local tag="$1" log "Installing Python dependencies..." - sudo -u "$APP_USER" \ - /usr/local/bin/uv sync --frozen --python 3.12 \ + /usr/local/bin/uv sync --frozen --python 3.12 \ --directory "$RELEASES_DIR/$tag" + # The venv is created after _fetch_release's ownership pass, so it needs + # its own: group-readable for the service accounts, not writable. + chown -R root:"$APP_GROUP" "$RELEASES_DIR/$tag/.venv" + chmod -R o-rwx,g-w "$RELEASES_DIR/$tag/.venv" } _init_database() { @@ -253,7 +937,40 @@ _init_database() { fi } +ADMIN_USERNAME="" +ADMIN_PASSWORD="" + +_bootstrap_admin() { + # Provision the initial admin out-of-band. Replaces the deleted + # /api/create-admin wizard endpoint, which was unauthenticated and + # claimable by the first network client during the install window. + # The password is fed to flask install over stdin (--password-stdin) + # so it is never visible in /proc//cmdline or `ps`. + local tag="$1" + local pw + pw=$(python3 -c "import secrets; print(secrets.token_urlsafe(20))" 2>/dev/null) || \ + pw=$(openssl rand -base64 24 | tr -d '/+=' | head -c 24) + + log "Bootstrapping initial admin user..." + local out + out=$(printf '%s\n' "$pw" \ + | flask_run "$tag" install --username admin --password-stdin 2>&1) || true + + if echo "$out" | grep -q "already installed"; then + log "Admin user already exists, skipping bootstrap" + elif echo "$out" | grep -q "installed successfully"; then + ADMIN_USERNAME="admin" + ADMIN_PASSWORD="$pw" + else + warn "Admin bootstrap unexpected output:" + warn "$out" + fi +} + _install_uwsgi_config() { + # Socket lives in /run/bayanat (systemd RuntimeDirectory): the release + # dir is read-only for the service user (BAY-01-032). touch-reload + # watches the shared runtime sentinel the app can write. cat > "$SHARED_DIR/uwsgi-prod.ini" << EOF [uwsgi] virtualenv=.venv @@ -261,27 +978,60 @@ module=run:app master=true processes=1 threads=2 -http-socket=$CURRENT_LINK/bayanat.sock +http-socket=/run/bayanat/bayanat.sock chmod-socket=660 vacuum=true buffer-size=8192 -touch-reload=reload.ini +touch-reload=$RUNTIME_DIR/reload.ini reload-mercy=5 worker-reload-mercy=3 EOF } _install_systemd() { + # Sandboxing directives (BAY-01-033): no privilege escalation, read-only + # host filesystem outside the explicit shared/log paths, isolated tmp and + # devices, restricted /proc and kernel interfaces, no retained + # capabilities. HOME points at the per-service cache dir for libraries + # that write font/model caches. + local hardening + hardening="NoNewPrivileges=true +ProtectSystem=strict +ProtectHome=true +ReadWritePaths=$SHARED_DIR $LOGS_DIR +PrivateTmp=true +PrivateDevices=true +ProtectProc=invisible +ProcSubset=pid +ProtectKernelTunables=true +ProtectKernelModules=true +ProtectKernelLogs=true +ProtectControlGroups=true +ProtectClock=true +RestrictNamespaces=true +RestrictSUIDSGID=true +RestrictRealtime=true +LockPersonality=true +RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 +SystemCallArchitectures=native +SystemCallFilter=@system-service +CapabilityBoundingSet=" + cat > /etc/systemd/system/bayanat.service << EOF [Unit] Description=Bayanat web application After=network.target postgresql.service redis.service [Service] -User=$APP_USER -Group=$APP_USER +User=$WEB_USER +Group=$APP_GROUP +UMask=0007 WorkingDirectory=$CURRENT_LINK EnvironmentFile=$SHARED_DIR/.env +Environment=HOME=/var/cache/bayanat-web +CacheDirectory=bayanat-web +RuntimeDirectory=bayanat +RuntimeDirectoryMode=0750 ExecStart=$CURRENT_LINK/.venv/bin/uwsgi --ini $SHARED_DIR/uwsgi-prod.ini Restart=always RestartSec=3 @@ -290,6 +1040,7 @@ KillSignal=SIGQUIT TimeoutStopSec=10 Type=notify NotifyAccess=all +$hardening [Install] WantedBy=multi-user.target @@ -301,16 +1052,43 @@ Description=Bayanat Celery worker After=network.target postgresql.service redis.service [Service] -User=$APP_USER -Group=$APP_USER +User=$CELERY_USER +Group=$APP_GROUP +UMask=0007 WorkingDirectory=$CURRENT_LINK EnvironmentFile=$SHARED_DIR/.env -ExecStart=$CURRENT_LINK/.venv/bin/celery -A enferno.tasks worker --autoscale 2,5 -B -Q celery,ocr +Environment=HOME=/var/cache/bayanat-celery +CacheDirectory=bayanat-celery +ExecStart=$CURRENT_LINK/.venv/bin/celery -A enferno.tasks worker --autoscale 2,5 -B -Q celery,ocr -s $RUNTIME_DIR/celerybeat-schedule Restart=always RestartSec=3 +$hardening + +[Install] +WantedBy=multi-user.target +EOF + + # Celery restart requests (BAY-01-032): the web app cannot (and must not) + # sudo; it touches the runtime sentinel instead and this path unit + # performs the restart as root. + cat > /etc/systemd/system/bayanat-celery-restart.path << EOF +[Unit] +Description=Watch for Bayanat Celery restart requests + +[Path] +PathChanged=$RUNTIME_DIR/restart-celery [Install] WantedBy=multi-user.target +EOF + + cat > /etc/systemd/system/bayanat-celery-restart.service << 'EOF' +[Unit] +Description=Restart Bayanat Celery worker on request + +[Service] +Type=oneshot +ExecStart=/usr/bin/systemctl restart bayanat-celery.service EOF } @@ -320,7 +1098,7 @@ _configure_caddy() { if [[ "$domain" == "localhost" ]]; then cat > /etc/caddy/Caddyfile << 'EOF' :80 { - reverse_proxy unix//opt/bayanat/current/bayanat.sock + reverse_proxy unix//run/bayanat/bayanat.sock handle_path /static/* { root * /opt/bayanat/current/enferno/static file_server @@ -333,7 +1111,7 @@ EOF else cat > /etc/caddy/Caddyfile << EOF $domain { - reverse_proxy unix//opt/bayanat/current/bayanat.sock + reverse_proxy unix//run/bayanat/bayanat.sock handle_path /static/* { root * /opt/bayanat/current/enferno/static file_server @@ -346,13 +1124,16 @@ EOF fi } -_install_sudoers() { - cat > /etc/sudoers.d/bayanat << 'EOF' -bayanat ALL=(root) NOPASSWD: /usr/local/bin/bayanat update -bayanat ALL=(root) NOPASSWD: /usr/local/bin/bayanat status -bayanat ALL=(root) NOPASSWD: /usr/bin/systemctl restart bayanat-celery -EOF - chmod 440 /etc/sudoers.d/bayanat +_install_self() { + # Copy the bayanat CLI from the given release directory to + # /usr/local/bin/bayanat. Source is $RELEASES_DIR/$tag/bayanat, NOT $0 — + # under `curl ... | sudo bash -s install`, $0 is "bash" and readlink + # resolves to the shell binary. $1 = tag. + local tag="${1:?_install_self requires a tag arg}" + local src="$RELEASES_DIR/$tag/bayanat" + [[ -f "$src" ]] || die "cannot find CLI source at $src" + install -m 0755 -o root -g root "$src" /usr/local/bin/bayanat + log "Installed CLI at /usr/local/bin/bayanat" } # --- Install --- @@ -373,6 +1154,7 @@ cmd_install() { # Users and database _setup_app_user _setup_database + _setup_redis # Application _create_directories @@ -382,18 +1164,19 @@ cmd_install() { [[ -n "$tag" ]] || die "No release tags found" log "Latest release: $tag" - _clone_release "$tag" + _fetch_release "$tag" if [[ ! -f "$SHARED_DIR/.env" ]]; then log "Generating .env..." _generate_env "$domain" fi - chown -R "$APP_USER:$APP_USER" "$BAYANAT_ROOT" + _set_permissions _link_shared "$tag" _install_deps "$tag" _init_database "$tag" + _bootstrap_admin "$tag" # Activate release swap_symlink "$RELEASES_DIR/$tag" @@ -401,22 +1184,46 @@ cmd_install() { # System integration _install_uwsgi_config - _install_systemd _configure_caddy "$domain" - _install_sudoers + _install_systemd + _install_self "$tag" + + # Services no longer get any sudo grant (BAY-01-032); drop the file a + # previous install may have written. + rm -f /etc/sudoers.d/bayanat - chown -R "$APP_USER:$APP_USER" "$BAYANAT_ROOT" + _set_permissions systemctl daemon-reload - systemctl enable --now bayanat bayanat-celery + systemctl enable --now bayanat bayanat-celery bayanat-celery-restart.path systemctl restart caddy _verify_service_health - log "Installation complete" + local access_url if [[ "$domain" == "localhost" ]]; then - log "Access: http://$(hostname -I | awk '{print $1}')" + access_url="http://$(hostname -I | awk '{print $1}')" else - log "Access: https://$domain" + access_url="https://$domain" + fi + + log "Installation complete" + log "Access: $access_url" + + if [[ -n "$ADMIN_PASSWORD" ]]; then + log "" + log "============================================================" + log " Bayanat is ready. Sign in to finish setup:" + log "" + log " URL : $access_url/login" + log " Username : $ADMIN_USERNAME" + log " Password : $ADMIN_PASSWORD" + log "" + log " Save these credentials now - the password is not stored" + log " in plaintext anywhere. After signing in, the setup wizard" + log " will walk you through language, default data, and other" + log " configuration. Change the password from your account" + log " settings." + log "============================================================" fi } @@ -435,11 +1242,11 @@ _verify_service_health() { fi done - # Wait for app to respond on socket - local sock="$CURRENT_LINK/bayanat.sock" + # Wait for the app's /health (DB + Redis) to go green; a bare / would + # pass on a redirect even when the app cannot reach its backends. local i for ((i=1; i<=retries; i++)); do - if curl -sf --unix-socket "$sock" http://localhost/ -o /dev/null 2>/dev/null; then + if _socket_health; then log "Application responding" return 0 fi @@ -448,6 +1255,63 @@ _verify_service_health() { warn "Application not responding on socket after $((retries * delay))s (may still be starting)" } +# --- Update --- + +_ensure_runtime_layout() { + # Layout migration for installs that predate shared/runtime + # (BAY-01-030/032). New releases are root-owned, so config.json must move + # out of the release dir before the app loses write access to it. + mkdir -p "$RUNTIME_DIR" "$SHARED_DIR/exports" "$SHARED_DIR/imports" + local d + for d in "$RUNTIME_DIR" "$SHARED_DIR/exports" "$SHARED_DIR/imports"; do + chown root:"$APP_GROUP" "$d" + chmod 2770 "$d" + done + if ! grep -q '^BAYANAT_CONFIG_FILE=' "$SHARED_DIR/.env" 2>/dev/null; then + if [[ -f "$CURRENT_LINK/config.json" ]]; then + cp -p "$CURRENT_LINK/config.json" "$RUNTIME_DIR/config.json" + fi + echo "BAYANAT_CONFIG_FILE=$RUNTIME_DIR/config.json" >> "$SHARED_DIR/.env" + fi +} + +cmd_update() { + local flag="${1:-}" + case "$flag" in + --check) + local cur latest + cur=$(current_version || echo "not installed") + latest=$(latest_remote_tag) + echo "current: $cur" + echo "latest: $latest" + if [[ "${cur#v}" == "${latest#v}" ]]; then + echo "up to date" + else + echo "update available" + fi + return 0 + ;; + --recover) + require_root + recover_state + return 0 + ;; + esac + require_root + recover_state + _ensure_runtime_layout + local target="${1:-}" + if [[ -z "$target" ]]; then + target=$(latest_remote_tag) + fi + # Keep $target verbatim. Tags and release dir names use the v-prefix form + # (e.g. v4.0.0), matching the installer's convention in _fetch_release and + # _link_shared. Stripping is only for display / comparison. + do_prepare "$target" + do_migrate + do_switch_verify +} + # --- Status --- cmd_status() { @@ -474,6 +1338,17 @@ cmd_status() { for svc in bayanat bayanat-celery caddy; do printf " %-20s %s\n" "$svc" "$(systemctl is-active "$svc" 2>/dev/null || echo 'unknown')" done + + echo "" + local phase + phase=$(read_phase) + echo "Update state: $phase" + if [[ "$phase" != "IDLE" ]]; then + echo " target: $(read_field target || echo '')" + echo " previous: $(read_field previous || echo '')" + echo " snapshot: $(read_field snapshot || echo '')" + echo " updated_at: $(read_field updated_at || echo '')" + fi } # --- Usage --- @@ -486,18 +1361,27 @@ Usage: bayanat [options] Commands: install [domain] Install Bayanat (default: localhost) - status Show version and service status + update [] Update Bayanat to (default: latest release) + update --check Show current vs latest; no changes + update --recover Recover from a stuck update state file + snapshots List pre-update snapshots + restore Restore a pre-update snapshot (prompts confirmation) + status Show version, services, and update state Environment: - BAYANAT_REPO GitHub repo (default: sjacorg/bayanat) + BAYANAT_REPO GitHub repo (default: sjacorg/bayanat) + BAYANAT_SNAPSHOT_RETENTION_DAYS Snapshot retention floor (default: 30) EOF } # --- Main --- case "${1:-}" in - install) shift; cmd_install "$@" ;; - status) cmd_status ;; + install) shift; cmd_install "$@" ;; + update) shift; cmd_update "$@" ;; + snapshots) require_root; list_snapshots ;; + restore) require_root; [[ -n "${2:-}" ]] || die "usage: bayanat restore "; restore_pg "$2" ;; + status) cmd_status ;; -h|--help|help) usage ;; *) usage; exit 1 ;; esac diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 63a6225c2..1141bb99d 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -1,7 +1,10 @@ services: postgres: container_name: postgres - image: 'postgis/postgis:15-3.3' + image: 'postgis/postgis:16-3.5@sha256:a780ff6331b384e4c6d1033d5c42f4fe1270721b3e19260448241fa1b64a7637' + # postgis/postgis publishes amd64 only; runs emulated on ARM hosts + platform: linux/amd64 + user: postgres environment: - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -14,7 +17,7 @@ services: security_opt: - no-new-privileges:true tmpfs: - - /var/run/postgresql + - /var/run/postgresql healthcheck: test: "pg_isready -d ${POSTGRES_DB} -U ${POSTGRES_USER}" interval: 3s @@ -22,17 +25,23 @@ services: redis: container_name: redis - image: 'redis:latest' + image: 'redis:7.4-alpine@sha256:6ab0b6e7381779332f97b8ca76193e45b0756f38d4c0dcda72dbb3c32061ab99' + user: redis expose: - '6379' - command: redis-server --requirepass '${REDIS_PASSWORD}' + environment: + - REDIS_PASSWORD=${REDIS_PASSWORD} + # password is written to a config file in-container, never passed via argv + command: sh -c 'echo "requirepass $$REDIS_PASSWORD" > /tmp/redis.conf && exec redis-server /tmp/redis.conf' read_only: true security_opt: - no-new-privileges:true + tmpfs: + - /tmp volumes: - - 'redis_dev_data:/var/lib/redis/data:rw' + - 'redis_dev_data:/data:rw' healthcheck: - test: [ "CMD", "redis-cli", "--no-auth-warning", "-a", "${REDIS_PASSWORD}", "ping" ] + test: [ "CMD-SHELL", "REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli ping | grep -q PONG" ] interval: 3s retries: 10 @@ -43,7 +52,6 @@ services: dockerfile: ./flask/Dockerfile args: - ROLE=flask - - ENV_FILE=${ENV_FILE:-.env.dev} volumes: - 'bayanat_dev_backups:/app/backups/:rw' - 'bayanat_dev_media:/app/enferno/media/:rw' @@ -67,7 +75,6 @@ services: dockerfile: ./flask/Dockerfile args: - ROLE=celery - - ENV_FILE=${ENV_FILE:-.env.dev} volumes_from: - bayanat read_only: true diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 31c37a9b5..2364f54f2 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -1,7 +1,10 @@ services: postgres: container_name: postgres - image: 'postgis/postgis:15-3.3' + image: 'postgis/postgis:16-3.5@sha256:a780ff6331b384e4c6d1033d5c42f4fe1270721b3e19260448241fa1b64a7637' + # postgis/postgis publishes amd64 only; runs emulated on ARM hosts + platform: linux/amd64 + user: postgres environment: - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -22,17 +25,23 @@ services: redis: container_name: redis - image: 'redis:latest' + image: 'redis:7.4-alpine@sha256:6ab0b6e7381779332f97b8ca76193e45b0756f38d4c0dcda72dbb3c32061ab99' + user: redis expose: - '6379' - command: redis-server --requirepass '${REDIS_PASSWORD}' + environment: + - REDIS_PASSWORD=${REDIS_PASSWORD} + # password is written to a config file in-container, never passed via argv + command: sh -c 'echo "requirepass $$REDIS_PASSWORD" > /tmp/redis.conf && exec redis-server /tmp/redis.conf' read_only: true security_opt: - no-new-privileges:true + tmpfs: + - /tmp volumes: - - 'redis_test_data:/var/lib/redis/data:rw' + - 'redis_test_data:/data:rw' healthcheck: - test: [ "CMD", "redis-cli", "--no-auth-warning", "-a", "${REDIS_PASSWORD}", "ping" ] + test: [ "CMD-SHELL", "REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli ping | grep -q PONG" ] interval: 3s retries: 10 diff --git a/docker-compose.yml b/docker-compose.yml index 3d7cf788e..ae9a42b60 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,11 @@ services: postgres: container_name: postgres - image: 'postgis/postgis:15-3.3' + # PG major upgrades (15 -> 16) require a dump/restore of postgres_data + image: 'postgis/postgis:16-3.5@sha256:a780ff6331b384e4c6d1033d5c42f4fe1270721b3e19260448241fa1b64a7637' + # postgis/postgis publishes amd64 only; runs emulated on ARM hosts + platform: linux/amd64 + user: postgres environment: - POSTGRES_USER=${POSTGRES_USER} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD} @@ -22,36 +26,41 @@ services: redis: container_name: redis - image: 'redis:latest' + image: 'redis:7.4-alpine@sha256:6ab0b6e7381779332f97b8ca76193e45b0756f38d4c0dcda72dbb3c32061ab99' + user: redis expose: - '6379' - command: redis-server --requirepass '${REDIS_PASSWORD}' + # password is read from the mounted secret, never passed via argv + environment: + - REDIS_PASSWORD=${REDIS_PASSWORD} + # password is written to a config file in-container, never passed via argv + command: sh -c 'echo "requirepass $$REDIS_PASSWORD" > /tmp/redis.conf && exec redis-server /tmp/redis.conf' read_only: true security_opt: - no-new-privileges:true + tmpfs: + - /tmp volumes: - - 'redis_data:/var/lib/redis/data:rw' + - 'redis_data:/data:rw' healthcheck: - test: [ "CMD", "redis-cli", "--no-auth-warning", "-a", "${REDIS_PASSWORD}", "ping" ] + test: [ "CMD-SHELL", "REDISCLI_AUTH=\"$$REDIS_PASSWORD\" redis-cli ping | grep -q PONG" ] interval: 3s retries: 10 bayanat: container_name: bayanat - image: 'bayanat/bayanat:latest' build: context: . dockerfile: ./flask/Dockerfile args: - ROLE=flask - - ENV_FILE=${ENV_FILE:-.env} volumes: - '${PWD}/backups:/app/backups/:rw' - '${MEDIA_PATH:-./enferno/media}:/app/enferno/media/:rw' - '${PWD}/enferno/imports:/app/enferno/imports/:rw' - '${PWD}/logs/:/app/logs/:rw' - '${PWD}/config.json:/app/config.json:rw' - - '${PWD}/${ENV_FILE:-.env}:/app/.env:ro' + - '${PWD}/${ENV_FILE:-.env.docker}:/app/.env:ro' depends_on: postgres: condition: service_healthy @@ -72,7 +81,6 @@ services: dockerfile: ./flask/Dockerfile args: - ROLE=celery - - ENV_FILE=${ENV_FILE:-.env} volumes_from: - bayanat read_only: true @@ -93,7 +101,6 @@ services: dockerfile: ./flask/Dockerfile args: - ROLE=celery-ocr - - ENV_FILE=${ENV_FILE:-.env} volumes_from: - bayanat read_only: true @@ -112,9 +119,8 @@ services: restart: always build: context: ./nginx - target: prod ports: - - '80:80' + - '80:8080' volumes: - './enferno/static/:/app/static/:ro' depends_on: @@ -123,13 +129,11 @@ services: security_opt: - no-new-privileges:true tmpfs: - - /opt/bitnami/nginx/tmp/ - - /opt/bitnami/nginx/logs/ - - /opt/bitnami/nginx/conf/bitnami/certs/ + - /tmp healthcheck: - test: [ "CMD", "service", "nginx", "status" ] - interval: 3s - retries: 10 + test: [ "CMD", "wget", "-q", "--spider", "http://127.0.0.1:8080/healthz" ] + interval: 10s + retries: 5 volumes: redis_data: diff --git a/docs/deployment/auto-update-runbook.md b/docs/deployment/auto-update-runbook.md new file mode 100644 index 000000000..b4c032079 --- /dev/null +++ b/docs/deployment/auto-update-runbook.md @@ -0,0 +1,137 @@ +# Bayanat Auto-Update Runbook + +Short operator reference for the `bayanat update` flow. Design notes live +in the development spec (not shipped with the repo). + +## Triggering an update + +- **One-click from UI:** an admin-role user clicks "Update now" from the + "Update available: X.Y.Z" banner in the nav bar. +- **From the shell (as root):** `sudo bayanat update []` + (defaults to the latest GitHub release). The CLI requires root to stop + / start services, write `/opt/bayanat`, and take snapshots. +- **Check only (no changes, no root):** `sudo -u bayanat bayanat update --check`. + +The update runs as `bayanat-update.service`, a transient systemd unit +that outlives Flask restarts, SSH disconnects, and browser closes. Tail +live logs with: + +``` +sudo journalctl -u bayanat-update -f +``` + +## Opt-in auto-apply for patch releases + +In the admin UI under System Administration, toggle "Auto-apply patch +releases" on. With the toggle on, any bump within the same minor line +(e.g. `4.1.0` to `4.1.1`) installs silently every 6 hours via the same +pipeline. Minor and major bumps (e.g. `4.1.x` to `4.2.0`) always notify +and wait for a manual click. + +## Expected timing + +| Phase | Duration | Production impact | +|---|---|---| +| PREPARE (fetch + deps) | 1-5 min | None, old version serves traffic | +| Stop services | ~3 s | 502 from Caddy begins | +| Snapshot (`pg_dump -Fc`) | 10-60 s | 502 | +| Migrate (`flask db upgrade`) | 1-30 s | 502 | +| Swap + start services | ~5 s | 502 | +| Verify (health probe) | 1-10 s | New version serving | +| **Total visible downtime** | **~30-90 s** | | + +Caddy returns `502 Bad Gateway` during the maintenance window. Browsers +retry automatically; partners see a brief "service unavailable" view. + +## Release verification + +The updater downloads each release as a signed tarball and verifies it against +SJAC's pinned minisign key before installing (BAY-01-017). An unsigned or +tampered release is refused during PREPARE with `Release is unsigned` or +`Signature verification FAILED`, and nothing is installed. If you hit this on a +legitimate release, the release is missing its `.minisig` asset; see +[release-signing.md](release-signing.md). + +## If something goes wrong + +### Migration failed (Alembic transaction rolled back) + +Nothing to do. Services restart on the previous release automatically. +The UI shows the `error` field. Report the broken release; the previous +version keeps running. + +### Health probe failed after swap (auto-rollback succeeded) + +Nothing to do. The updater reverted the symlink and restarted on the +previous release. The pre-update snapshot is retained at +`/opt/bayanat/shared/backups/`. + +### NEEDS_INTERVENTION + +This state only happens when two independent failures compound: the new +release was broken AND rolling back did not reach a healthy state. The +maintenance flag stays up so users see a 502 instead of raw errors. +Recover: + +``` +sudo -u bayanat bayanat status # read-only; confirm state +sudo bayanat snapshots # list snapshots (needs root) +sudo bayanat restore pre-.dump # restores DB (needs root) +sudo systemctl start bayanat bayanat-celery +``` + +Then file a bug with journal logs from `journalctl -u bayanat-update`. + +### Stuck state (process died, state file orphaned) + +``` +sudo bayanat update --recover +``` + +## Snapshots + +- Location: `/opt/bayanat/shared/backups/pre-*.dump` +- Format: `pg_dump -Fc` (PostgreSQL custom format) +- Retention: last 5 snapshots OR last 30 days, whichever is greater +- Override retention: `export BAYANAT_SNAPSHOT_RETENTION_DAYS=60` +- List: `sudo bayanat snapshots` or visit `/admin/snapshots/` in the UI + (read-only) +- Restore: `sudo bayanat restore ` (prompts for confirmation; + stops services; pipes through `pg_restore --clean --if-exists`; + restarts services). Requires root. Not available from the web UI by + design. + +## Files + +| Path | Purpose | +|---|---| +| `/usr/local/bin/bayanat` | The CLI script | +| `/opt/bayanat/state/update.json` | Current update state (sanitized JSON) | +| `/opt/bayanat/state/update.lock` | PID lock file | +| `/opt/bayanat/shared/backups/` | Pre-update snapshots | +| `/health` (Flask endpoint) | 200 = DB + Redis reachable | + +## Admin UI surface + +The UI is read-only for updates: it surfaces availability but never applies +an update. Updates run from the CLI as root (`sudo bayanat update`). + +- Nav-bar banner chip: shows when `latest != current`, with the CLI command + to run on the server +- Status: `/admin/api/updates/status` reflects a CLI-initiated update's state +- Snapshots page: `/admin/snapshots/` (read-only list; restore stays on + the CLI) + +## Manual CLI reference + +Commands marked `(root)` require `sudo bayanat ...`; the others can run +as the app user via `sudo -u bayanat bayanat ...`. + +``` +bayanat update [] (root) default: latest GitHub release +bayanat update --check show current vs latest; no changes +bayanat update --recover (root) recover a stuck state file +bayanat snapshots (root) list pre-update snapshots +bayanat restore (root) interactive restore from a snapshot +bayanat status version + services + update state +``` diff --git a/docs/deployment/docker.md b/docs/deployment/docker.md index 7652ec481..1a118b8a5 100644 --- a/docs/deployment/docker.md +++ b/docs/deployment/docker.md @@ -6,31 +6,53 @@ Docker Compose deployment is still in beta. For production environments, [native ## Prerequisites -- Docker and Docker Compose installed -- `.env` file configured (see [Configuration](/deployment/configuration)) +- Docker Engine with the Compose v2 plugin (`docker compose`, not the legacy `docker-compose` binary) +- `.env.docker` file configured (see [Configuration](/deployment/configuration)) ## Quick Start ```bash -docker-compose up -d +docker compose --env-file .env.docker up -d ``` This starts PostgreSQL, Redis, the Flask app, NGINX, and Celery. -## Create Admin User +::: tip +The `--env-file .env.docker` flag is required so Compose can substitute `${POSTGRES_USER}`, `${POSTGRES_PASSWORD}`, and `${REDIS_PASSWORD}` placeholders in `docker-compose.yml`. Without it, those services boot with empty credentials and the Flask container fails to connect. +::: + +## First Admin User + +The entrypoint creates an `admin` user automatically on the first startup +(when the database has no schema yet) and prints a one-time random +password to the container logs. Retrieve it with: ```bash -docker-compose exec bayanat uv run flask install +docker compose --env-file .env.docker logs bayanat | grep -A4 "Generated password" ``` +Sign in at the Bayanat URL with `admin` and the printed password. The +setup wizard runs after first login. Change the admin password from your +account settings afterwards. + +If the auto-bootstrap was missed or the admin account was deleted, run +the CLI directly: + +```bash +docker compose --env-file .env.docker exec bayanat uv run flask install -u admin +``` + +It generates a fresh password and prints it. If an admin already exists +the command exits without changing anything. + ## Development ```bash -docker-compose -f docker-compose-dev.yml up +docker compose -f docker-compose-dev.yml up ``` ## Testing ```bash -docker-compose -f docker-compose-test.yml up +docker compose -f docker-compose-test.yml up ``` diff --git a/docs/deployment/installation.md b/docs/deployment/installation.md index 49d20529d..ca918fce0 100644 --- a/docs/deployment/installation.md +++ b/docs/deployment/installation.md @@ -34,7 +34,7 @@ This will: - Set up systemd services for Bayanat and Celery - Start everything -Once complete, open your domain in a browser. The setup wizard will guide you through creating an admin account and configuring the application. +Once complete, the installer prints the initial `admin` username and a one-time generated password to the terminal. Open your domain in a browser, sign in with those credentials, then the setup wizard will guide you through configuring the application. Change the admin password from your account settings after first login. **Check status:** @@ -150,7 +150,7 @@ uv run flask install uv run flask run ``` -Access at [http://127.0.0.1:5000](http://127.0.0.1:5000). The setup wizard will guide further configuration. +Access at [http://127.0.0.1:5000](http://127.0.0.1:5000). Sign in with the credentials printed by `flask install`, then the setup wizard will guide further configuration. ::: warning `flask run` is development mode only. Continue with the steps below for production. @@ -243,14 +243,19 @@ sudo systemctl enable --now bayanat-celery Docker deployment is still in beta. For production, native deployment is recommended. ::: -After [configuring](/deployment/configuration) and generating a `.env` file: +After [configuring](/deployment/configuration) and generating a `.env.docker` file: ```bash -docker-compose up -d +docker compose --env-file .env.docker up -d ``` -Install the admin user: +The first startup creates an `admin` user and prints a generated +password to the container logs. Retrieve it with: ```bash -docker-compose exec bayanat uv run flask install +docker compose --env-file .env.docker logs bayanat | grep -A4 "Generated password" ``` + +If the auto-bootstrap was missed or the admin was deleted, run +`docker compose --env-file .env.docker exec bayanat uv run flask install -u admin` +to mint a fresh credential. diff --git a/docs/deployment/release-signing.md b/docs/deployment/release-signing.md new file mode 100644 index 000000000..0fb8854f5 --- /dev/null +++ b/docs/deployment/release-signing.md @@ -0,0 +1,80 @@ +# Release Signing + +Bayanat releases are signed with [minisign](https://jedisct1.github.io/minisign/). +The `bayanat` CLI verifies every release tarball against a pinned public key +before installing it, so `sudo bayanat update` (and the installer) will refuse +an unsigned or tampered release. This is the BAY-01-017 control. + +## What the updater expects + +For each GitHub release tagged `` (e.g. `v4.1.0`), two assets must be +attached: + +| Asset | What it is | +|---|---| +| `bayanat-.tar.gz` | the release source tree | +| `bayanat-.tar.gz.minisig` | its minisign signature | + +The updater downloads both, runs `minisign -V` against the pinned key, and only +extracts the tarball if the signature verifies. A missing `.minisig` is treated +as unsigned and refused. + +## The pinned key + +The verifying public key is baked into the `bayanat` script as `RELEASE_PUBKEY` +(root-owned, not swappable at update time): + +``` +RWS7XvDVF0InHWTCh/86K8sXGcHU/PmzCl4uH9GUDjNnNzHhcX1BvGqZ +``` + +key ID `1D274217D5F05EBB`. + +The matching **secret key is held offline** (never in CI, never in the repo). +It signs releases on a maintainer's machine. CI and GitHub only ever carry the +already-made `.minisig`. + +## Signing a release + +On the machine that holds the secret key, from a clean checkout at the tag: + +```bash +TAG=v4.1.0 + +# 1. Build the exact tree the tag points at. +git archive --format=tar.gz --prefix="bayanat-$TAG/" -o "bayanat-$TAG.tar.gz" "$TAG" + +# 2. Sign it (prompts for the key password). +minisign -Sm "bayanat-$TAG.tar.gz" + +# 3. Verify locally against the pinned public key before publishing. +minisign -Vm "bayanat-$TAG.tar.gz" -P RWS7XvDVF0InHWTCh/86K8sXGcHU/PmzCl4uH9GUDjNnNzHhcX1BvGqZ +``` + +Then attach `bayanat-$TAG.tar.gz` and `bayanat-$TAG.tar.gz.minisig` to the +GitHub release for `$TAG`. + +`git archive` is deterministic for a given tree, and the signature covers the +exact bytes you upload, so there is no cross-machine reproducibility +requirement: the updater verifies the same file you signed. + +## Key custody + +- Working copy: `~/.config/minisign/bayanat-release.key` (chmod 600). +- Password: stored in a password manager. minisign cannot regenerate the key + from the password alone, so the key file **and** the password must both + survive. Keep an encrypted backup. +- More than one maintainer should hold the key file and password (bus factor). + +## Rotation + +minisign has no revocation. If the key is lost or compromised, generate a new +keypair, update `RELEASE_PUBKEY` in the `bayanat` script, and ship the new +pinned key in a fresh install or a documented manual swap. Until a host runs a +`bayanat` build carrying the new key, it will keep trusting the old one, so a +rotation reaches existing hosts only through an update they install with the +old key still trusted, or through a manual key swap on the host. + +The manual update path always remains available, so a lost key never bricks an +install: an operator can still update the host by hand per +[upgrading.md](upgrading.md). diff --git a/e2e-auto-update.sh b/e2e-auto-update.sh new file mode 100755 index 000000000..8f9b3a5a0 --- /dev/null +++ b/e2e-auto-update.sh @@ -0,0 +1,229 @@ +#!/usr/bin/env bash +# +# End-to-end test for the `bayanat update` pipeline on a disposable +# Hetzner VM. Provisions → installs → runs S1-S4 → teardown. +# +# Requires: +# - hcloud CLI authenticated to the right project (see `hcloud context list`) +# - ssh-agent loaded with the private key matching the registered hcloud key +# - a public test fork with tags v4.0.0 (baseline), v4.0.1 (additive), +# v4.0.2 (bad migration), v4.0.3 (/health 503 at runtime), v4.0.4 (recovery) +# +# Usage: +# ./e2e-auto-update.sh # full run: provision → S1-S4 → destroy +# KEEP_VM=1 ./e2e-auto-update.sh # leave VM running at end +# VM_IP=1.2.3.4 ./e2e-auto-update.sh # reuse an existing VM (skip provision) +# SCENARIOS="S1 S2" ./e2e-auto-update.sh # run a subset +# TEST_FORK=you/yourfork ./e2e-auto-update.sh +# +set -euo pipefail + +# --- Config --- +TEST_FORK="${TEST_FORK:-level09/bayanat-update-test}" +SSH_KEY="${SSH_KEY:-level09@Black09}" +SERVER_TYPE="${SERVER_TYPE:-cpx22}" +LOCATION="${LOCATION:-nbg1}" +SCENARIOS="${SCENARIOS:-S1 S2 S3 S4}" +KEEP_VM="${KEEP_VM:-0}" +VM_IP="${VM_IP:-}" +SERVER_NAME="" + +SSHOPTS="-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5" + +log() { printf '\n\033[1;34m[%s] %s\033[0m\n' "$(date +%H:%M:%S)" "$*"; } +pass() { printf '\033[1;32m ✓ %s\033[0m\n' "$*"; } +fail() { printf '\033[1;31m ✗ %s\033[0m\n' "$*" >&2; exit 1; } + +on_vm() { ssh $SSHOPTS "root@$VM_IP" "$@"; } + +# --- Prereqs --- +command -v hcloud >/dev/null || { echo "hcloud CLI not found"; exit 2; } +command -v gh >/dev/null || { echo "gh CLI not found (needed to rewrite tags)"; exit 2; } +git ls-remote --tags "https://github.com/$TEST_FORK.git" >/dev/null \ + || { echo "test fork $TEST_FORK not reachable"; exit 2; } + +# --- Tag ladder prep: hide upper tags so installer picks v4.0.0 --- +stash_upper_tags() { + log "Hiding upper tags on $TEST_FORK so installer picks v4.0.0" + for t in v4.0.1 v4.0.2 v4.0.3 v4.0.4; do + gh api -X DELETE "repos/$TEST_FORK/git/refs/tags/$t" 2>&1 \ + | grep -v '^$' | head -1 || true + done +} + +restore_upper_tags() { + log "Restoring upper tags v4.0.1..v4.0.4" + # Must push via a configured remote (SSH auth), not an HTTPS URL. + for t in v4.0.1 v4.0.2 v4.0.3 v4.0.4; do + if ! git show-ref --tags --verify --quiet "refs/tags/$t"; then + echo " LOCAL TAG MISSING: $t (run the rebase block in the README)"; continue + fi + git push test-fork "+refs/tags/$t:refs/tags/$t" 2>&1 | tail -1 + done + # Sanity: confirm remote has them + local remote_tags + remote_tags=$(git ls-remote --tags test-fork | awk '{print $2}' | grep -E 'v4\.0\.[1-4]$' | wc -l | tr -d ' ') + [[ "$remote_tags" == "4" ]] || fail "expected 4 upper tags on remote, found $remote_tags" + pass "4 upper tags visible on remote" +} + +# --- Provision --- +provision() { + SERVER_NAME="bayanat-update-test-$(date +%Y%m%d-%H%M%S)" + log "Provisioning Hetzner VM $SERVER_NAME ($SERVER_TYPE in $LOCATION)" + VM_IP=$(hcloud server create \ + --name "$SERVER_NAME" \ + --type "$SERVER_TYPE" \ + --image ubuntu-24.04 \ + --ssh-key "$SSH_KEY" \ + --location "$LOCATION" \ + -o json | python3 -c 'import json,sys; print(json.load(sys.stdin)["server"]["public_net"]["ipv4"]["ip"])') + log "IP: $VM_IP" + log "Waiting for SSH..." + until on_vm 'true' 2>/dev/null; do sleep 3; done + pass "SSH ready" +} + +teardown() { + if [[ "$KEEP_VM" == "1" ]]; then + log "KEEP_VM=1 — leaving $SERVER_NAME (IP $VM_IP) alive" + return + fi + if [[ -n "$SERVER_NAME" ]]; then + log "Destroying $SERVER_NAME" + hcloud server delete "$SERVER_NAME" >/dev/null + pass "destroyed" + fi +} + +# --- Install --- +install_baseline() { + log "Installing v4.0.0 via curl | sudo bash -s install (validates \$0-free install)" + on_vm 'echo "BAYANAT_REPO='"$TEST_FORK"'" >> /etc/environment' + on_vm 'curl -fsSL https://raw.githubusercontent.com/'"$TEST_FORK"'/v4.0.0/bayanat | BAYANAT_REPO='"$TEST_FORK"' sudo -E bash -s install localhost' \ + >/tmp/e2e-install.log 2>&1 \ + || { tail -30 /tmp/e2e-install.log; fail "install failed"; } + pass "install succeeded" + + # Work around the SETUP_COMPLETE gating until that lands in installer + on_vm 'echo "BAYANAT_CONFIG_FILE=/opt/bayanat/shared/config.json" >> /opt/bayanat/shared/.env + echo "{\"SETUP_COMPLETE\": true}" > /opt/bayanat/shared/config.json + chown bayanat:bayanat /opt/bayanat/shared/config.json + systemctl restart bayanat bayanat-celery' + sleep 3 + + local health + health=$(on_vm 'curl -s --unix-socket /opt/bayanat/current/bayanat.sock http://localhost/health') + [[ "$health" == *'"status":"ok"'* ]] || fail "/health not ok: $health" + pass "/health OK: $health" + + local cur + cur=$(on_vm 'bayanat status | grep "Current version" | awk "{print \$3}"') + [[ "$cur" == "v4.0.0" ]] || fail "expected v4.0.0, got $cur" + pass "installed version: $cur" +} + +# --- Scenario helpers --- +assert_version() { + local expected="$1" + local actual + actual=$(on_vm 'bayanat status | grep "Current version" | awk "{print \$3}"') + [[ "$actual" == "$expected" ]] || fail "expected version $expected, got $actual" + pass "version = $expected" +} + +assert_state() { + local expected="$1" + local actual + actual=$(on_vm 'bayanat status | grep "Update state" | awk "{print \$3}"') + [[ "$actual" == "$expected" ]] || fail "expected state $expected, got $actual" + pass "update state = $expected" +} + +assert_state_file_phase() { + local expected="$1" + local phase + phase=$(on_vm 'python3 -c "import json; print(json.load(open(\"/opt/bayanat/state/update.json\")).get(\"phase\",\"\"))"' 2>/dev/null || echo "") + [[ "$phase" == "$expected" ]] || fail "expected state file phase $expected, got '$phase'" + pass "state file phase = $expected" +} + +assert_services_active() { + on_vm 'systemctl is-active --quiet bayanat bayanat-celery caddy' \ + || fail "services not all active" + pass "services all active" +} + +clear_state_file() { + on_vm 'rm -f /opt/bayanat/state/update.json /opt/bayanat/state/update.lock' +} + +run_update() { + local tag="$1" + log " -> bayanat update $tag" + on_vm 'sudo BAYANAT_REPO='"$TEST_FORK"' /usr/local/bin/bayanat update '"$tag" \ + >/tmp/e2e-update.log 2>&1 || true # we inspect state, exit code is scenario-dependent + tail -5 /tmp/e2e-update.log | sed 's/^/ /' +} + +# --- Scenarios --- +S1() { + log "S1: happy path v4.0.0 -> v4.0.1" + run_update v4.0.1 + assert_version v4.0.1 + assert_state IDLE + assert_services_active + on_vm 'sudo -u bayanat psql -d bayanat -c "\d bulletin" | grep -q auto_update_test' \ + || fail "auto_update_test column missing" + pass "auto_update_test column present" +} + +S2() { + log "S2: bad migration v4.0.1 -> v4.0.2 -> NEEDS_INTERVENTION" + run_update v4.0.2 + assert_version v4.0.1 + assert_state_file_phase NEEDS_INTERVENTION + assert_services_active + clear_state_file +} + +S3() { + log "S3: bad /health v4.0.1 -> v4.0.3 -> ROLLED_BACK" + run_update v4.0.3 + assert_version v4.0.1 + assert_state IDLE + assert_services_active + local health + health=$(on_vm 'curl -s --unix-socket /opt/bayanat/current/bayanat.sock http://localhost/health') + [[ "$health" == *'"status":"ok"'* ]] || fail "/health not ok after rollback" + pass "/health back to OK after rollback" +} + +S4() { + log "S4: recovery v4.0.1 -> v4.0.4" + run_update v4.0.4 + assert_version v4.0.4 + assert_state IDLE + assert_services_active + on_vm 'sudo -u bayanat psql -d bayanat -c "\d bulletin" | grep -q auto_update_recovery_test' \ + || fail "auto_update_recovery_test column missing" + pass "auto_update_recovery_test column present" +} + +# --- Main --- +trap 'restore_upper_tags; teardown' EXIT + +if [[ -z "$VM_IP" ]]; then + stash_upper_tags + provision + install_baseline + restore_upper_tags +else + log "Reusing existing VM at $VM_IP" +fi + +for s in $SCENARIOS; do + "$s" +done + +log "ALL PASSED" diff --git a/enferno/admin/models/Actor.py b/enferno/admin/models/Actor.py index b0f1ef9a9..14380ccf0 100644 --- a/enferno/admin/models/Actor.py +++ b/enferno/admin/models/Actor.py @@ -28,7 +28,7 @@ ) from enferno.admin.models.Country import Country from enferno.admin.models.Ethnography import Ethnography -from enferno.admin.models.utils import check_roles +from enferno.admin.models.utils import check_roles, can_view_media from enferno.extensions import db from enferno.utils.base import BaseMixin from enferno.utils.csv_utils import convert_simple_relation, convert_complex_relation @@ -452,75 +452,36 @@ def from_json(self, json: dict[str, Any]) -> "Actor": # Related Actors (actor_relations) if "actor_relations" in json: - # collect related actors ids (helps with finding removed ones) - rel_ids = [] - for relation in json["actor_relations"]: - actor = db.session.get(Actor, relation["actor"]["id"]) - - # Extra (check those actors exit) - - if actor: - rel_ids.append(actor.id) - # this will update/create the relationship (will flush to db!) - self.relate_actor(actor, relation=relation) - - # Find out removed relations and remove them - # just loop existing relations and remove if the destination actor not in the related ids - - for r in self.actor_relations: - # get related actor (in or out) - rid = r.get_other_id(self.id) - if not (rid in rel_ids): - r.delete() - - # -revision related - db.session.get(Actor, rid).create_revision() + self.sync_relations( + json["actor_relations"], + Actor, + "actor", + self.relate_actor, + self.actor_relations, + lambda r: r.get_other_id(self.id), + ) # Related Bulletins (bulletin_relations) if "bulletin_relations" in json: - # collect related bulletin ids (helps with finding removed ones) - rel_ids = [] - for relation in json["bulletin_relations"]: - bulletin = db.session.get(Bulletin, relation["bulletin"]["id"]) - - # Extra (check those bulletins exit) - if bulletin: - rel_ids.append(bulletin.id) - # this will update/create the relationship (will flush to db!) - self.relate_bulletin(bulletin, relation=relation) - - # Find out removed relations and remove them - # just loop existing relations and remove if the destination bulletin not in the related ids - for r in self.bulletin_relations: - if not (r.bulletin_id in rel_ids): - rel_bulletin = r.bulletin - r.delete() - - # -revision related - rel_bulletin.create_revision() - - # Related Incidents (incidents_relations) + self.sync_relations( + json["bulletin_relations"], + Bulletin, + "bulletin", + self.relate_bulletin, + self.bulletin_relations, + lambda r: r.bulletin_id, + ) + + # Related Incidents (incident_relations) if "incident_relations" in json: - # collect related incident ids (helps with finding removed ones) - rel_ids = [] - for relation in json["incident_relations"]: - incident = db.session.get(Incident, relation["incident"]["id"]) - if incident: - rel_ids.append(incident.id) - # helper method to update/create the relationship (will flush to db) - self.relate_incident(incident, relation=relation) - - # Find out removed relations and remove them - # just loop existing relations and remove if the destination incident no in the related ids - - for r in self.incident_relations: - # get related bulletin (in or out) - if not (r.incident_id in rel_ids): - rel_incident = r.incident - r.delete() - - # -revision related incident - rel_incident.create_revision() + self.sync_relations( + json["incident_relations"], + Incident, + "incident", + self.relate_incident, + self.incident_relations, + lambda r: r.incident_id, + ) if "comments" in json: self.comments = json["comments"] @@ -763,9 +724,9 @@ def to_dict(self, mode: Optional[str] = None) -> dict[str, Any]: for event in self.events: events_json.append(event.to_dict()) - # medias json + # medias json (hidden from users without media access, BAY-01-012) medias_json = [] - if self.medias and len(self.medias): + if can_view_media() and self.medias and len(self.medias): for media in self.medias: medias_json.append(media.to_dict()) diff --git a/enferno/admin/models/Bulletin.py b/enferno/admin/models/Bulletin.py index 3a48fa3d8..296fe0334 100644 --- a/enferno/admin/models/Bulletin.py +++ b/enferno/admin/models/Bulletin.py @@ -31,7 +31,7 @@ bulletin_verlabels, bulletin_events, ) -from enferno.admin.models.utils import check_roles +from enferno.admin.models.utils import check_roles, can_view_media logger = get_logger() @@ -415,74 +415,36 @@ def from_json(self, json: dict[str, Any]) -> "Bulletin": # Related Bulletins (bulletin_relations) if "bulletin_relations" in json: - # collect related bulletin ids (helps with finding removed ones) - rel_ids = [] - for relation in json["bulletin_relations"]: - bulletin = db.session.get(Bulletin, relation["bulletin"]["id"]) - # Extra (check those bulletins exit) - - if bulletin: - rel_ids.append(bulletin.id) - # this will update/create the relationship (will flush to db) - self.relate_bulletin(bulletin, relation=relation) - - # Find out removed relations and remove them - # just loop existing relations and remove if the destination bulletin no in the related ids - - for r in self.bulletin_relations: - # get related bulletin (in or out) - rid = r.get_other_id(self.id) - if not (rid in rel_ids): - r.delete() - - # ------- create revision on the other side of the relationship - db.session.get(Bulletin, rid).create_revision() - - # Related Actors (actors_relations) + self.sync_relations( + json["bulletin_relations"], + Bulletin, + "bulletin", + self.relate_bulletin, + self.bulletin_relations, + lambda r: r.get_other_id(self.id), + ) + + # Related Actors (actor_relations) if "actor_relations" in json: - # collect related bulletin ids (helps with finding removed ones) - rel_ids = [] - for relation in json["actor_relations"]: - actor = db.session.get(Actor, relation["actor"]["id"]) - if actor: - rel_ids.append(actor.id) - # helper method to update/create the relationship (will flush to db) - self.relate_actor(actor, relation=relation) - - # Find out removed relations and remove them - # just loop existing relations and remove if the destination actor no in the related ids - - for r in self.actor_relations: - # get related bulletin (in or out) - if not (r.actor_id in rel_ids): - rel_actor = r.actor - r.delete() - - # --revision relation - rel_actor.create_revision() - - # Related Incidents (incidents_relations) + self.sync_relations( + json["actor_relations"], + Actor, + "actor", + self.relate_actor, + self.actor_relations, + lambda r: r.actor_id, + ) + + # Related Incidents (incident_relations) if "incident_relations" in json: - # collect related incident ids (helps with finding removed ones) - rel_ids = [] - for relation in json["incident_relations"]: - incident = db.session.get(Incident, relation["incident"]["id"]) - if incident: - rel_ids.append(incident.id) - # helper method to update/create the relationship (will flush to db) - self.relate_incident(incident, relation=relation) - - # Find out removed relations and remove them - # just loop existing relations and remove if the destination incident no in the related ids - - for r in self.incident_relations: - # get related bulletin (in or out) - if not (r.incident_id in rel_ids): - rel_incident = r.incident - r.delete() - - # --revision relation - rel_incident.create_revision() + self.sync_relations( + json["incident_relations"], + Incident, + "incident", + self.relate_incident, + self.incident_relations, + lambda r: r.incident_id, + ) self.publish_date = json.get("publish_date", None) if self.publish_date == "": @@ -755,9 +717,9 @@ def to_dict(self, mode: Optional[str] = None) -> dict[str, Any]: for event in self.events: events_json.append(event.to_dict()) - # medias json + # medias json (hidden from users without media access, BAY-01-012) medias_json = [] - if self.medias and len(self.medias): + if can_view_media() and self.medias and len(self.medias): for media in self.medias: medias_json.append(media.to_dict()) diff --git a/enferno/admin/models/Incident.py b/enferno/admin/models/Incident.py index 09bb06fd6..aeba2732e 100644 --- a/enferno/admin/models/Incident.py +++ b/enferno/admin/models/Incident.py @@ -283,76 +283,36 @@ def from_json(self, json: dict[str, Any]) -> "Incident": # Related Actors (actor_relations) if "actor_relations" in json and "check_ar" in json: - # collect related actors ids (helps with finding removed ones) - rel_ids = [] - for relation in json["actor_relations"]: - actor = db.session.get(Actor, relation["actor"]["id"]) - - # Extra (check those actors exit) - - if actor: - rel_ids.append(actor.id) - # this will update/create the relationship (will flush to db!) - self.relate_actor(actor, relation=relation) - - # Find out removed relations and remove them - # just loop existing relations and remove if the destination actor not in the related ids - - for r in self.actor_relations: - if not (r.actor_id in rel_ids): - rel_actor = r.actor - r.delete() - - # -revision related actor - rel_actor.create_revision() + self.sync_relations( + json["actor_relations"], + Actor, + "actor", + self.relate_actor, + self.actor_relations, + lambda r: r.actor_id, + ) # Related Bulletins (bulletin_relations) if "bulletin_relations" in json and "check_br" in json: - # collect related bulletin ids (helps with finding removed ones) - rel_ids = [] - for relation in json["bulletin_relations"]: - bulletin = db.session.get(Bulletin, relation["bulletin"]["id"]) - - # Extra (check those bulletins exit) - if bulletin: - rel_ids.append(bulletin.id) - # this will update/create the relationship (will flush to db!) - self.relate_bulletin(bulletin, relation=relation) - - # Find out removed relations and remove them - # just loop existing relations and remove if the destination bulletin not in the related ids - for r in self.bulletin_relations: - if not (r.bulletin_id in rel_ids): - rel_bulletin = r.bulletin - r.delete() - - # -revision related bulletin - rel_bulletin.create_revision() - - # Related Incidnets (incident_relations) + self.sync_relations( + json["bulletin_relations"], + Bulletin, + "bulletin", + self.relate_bulletin, + self.bulletin_relations, + lambda r: r.bulletin_id, + ) + + # Related Incidents (incident_relations) if "incident_relations" in json and "check_ir" in json: - # collect related incident ids (helps with finding removed ones) - rel_ids = [] - for relation in json["incident_relations"]: - incident = db.session.get(Incident, relation["incident"]["id"]) - # Extra (check those incidents exit) - - if incident: - rel_ids.append(incident.id) - # this will update/create the relationship (will flush to db) - self.relate_incident(incident, relation=relation) - - # Find out removed relations and remove them - # just loop existing relations and remove if the destination incident no in the related ids - - for r in self.incident_relations: - # get related incident (in or out) - rid = r.get_other_id(self.id) - if not (rid in rel_ids): - r.delete() - - # - revision related incident - db.session.get(Incident, rid).create_revision() + self.sync_relations( + json["incident_relations"], + Incident, + "incident", + self.relate_incident, + self.incident_relations, + lambda r: r.get_other_id(self.id), + ) if "comments" in json: self.comments = json["comments"] diff --git a/enferno/admin/models/Media.py b/enferno/admin/models/Media.py index 61bb3723c..1c6011581 100644 --- a/enferno/admin/models/Media.py +++ b/enferno/admin/models/Media.py @@ -1,5 +1,6 @@ import json import pathlib +import secrets from pathlib import Path from typing import Any from unidecode import unidecode @@ -147,6 +148,20 @@ def generate_file_name(filename: str) -> str: decoded = secure_filename(unidecode(filename)).lower() return f"{DateHelper.utcnow().strftime('%Y%m%d-%H%M%S')}-{decoded}" + @staticmethod + def generate_inline_file_name(filename: str) -> str: + """Opaque, unguessable name for inline rich-text uploads (BAY-01-020). + + Inline media is served on a session-only route with no per-item access + check, so the old timestamp+basename name let any authenticated user + reconstruct a filename and fetch media for items they can't access. A + random token makes the URL a capability only held by viewers of the + (access-controlled) description that embeds it. + """ + decoded = secure_filename(unidecode(filename)).lower().rsplit(".", 1) + suffix = f".{decoded[1]}" if len(decoded) == 2 and decoded[1] else "" + return f"{secrets.token_urlsafe(24)}{suffix}" + @staticmethod def validate_file_extension(filepath: str, allowed_extensions: list[str]) -> bool: """ diff --git a/enferno/admin/models/utils.py b/enferno/admin/models/utils.py index a79857eb5..947841c5c 100644 --- a/enferno/admin/models/utils.py +++ b/enferno/admin/models/utils.py @@ -5,6 +5,20 @@ # Role based Access Control Decorator for Bulletins / Actors / Incidents # +def can_view_media(): + """Whether the current actor may see media metadata in entity payloads. + + CLI/Celery (no request context) are trusted. Otherwise mirror the + _require_media_access gate on the direct media endpoints: Admins and users + with can_access_media only. Without this, a user blocked from the media + endpoints could still read filenames, etags and OCR text embedded in a + parent bulletin/actor payload (BAY-01-012). + """ + if not has_request_context(): + return True + return current_user.has_role("Admin") or current_user.can_access_media + + def check_roles(method): """ Decorator to check if the current user has access to the resource. If the diff --git a/enferno/admin/templates/admin/snapshots.html b/enferno/admin/templates/admin/snapshots.html new file mode 100644 index 000000000..71d82847f --- /dev/null +++ b/enferno/admin/templates/admin/snapshots.html @@ -0,0 +1,28 @@ +{% extends 'layout.html' %} {% block content %} + + + + + + + +{% endblock %} {% block js %} + + +{% endblock %} diff --git a/enferno/admin/templates/nav-bar.html b/enferno/admin/templates/nav-bar.html index 3a724aae3..a6be982fd 100644 --- a/enferno/admin/templates/nav-bar.html +++ b/enferno/admin/templates/nav-bar.html @@ -11,6 +11,8 @@