From 766811d4983769ba9f3f10b229bd902b4d84b4f6 Mon Sep 17 00:00:00 2001 From: Eduardo Borjas Date: Sun, 14 Jun 2026 12:14:21 -0600 Subject: [PATCH 1/2] feat(backup-system): autonomous, declarative 3-2-1 backup framework MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-hosting backup engine: one ckis-manifest.json classifies every path (track/regenerable/secret/sensitive/snapshot); three rings of autonomy (session-end push, daily systemd safety net, passive health line); plug-and-play restore on a fresh machine/OS. Zero heavy deps (bash/git/jq/ rsync/flock). Pure-bash test suite (94 assertions). Generic example manifest + README so anyone can implement their own. Adds a scoped CI workflow and a README section. ๐Ÿค– Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.8 --- .github/workflows/backup-system-tests.yml | 18 +++ README.md | 14 ++ backup-system/.gitignore | 3 + backup-system/README.md | 95 ++++++++++++++ backup-system/bin/ckis-apparatus-export.sh | 50 +++++++ backup-system/bin/ckis-backup-all.sh | 75 +++++++++++ backup-system/bin/ckis-backup-doctor.sh | 66 ++++++++++ backup-system/bin/ckis-backup-physical.sh | 83 ++++++++++++ backup-system/bin/ckis-push.sh | 70 ++++++++++ backup-system/bin/ckis-restore.sh | 58 +++++++++ backup-system/bin/cli-brains-sync.sh | 66 ++++++++++ backup-system/ckis-manifest.example.json | 46 +++++++ backup-system/hooks/pre-commit-secret-scan.sh | 39 ++++++ backup-system/install.sh | 44 +++++++ backup-system/lib/common.sh | 123 ++++++++++++++++++ backup-system/systemd/ckis-backup.service | 11 ++ backup-system/systemd/ckis-backup.timer | 12 ++ backup-system/tests/assert.sh | 41 ++++++ backup-system/tests/run.sh | 32 +++++ backup-system/tests/test_apparatus_export.sh | 48 +++++++ backup-system/tests/test_backup_all.sh | 40 ++++++ backup-system/tests/test_cli_brains_sync.sh | 51 ++++++++ backup-system/tests/test_doctor.sh | 51 ++++++++ backup-system/tests/test_install.sh | 33 +++++ backup-system/tests/test_lib.sh | 57 ++++++++ backup-system/tests/test_manifest.sh | 38 ++++++ backup-system/tests/test_physical.sh | 40 ++++++ backup-system/tests/test_push.sh | 48 +++++++ backup-system/tests/test_restore.sh | 35 +++++ backup-system/tests/test_secret_scan.sh | 28 ++++ 30 files changed, 1415 insertions(+) create mode 100644 .github/workflows/backup-system-tests.yml create mode 100644 backup-system/.gitignore create mode 100644 backup-system/README.md create mode 100755 backup-system/bin/ckis-apparatus-export.sh create mode 100755 backup-system/bin/ckis-backup-all.sh create mode 100755 backup-system/bin/ckis-backup-doctor.sh create mode 100755 backup-system/bin/ckis-backup-physical.sh create mode 100755 backup-system/bin/ckis-push.sh create mode 100755 backup-system/bin/ckis-restore.sh create mode 100755 backup-system/bin/cli-brains-sync.sh create mode 100644 backup-system/ckis-manifest.example.json create mode 100755 backup-system/hooks/pre-commit-secret-scan.sh create mode 100755 backup-system/install.sh create mode 100644 backup-system/lib/common.sh create mode 100644 backup-system/systemd/ckis-backup.service create mode 100644 backup-system/systemd/ckis-backup.timer create mode 100755 backup-system/tests/assert.sh create mode 100755 backup-system/tests/run.sh create mode 100755 backup-system/tests/test_apparatus_export.sh create mode 100755 backup-system/tests/test_backup_all.sh create mode 100755 backup-system/tests/test_cli_brains_sync.sh create mode 100755 backup-system/tests/test_doctor.sh create mode 100755 backup-system/tests/test_install.sh create mode 100755 backup-system/tests/test_lib.sh create mode 100755 backup-system/tests/test_manifest.sh create mode 100755 backup-system/tests/test_physical.sh create mode 100755 backup-system/tests/test_push.sh create mode 100755 backup-system/tests/test_restore.sh create mode 100755 backup-system/tests/test_secret_scan.sh diff --git a/.github/workflows/backup-system-tests.yml b/.github/workflows/backup-system-tests.yml new file mode 100644 index 0000000..327b6f8 --- /dev/null +++ b/.github/workflows/backup-system-tests.yml @@ -0,0 +1,18 @@ +name: backup-system tests + +on: + push: + paths: ["backup-system/**"] + pull_request: + paths: ["backup-system/**"] + +jobs: + gate: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Install deps + run: sudo apt-get update && sudo apt-get install -y jq rsync + - name: Run gate (bash -n + behavior tests) + working-directory: backup-system + run: bash tests/run.sh diff --git a/README.md b/README.md index e7c8955..ee2fbf4 100644 --- a/README.md +++ b/README.md @@ -229,6 +229,20 @@ CKIS runs 5 crons that maintain the system without manual effort: --- +## Autonomous Backup (3-2-1) + +A knowledge base with no backup is a liability. CKIS ships a **self-hosting, declarative backup system** โ€” see [`backup-system/`](backup-system/). + +- **3-2-1**: local + private GitHub remotes (off-site) + external-drive git bundles (off-device). +- **Declarative**: one `ckis-manifest.json` classifies every path (`track` / `regenerable` / `secret` / `sensitive` / `snapshot`). Adding a repo or tool is one entry. +- **Three rings of autonomy**: push on session end, a daily `systemd --user` safety net (self-heals missing remotes, aggregates, runs physical), and a one-line health status in your session banner. +- **Plug-and-play restore**: `ckis-restore.sh` rebuilds everything on a fresh machine/OS; only secrets are re-provisioned by hand (never in backup). +- **Zero heavy deps** (`bash`, `git`, `jq`, `rsync`, `flock`) and a pure-bash test suite (`bash backup-system/tests/run.sh`). + +**Start here:** [`backup-system/README.md`](backup-system/README.md). + +--- + ## From Execution Plan to Agentic OS โ€” The Evolution CKIS v1.0 (April 2026) started as a simple answer to one question: *how do you stop a second brain from becoming 822 empty files?* diff --git a/backup-system/.gitignore b/backup-system/.gitignore new file mode 100644 index 0000000..a57946a --- /dev/null +++ b/backup-system/.gitignore @@ -0,0 +1,3 @@ +ckis-manifest.json +apparatus/ +*.log diff --git a/backup-system/README.md b/backup-system/README.md new file mode 100644 index 0000000..e377b69 --- /dev/null +++ b/backup-system/README.md @@ -0,0 +1,95 @@ +# Autonomous Backup System + +A small, dependency-light, **self-hosting** backup system for a knowledge base / dotfiles / multi-repo setup. It gives you the **3-2-1 rule** (3 copies, 2 media, 1 off-site), runs itself, and is **plug-and-play** across machines and OSes. + +It was built to protect a multi-layer "second brain" (an Obsidian vault + an engineering knowledge base + per-project memory + the machine-level agent config), but the engine is generic: **you describe what to back up in one JSON manifest, and the scripts do the rest.** + +> Zero hard dependencies beyond `bash`, `git`, `jq`, `rsync`, `flock` (coreutils). No `yq`, no `bats`, no `gitleaks`, no cloud SDKs โ€” so it restores on a bare machine. + +--- + +## Why + +Most "backups" are a static cron job that rots the moment your setup changes. This one is **declarative and self-hosting**: the rules live in a versioned `ckis-manifest.json` that is itself backed up, and discovery is registry-driven, so adding a repo or a tool is **one manifest entry** โ€” never a script edit. + +## How it works โ€” three rings of autonomy + +1. **Ring 1 โ€” push on event (real time).** A git hook calls `ckis-push.sh ` at the end of a work session: stage-all โ†’ commit-if-dirty โ†’ push with retry. Detached, lock-guarded, never blocks you. 100 new files = 1 commit = 1 push. +2. **Ring 2 โ€” daily safety net.** A `systemd --user` timer runs `ckis-backup-all.sh`: reads the manifest, **auto-creates missing private remotes** (self-healing onboarding), pushes any drift, aggregates public repos' gitignored subdirs into a private repo, exports the curated config apparatus, and runs the physical backup if a drive is mounted. +3. **Ring 3 โ€” passive visibility.** `ckis-backup-doctor.sh --oneline` prints a one-line health status (`BACKUP โœ… all pushed ยท physical 2d`) you can drop into a shell prompt or a session banner โ€” so you see drift without checking. + +## Layers it backs up + +| Class | Meaning | Destination | +|---|---|---| +| `track` | source of truth (text/config/scripts) | private git remote | +| `regenerable` | derivable from tracked inputs | **excluded**, with a recorded rebuild command | +| `secret` | credentials/tokens/keys | **never** a remote โ€” encrypted physical disk only | +| `sensitive` | private now, encrypt later | private remote (flagged for a future git-crypt repo) | +| `snapshot` | costly binary | physical disk / Git LFS, not the main repo | + +## Install + +```bash +git clone https://github.com/YOUR_USER/your-infra-repo.git ~/infra +cd ~/infra +cp ckis-manifest.example.json ckis-manifest.json +$EDITOR ckis-manifest.json # set github_owner + your targets +./install.sh # symlinks bin/ -> ~/bin, systemd timer, secret-scan hooks +``` + +Wire Ring 1 into wherever a "work session" ends (a git `post-commit` hook, an editor hook, or an agent's stop hook): +```bash +~/bin/ckis-push "$HOME/path/to/repo" & +``` + +## Usage + +```bash +ckis-backup-doctor # health report +ckis-backup-all # force a full run (push drift, aggregate, physical) +ckis-backup-physical # physical backup to a mounted drive (auto-detected) +``` + +## Disaster restore (plug-and-play) + +On a brand-new machine or OS: +```bash +gh auth login # or set up git credentials +git clone https://github.com/YOUR_USER/your-infra-repo.git ~/infra +cd ~/infra && ./bin/ckis-restore.sh +``` +It clones every target to its manifest path, restores the config apparatus, and installs the runtime. **The only manual step is re-providing secrets** โ€” by design, they are never in the backup. + +## Mobile (iOS + Android) + +Every target is just a git repo, so any mobile git client works. For Obsidian vaults, the **obsidian-git** plugin (`isDesktopOnly:false`) syncs each vault on iOS and Android with a fine-grained PAT โ€” which also sidesteps Obsidian Sync's one-vault limit. + +## Design notes & hard-won lessons + +- **`$HOME` may itself be a git repo.** A directory without its own `.git` then resolves to the `$HOME` repo, and `git add -A` will try to stage your entire home directory. Every script guards with `is_repo_root` and refuses to operate on a parent repo. *(This one cost an 11-minute CPU hang to find.)* +- **FAT/exFAT can't store `:` in filenames.** On FAT drives the per-file mirror is skipped; **git bundles** (single files holding full history + all filenames) are the authoritative physical copy. +- **Heavy/third-party content is `regenerable`, not backed up** โ€” e.g. a skill that bundles a 1 GB headless browser is reinstalled, not committed. +- **Secrets never leave the machine.** A `pre-commit` secret scanner (dependency-free regex for GitHub/AWS/GCP keys + private keys) blocks accidental commits; the manifest `deny[]` and a defense-in-depth purge keep credentials out of the config export. + +## Layout + +``` +bin/ ckis-push, ckis-backup-all, ckis-backup-doctor, ckis-backup-physical, + cli-brains-sync, ckis-apparatus-export, ckis-restore +lib/ common.sh (logging, flock, retry, manifest accessors, git helpers) +hooks/ pre-commit-secret-scan.sh +systemd/ ckis-backup.{service,timer} +tests/ pure-bash test harness (run: bash tests/run.sh) โ€” 106 assertions +install.sh ยท ckis-manifest.example.json +``` + +## Tests + +```bash +bash tests/run.sh # bash -n + (optional) shellcheck + behavior tests -> GATE: โœ… GREEN +``` + +## License + +See `LICENSE` in the repository root. diff --git a/backup-system/bin/ckis-apparatus-export.sh b/backup-system/bin/ckis-apparatus-export.sh new file mode 100755 index 0000000..ec3a5d6 --- /dev/null +++ b/backup-system/bin/ckis-apparatus-export.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# ckis-apparatus-export.sh [dest-dir] +# Curated allowlist export of the L0 machine apparatus (~/.claude) into the +# infra repo's apparatus/ dir. NEVER a raw mirror; deny[] always wins so secrets +# (.credentials.json, tokens, caches, session blobs) are physically excluded. +set -uo pipefail +HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +source "$HERE/../lib/common.sh" +ckis::init + +SRC="${CKIS_APPARATUS_SRC:-$(ckis::expand "$(ckis::manifest '.apparatus.source')")}" +DEST="${1:-$CKIS_INFRA_ROOT/$(ckis::manifest '.apparatus.dest')}" +[ -d "$SRC" ] || { ckis::warn "apparatus source missing, skip: $SRC"; exit 0; } +mkdir -p "$DEST" + +# deny list -> rsync --exclude args (applied to every copy, defense in depth) +EXC=() +while IFS= read -r d; do [ -n "$d" ] && EXC+=(--exclude "$d"); done \ + < <(ckis::manifest '.apparatus.deny[]') + +copied=0 +# 1. allowlisted top-level entries +while IFS= read -r item; do + [ -n "$item" ] || continue + if [ -e "$SRC/$item" ]; then + rsync -a --delete "${EXC[@]}" "$SRC/$item" "$DEST/" 2>>"$CKIS_LOG_FILE" \ + && { ckis::info "apparatus: exported $item"; copied=$((copied+1)); } + fi +done < <(ckis::manifest '.apparatus.allow[]') + +# 2. auto-memory: projects/*/memory (re-included despite projects/ being denied) +mg="$(ckis::manifest '.apparatus.memory_glob // empty')" +if [ -n "$mg" ]; then + shopt -s nullglob + for md in "$SRC"/$mg; do + [ -d "$md" ] || continue + rel="${md#$SRC/}" + mkdir -p "$DEST/$(dirname "$rel")" + rsync -a --delete "$md/" "$DEST/$rel/" 2>>"$CKIS_LOG_FILE" \ + && { ckis::info "apparatus: exported memory $rel"; copied=$((copied+1)); } + done + shopt -u nullglob +fi + +# 3. hard safety net: never let a credential file survive in the export +find "$DEST" -name '.credentials.json' -o -name '*.pem' -o -name 'id_rsa*' 2>/dev/null \ + | while IFS= read -r leak; do rm -f "$leak"; ckis::warn "apparatus: purged stray secret $leak"; done + +ckis::info "apparatus export: $copied item(s) -> $DEST" +exit 0 diff --git a/backup-system/bin/ckis-backup-all.sh b/backup-system/bin/ckis-backup-all.sh new file mode 100755 index 0000000..62fa616 --- /dev/null +++ b/backup-system/bin/ckis-backup-all.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +# ckis-backup-all.sh โ€” Ring 2: the daily safety-net orchestrator. +# Registry+manifest driven, self-healing, degrades gracefully when a piece is +# absent. Catches drift from interrupted sessions, edits outside Claude, hook +# failures and brand-new projects. Idempotent; global lock. +set -uo pipefail +HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +source "$HERE/../lib/common.sh" +ckis::init + +_have_gh() { command -v gh >/dev/null 2>&1 && gh auth status >/dev/null 2>&1; } + +_run() { + ckis::info "โ•โ•โ• backup-all start โ•โ•โ•" + + # 1. Export L0 apparatus into the infra repo (so the next push captures it). + local app_src; app_src="$(ckis::manifest '.apparatus.source // empty')" + if [ -n "$app_src" ] && [ -d "$(ckis::expand "$app_src")" ]; then + bash "$HERE/ckis-apparatus-export.sh" || ckis::warn "apparatus export had issues" + fi + + # 2. Every target: self-heal missing remote, then push drift. + while IFS=$'\t' read -r slug path remote class kind; do + [ -d "$path" ] || { ckis::warn "target missing on disk: $slug ($path)"; continue; } + ckis::is_repo "$path" || { ckis::warn "target not a git repo: $slug"; continue; } + if ! ckis::repo_has_remote "$path" && [ -n "$remote" ] && [ "$remote" != "null" ]; then + if _have_gh; then + ckis::info "$slug: no remote โ€” auto-creating private $remote" + gh repo create "$remote" --private --source="$path" --remote=origin >>"$CKIS_LOG_FILE" 2>&1 \ + || ckis::err "$slug: auto-create failed" + else + ckis::warn "$slug: no remote and gh unavailable โ€” committing locally only" + fi + fi + bash "$HERE/ckis-push.sh" "$path" || ckis::warn "$slug: push reported an error" + done < <(ckis::targets) + + # 3. Aggregate public .brain/ dirs, then push the aggregator repo. + local reg; reg="$(ckis::expand "$(ckis::manifest '.discovery.registry // empty')")" + if [ -n "$reg" ] && [ -f "$reg" ]; then + bash "$HERE/cli-brains-sync.sh" || ckis::warn "aggregator sync had issues" + local agg_dir agg_remote + agg_dir="$(ckis::expand "$(ckis::manifest '.discovery.aggregate_workdir // empty')")" + agg_remote="$(ckis::manifest '.discovery.public_brain_aggregator // empty')" + if [ -n "$agg_dir" ] && [ -d "$agg_dir" ]; then + # Ensure the aggregator has its OWN .git (a dir without one would resolve + # to a parent repo such as $HOME and add the whole home tree). + if ! ckis::is_repo_root "$agg_dir"; then + git -C "$agg_dir" init -q + git -C "$agg_dir" symbolic-ref HEAD refs/heads/master 2>/dev/null || true + ckis::info "aggregator: initialized own git repo at $agg_dir" + fi + if ! ckis::repo_has_remote "$agg_dir" && [ -n "$agg_remote" ] && _have_gh; then + git -C "$agg_dir" remote add origin "https://github.com/$agg_remote.git" 2>/dev/null || true + fi + bash "$HERE/ckis-push.sh" "$agg_dir" || ckis::warn "aggregator push had issues" + fi + fi + + # 4. Physical backup if a drive is mounted (best-effort). + if bash "$HERE/ckis-backup-physical.sh" >>"$CKIS_LOG_FILE" 2>&1; then + ckis::info "physical backup ran" + else + ckis::info "physical backup skipped (no drive mounted)" + fi + + # 5. Health summary. + ckis::info "health: $(bash "$HERE/ckis-backup-doctor.sh" --oneline)" + ckis::info "โ•โ•โ• backup-all done โ•โ•โ•" +} + +ckis::with_lock "backup-all" _run +rc=$? +[ "$rc" = "75" ] && { ckis::info "backup-all already running, skipped"; exit 0; } +exit "$rc" diff --git a/backup-system/bin/ckis-backup-doctor.sh b/backup-system/bin/ckis-backup-doctor.sh new file mode 100755 index 0000000..648561c --- /dev/null +++ b/backup-system/bin/ckis-backup-doctor.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# ckis-backup-doctor.sh [--oneline] +# Passive health reporter for the backup system. Always exits 0. +# default multi-line report per target +# --oneline single status line for the SessionStart banner +set -uo pipefail +HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +source "$HERE/../lib/common.sh" + +ONELINE=0; [ "${1:-}" = "--oneline" ] && ONELINE=1 + +# Reason string for a target's drift, empty if fully backed up. +_drift() { + local dir="$1" + ckis::is_repo "$dir" || { printf 'missing'; return; } + local r=() + ckis::repo_dirty "$dir" && r+=("uncommitted") + if git -C "$dir" rev-parse --abbrev-ref --symbolic-full-name '@{u}' >/dev/null 2>&1; then + local a; a="$(ckis::repo_ahead "$dir")" + [ "${a:-0}" != "0" ] && r+=("+$a unpushed") + else + # no upstream: if there are commits, they are unbacked + git -C "$dir" rev-parse HEAD >/dev/null 2>&1 && r+=("no-upstream") + fi + ckis::repo_has_remote "$dir" || r+=("no-remote") + local IFS=','; printf '%s' "${r[*]}" +} + +# physical-backup age line +_phys() { + local m="$CKIS_LOG_DIR/last-physical" + if [ -f "$m" ]; then + local t now d + t="$(cat "$m" 2>/dev/null)"; now="$(date +%s)" + case "$t" in ''|*[!0-9]*) t="$now" ;; esac # guard against empty/corrupt marker + d=$(( (now - t) / 86400 )) + printf 'physical %sd' "$d" + else + printf 'physical never' + fi +} + +drift_list=() +report="" +while IFS=$'\t' read -r slug path remote class kind; do + d="$(_drift "$path")" + if [ -n "$d" ]; then + drift_list+=("$slug ($d)") + report+=" โš  $slug โ€” $d\n" + else + report+=" โœ… $slug โ€” backed up\n" + fi +done < <(ckis::targets) + +if [ "$ONELINE" -eq 1 ]; then + if [ "${#drift_list[@]}" -eq 0 ]; then + printf 'BACKUP โœ… all pushed ยท %s\n' "$(_phys)" + else + printf 'BACKUP โš  %s ยท %s\n' "$(IFS='; '; echo "${drift_list[*]}")" "$(_phys)" + fi +else + printf 'CKIS backup health:\n' + printf '%b' "$report" + printf ' %s\n' "$(_phys)" +fi +exit 0 diff --git a/backup-system/bin/ckis-backup-physical.sh b/backup-system/bin/ckis-backup-physical.sh new file mode 100755 index 0000000..d442a9c --- /dev/null +++ b/backup-system/bin/ckis-backup-physical.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# ckis-backup-physical.sh [dest-root] +# Physical (off-device) backup: mirror every target + full git bundles + the +# class=secret files (the ONLY place secrets are allowed to land โ€” on an +# encrypted external drive). Closes the 3-2-1 rule. +# +# dest-root: explicit path, else auto-detect first mounted candidate from the +# manifest under /. Exits non-zero only if no dest usable. +set -uo pipefail +HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +source "$HERE/../lib/common.sh" +ckis::init + +DEST="${1:-}" +if [ -z "$DEST" ]; then + sub="$(ckis::manifest '.physical.dest_subdir // "CKIS-Backup"')" + while IFS= read -r base; do + base="$(ckis::expand "$base")" + [ -d "$base" ] || continue + # pick the first writable mountpoint under the candidate + for mp in "$base"/*/ "$base"/; do + [ -d "$mp" ] && [ -w "$mp" ] || continue + DEST="${mp%/}/$sub"; break 2 + done + done < <(ckis::manifest '.physical.mount_candidates[]') +fi +[ -n "$DEST" ] || { ckis::err "no physical dest found (pass dest-root or mount the drive)"; exit 3; } + +mkdir -p "$DEST/data" "$DEST/bundles" "$DEST/secrets" 2>/dev/null \ + || { ckis::err "cannot write to dest: $DEST"; exit 3; } +ckis::info "physical backup -> $DEST" + +# FAT/exFAT cannot store POSIX filenames (e.g. ':' is illegal) โ€” the per-file +# mirror would be lossy. git bundles are single FAT-safe files holding full +# history + all filenames, so on FAT we rely on bundles only. +MIRROR=1 +FSTYPE="$(stat -f -c %T "$DEST" 2>/dev/null || echo unknown)" +case "$FSTYPE" in + msdos|vfat|exfat) MIRROR=0 + ckis::warn "dest filesystem is $FSTYPE โ€” skipping per-file mirror; git bundles are the authoritative copy" ;; +esac + +rc=0 +mapfile -t SECRET_GLOBS < <(ckis::manifest '.classes.secret[]') + +while IFS=$'\t' read -r slug path remote class kind; do + [ -d "$path" ] || { ckis::warn "skip missing target: $slug"; continue; } + + # 1. full mirror (working tree incl. .git) โ€” non-FAT only. FAT-tolerant flags + # so it also works on perms-less filesystems. + if [ "$MIRROR" = 1 ]; then + if rsync -rltD --delete --no-perms --no-owner --no-group --modify-window=2 \ + "$path"/ "$DEST/data/$slug"/ 2>>"$CKIS_LOG_FILE"; then + ckis::info "$slug: mirrored" + else + ckis::warn "$slug: rsync mirror had errors (bundle is authoritative)" + fi + fi + + # 2. full-history bundle (single-file, restorable offline) + if ckis::is_repo "$path"; then + if git -C "$path" bundle create "$DEST/bundles/$slug.bundle" --all >>"$CKIS_LOG_FILE" 2>&1 \ + && git -C "$path" bundle verify "$DEST/bundles/$slug.bundle" >/dev/null 2>&1; then + ckis::info "$slug: bundle ok" + else + ckis::err "$slug: bundle failed"; rc=1 + fi + fi + + # 3. class=secret files (physical only) + for g in "${SECRET_GLOBS[@]}"; do + while IFS= read -r -d '' sf; do + rel="${sf#$path/}" + mkdir -p "$DEST/secrets/$slug/$(dirname "$rel")" + cp -a "$sf" "$DEST/secrets/$slug/$rel" 2>/dev/null \ + && ckis::warn "$slug: copied secret-class file to encrypted dest only: $rel" + done < <(find "$path" -name "$g" -type f -print0 2>/dev/null) + done +done < <(ckis::targets) + +date +%s > "$CKIS_LOG_DIR/last-physical" +[ "$rc" -eq 0 ] && ckis::info "physical backup complete" || ckis::err "physical backup completed with errors" +exit "$rc" diff --git a/backup-system/bin/ckis-push.sh b/backup-system/bin/ckis-push.sh new file mode 100755 index 0000000..025b8c0 --- /dev/null +++ b/backup-system/bin/ckis-push.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# ckis-push.sh [commit-msg] +# Real-time per-repo backup push: stage all, commit if dirty, push with retry. +# Idempotent, lock-guarded, never blocks (safe to call from hooks in background). +# * not a git repo -> warn, exit 0 +# * clean and 0 ahead -> no-op, exit 0 +# * dirty -> commit (timestamp msg) then push +# * has unpushed commits -> push +set -uo pipefail +HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +source "$HERE/../lib/common.sh" +ckis::init + +DIR="${1:-}" +[ -n "$DIR" ] || { ckis::err "usage: ckis-push.sh "; exit 2; } +DIR="$(ckis::expand "$DIR")" + +if ! ckis::is_repo "$DIR"; then + ckis::warn "not a git repo, skipping: $DIR"; exit 0 +fi +# Safety: never operate on a parent repo (e.g. $HOME if it is a git repo). +if ! ckis::is_repo_root "$DIR"; then + ckis::err "refusing to push: $DIR is inside a parent git repo, not its own root โ€” skipping to avoid touching the parent" + exit 0 +fi + +_push_one() { + local dir="$1" msg="${2:-}" + local slug; slug="$(basename "$dir")" + + if ckis::repo_dirty "$dir"; then + local big; big="$(ckis::max_staged_mb "$dir")" + git -C "$dir" add -A + big="$(ckis::max_staged_mb "$dir")" + local guard; guard="$(ckis::manifest '.physical.size_guard_mb // 25')" + if [ "${big:-0}" -ge "${guard:-25}" ]; then + ckis::warn "$slug: staging a ${big}MB blob (>= ${guard}MB guard) โ€” backing up anyway" + fi + [ -n "$msg" ] || msg="ckis-backup: auto $(date -u +%Y-%m-%dT%H:%M:%SZ) on $(hostname -s 2>/dev/null || echo host)" + git -C "$dir" commit -q -m "$msg" || { ckis::warn "$slug: nothing to commit after add"; } + ckis::info "$slug: committed local changes" + fi + + if ! ckis::repo_has_remote "$dir"; then + ckis::warn "$slug: no remote configured โ€” committed locally, not pushed" + return 0 + fi + + # Push when no upstream exists yet (establish it) or there are unpushed commits. + # If an upstream exists and we are 0 ahead, it's a genuine no-op. + if git -C "$dir" rev-parse --abbrev-ref --symbolic-full-name '@{u}' >/dev/null 2>&1; then + local ahead; ahead="$(ckis::repo_ahead "$dir")" + if [ "${ahead:-0}" = "0" ]; then + ckis::info "$slug: up to date, nothing to push"; return 0 + fi + fi + local branch; branch="$(git -C "$dir" symbolic-ref --short HEAD 2>/dev/null || echo HEAD)" + if ckis::retry 3 git -C "$dir" push -u origin "$branch"; then + ckis::info "$slug: pushed ($branch)" + else + ckis::err "$slug: push failed after retries โ€” local commit safe, will retry next run" + return 1 + fi +} + +ckis::with_lock "push-$(basename "$DIR")" _push_one "$DIR" "${2:-}" +rc=$? +# Treat "lock held" (75) as a successful no-op for the caller. +[ "$rc" = "75" ] && exit 0 +exit "$rc" diff --git a/backup-system/bin/ckis-restore.sh b/backup-system/bin/ckis-restore.sh new file mode 100755 index 0000000..325f3ee --- /dev/null +++ b/backup-system/bin/ckis-restore.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# ckis-restore.sh [--no-system] [--no-apparatus] +# Plug-and-play rebuild on a fresh machine/OS. Reads the manifest, clones every +# missing target from its private remote, restores the L0 apparatus into +# ~/.claude, and installs the runtime (symlinks, systemd timer, hooks). +# The ONLY manual step afterwards is re-provisioning secrets (never in backup). +set -uo pipefail +HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +source "$HERE/../lib/common.sh" +ckis::init + +NO_SYSTEM=0; NO_APP=0 +for a in "$@"; do case "$a" in --no-system) NO_SYSTEM=1;; --no-apparatus) NO_APP=1;; esac; done + +REMOTE_BASE="${CKIS_REMOTE_BASE:-https://github.com/}" +CLAUDE_HOME="${CKIS_CLAUDE_HOME:-$HOME/.claude}" + +ckis::info "โ•โ•โ• restore start โ•โ•โ•" + +# 1. Clone every missing target. +while IFS=$'\t' read -r slug path remote class kind; do + if [ -d "$path/.git" ]; then + ckis::info "$slug: already present, skip"; continue + fi + [ -n "$remote" ] && [ "$remote" != "null" ] || { ckis::warn "$slug: no remote, cannot restore"; continue; } + local_url="${REMOTE_BASE}${remote}.git" + mkdir -p "$(dirname "$path")" + if ckis::retry 3 git clone "$local_url" "$path" >>"$CKIS_LOG_FILE" 2>&1; then + ckis::info "$slug: cloned from $local_url" + else + ckis::err "$slug: clone failed ($local_url)" + fi +done < <(ckis::targets) + +# 2. Restore L0 apparatus into ~/.claude (merge; never deletes user data). +if [ "$NO_APP" -eq 0 ] && [ -d "$CKIS_INFRA_ROOT/apparatus" ]; then + mkdir -p "$CLAUDE_HOME" + rsync -a "$CKIS_INFRA_ROOT/apparatus"/ "$CLAUDE_HOME"/ >>"$CKIS_LOG_FILE" 2>&1 \ + && ckis::info "apparatus restored into $CLAUDE_HOME" +fi + +# 3. Install runtime (symlinks, systemd, hooks). +if [ "$NO_SYSTEM" -eq 0 ] && [ -x "$CKIS_INFRA_ROOT/install.sh" ]; then + bash "$CKIS_INFRA_ROOT/install.sh" || ckis::warn "install.sh reported issues" +fi + +cat </dev/null) + +# Resolve owner/name from a repo's origin remote URL. +_owner_name() { + local url name rest owner + url="$(git -C "$1" remote get-url origin 2>/dev/null)" || return 1 + url="${url%.git}" + name="${url##*/}" + rest="${url%/*}" + owner="${rest##*[:/]}" + [ -n "$owner" ] && [ -n "$name" ] && printf '%s/%s' "$owner" "$name" +} + +# Visibility: override hook for tests, else gh. +_visibility() { + if [ -n "${CKIS_VIS_CMD:-}" ]; then eval "$CKIS_VIS_CMD \"$1\""; return; fi + gh repo view "$1" --json visibility -q .visibility 2>/dev/null +} + +synced=0 +while IFS=$'\t' read -r slug repo; do + [ -n "$repo" ] || continue + brain="$repo/.brain" + [ -d "$brain" ] || continue + on="$(_owner_name "$repo")" || { ckis::warn "$slug: no origin remote, skipping"; continue; } + vis="$(_visibility "$on")" + case "$vis" in + PUBLIC|public) + mkdir -p "$WORKDIR/$slug" + if rsync -a --delete "${EXC[@]}" "$brain"/ "$WORKDIR/$slug/.brain"/ 2>>"$CKIS_LOG_FILE"; then + ckis::info "$slug: public .brain aggregated"; synced=$((synced+1)) + else + ckis::err "$slug: rsync failed" + fi ;; + PRIVATE|private) + ckis::info "$slug: private repo, .brain backed up via own remote โ€” skip" ;; + *) + ckis::warn "$slug: unknown visibility ('$vis'), skipping for safety" ;; + esac +done < <(jq -r '.projects[] | [.slug, .repo_root] | @tsv' "$REGISTRY") + +ckis::info "aggregator: $synced public .brain dir(s) synced into $WORKDIR" +exit 0 diff --git a/backup-system/ckis-manifest.example.json b/backup-system/ckis-manifest.example.json new file mode 100644 index 0000000..666b31b --- /dev/null +++ b/backup-system/ckis-manifest.example.json @@ -0,0 +1,46 @@ +{ + "version": 1, + "_doc": "EXAMPLE manifest โ€” copy to ckis-manifest.json and edit for your machine. Single source of truth for WHAT to back up and HOW each path is classified. Parsed with jq (the only hard dependency beyond coreutils + git + rsync).", + + "github_owner": "YOUR_GITHUB_USERNAME", + "log_dir": "~/.local/state/ckis-backup", + + "targets": [ + { "slug": "my-notes", "path": "~/Documents/Notes", "remote": "YOUR_GITHUB_USERNAME/my-notes", "class": "track", "kind": "vault" }, + { "slug": "my-configs", "path": "~/Documents/infra", "remote": "YOUR_GITHUB_USERNAME/my-configs", "class": "track", "kind": "infra" } + ], + + "discovery": { + "_doc": "OPTIONAL: aggregate gitignored subdirs (e.g. per-project .brain/) of PUBLIC repos into one private repo. Private repos back up via their own remote and are skipped. Registry is any json with .projects[].{slug, repo_root}.", + "registry": "~/path/to/projects.json", + "public_brain_aggregator": "YOUR_GITHUB_USERNAME/private-aggregator", + "aggregate_workdir": "~/.local/state/ckis-backup/aggregator", + "aggregate_exclude": ["graph/", "node_modules/", ".git/"] + }, + + "apparatus": { + "_doc": "OPTIONAL: curated allowlist export of a config dir (e.g. ~/.claude, ~/.config) into /apparatus/. NEVER a raw mirror; deny[] always wins so secrets and caches are excluded.", + "source": "~/.claude", + "dest": "apparatus", + "allow": ["CLAUDE.md", "settings.json", "skills", "agents", "templates", "scripts"], + "memory_glob": "projects/*/memory", + "deny": [".git", ".credentials.json", "*.local.json", "cache", "node_modules", "*.zip"] + }, + + "classes": { + "_doc": "track=git source of truth; regenerable=excluded with a rebuild recipe; secret=physical encrypted disk only, never a remote; sensitive=private remote now, encrypt later; snapshot=costly binary, physical/LFS only.", + "regenerable": [ + { "glob": "node_modules", "rebuild": "npm ci" }, + { "glob": "code-graph", "rebuild": "" } + ], + "secret": [".credentials.json", "*.env", ".env", "*.pem", "*.key", "id_rsa*", "id_ed25519*"], + "sensitive": [] + }, + + "physical": { + "_doc": "External-drive target. The ONLY home for class=secret/snapshot. Use a LUKS/VeraCrypt-encrypted drive. FAT/exFAT sticks fall back to git bundles only (FAT can't store ':' filenames).", + "mount_candidates": ["/media/$USER", "/run/media/$USER", "/mnt"], + "dest_subdir": "CKIS-Backup", + "size_guard_mb": 25 + } +} diff --git a/backup-system/hooks/pre-commit-secret-scan.sh b/backup-system/hooks/pre-commit-secret-scan.sh new file mode 100755 index 0000000..d08ffad --- /dev/null +++ b/backup-system/hooks/pre-commit-secret-scan.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# pre-commit-secret-scan.sh โ€” block commits that introduce secrets. +# Dependency-free regex scanner (no gitleaks needed). Install as a repo's +# .git/hooks/pre-commit (or call directly to scan staged changes). +# +# Exit 0 = clean, exit 1 = secret found (commit blocked). +# Modes: +# (no args) scan staged diff (git diff --cached) โ€” hook mode +# --files F... scan the given files โ€” standalone/test mode +set -uo pipefail + +# High-signal secret patterns. Conservative to avoid false positives on notes. +PATTERNS='(ghp_[A-Za-z0-9]{36})|(gho_[A-Za-z0-9]{36})|(github_pat_[A-Za-z0-9_]{40,})|(sk-[A-Za-z0-9]{32,})|(AKIA[0-9A-Z]{16})|(AIza[0-9A-Za-z_-]{35})|(xox[baprs]-[A-Za-z0-9-]{10,})|(-----BEGIN ([A-Z]+ )?PRIVATE KEY-----)|(glpat-[A-Za-z0-9_-]{20})' + +scan_stream() { grep -nIE "$PATTERNS"; } # returns 0 if a match is found + +found=0 +report() { echo "๐Ÿ”ด SECRET BLOCK: potential secret in $1"; echo " $2"; found=1; } + +if [ "${1:-}" = "--files" ]; then + shift + for f in "$@"; do + [ -f "$f" ] || continue + while IFS= read -r hit; do report "$f" "$hit"; done < <(scan_stream <"$f") + done +else + # Hook mode: scan the staged content of added/modified files. + while IFS= read -r f; do + [ -n "$f" ] || continue + while IFS= read -r hit; do report "$f" "$hit"; done \ + < <(git show ":$f" 2>/dev/null | scan_stream) + done < <(git diff --cached --name-only --diff-filter=AM 2>/dev/null) +fi + +if [ "$found" -ne 0 ]; then + echo " Commit aborted. Remove the secret or move it to a class=secret path (physical-disk only)." >&2 + exit 1 +fi +exit 0 diff --git a/backup-system/install.sh b/backup-system/install.sh new file mode 100755 index 0000000..2da7150 --- /dev/null +++ b/backup-system/install.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# install.sh โ€” wire the backup runtime into the machine. Idempotent. +# * symlink bin/*.sh -> ~/bin (CKIS_BIN_DIR) +# * install + enable systemd --user timer (unless CKIS_NO_SYSTEMD=1) +# * install secret-scan pre-commit hook into every manifest target repo +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$HERE/lib/common.sh" +ckis::init + +BIN_DIR="${CKIS_BIN_DIR:-$HOME/bin}" +SYSTEMD_DIR="${CKIS_SYSTEMD_DIR:-$HOME/.config/systemd/user}" + +# 1. symlink executables onto PATH +mkdir -p "$BIN_DIR" +for s in "$HERE"/bin/*.sh; do + ln -sf "$s" "$BIN_DIR/$(basename "$s" .sh)" +done +ckis::info "linked $(ls "$HERE"/bin/*.sh | wc -l) scripts into $BIN_DIR" + +# 2. systemd --user timer (daily safety net) +if [ "${CKIS_NO_SYSTEMD:-0}" != "1" ] && command -v systemctl >/dev/null 2>&1; then + mkdir -p "$SYSTEMD_DIR" + sed "s|@BIN@|$BIN_DIR|g" "$HERE/systemd/ckis-backup.service" >"$SYSTEMD_DIR/ckis-backup.service" + cp "$HERE/systemd/ckis-backup.timer" "$SYSTEMD_DIR/ckis-backup.timer" + systemctl --user daemon-reload 2>/dev/null || true + systemctl --user enable --now ckis-backup.timer 2>/dev/null \ + && ckis::info "systemd timer enabled (daily)" \ + || ckis::warn "could not enable systemd timer (headless? run: systemctl --user enable --now ckis-backup.timer)" +fi + +# 3. secret-scan pre-commit into each target repo (don't clobber existing hooks) +while IFS=$'\t' read -r slug path remote class kind; do + [ -d "$path/.git" ] || continue + hook="$path/.git/hooks/pre-commit" + if [ ! -e "$hook" ]; then + ln -sf "$HERE/hooks/pre-commit-secret-scan.sh" "$hook" + ckis::info "$slug: secret-scan pre-commit installed" + elif ! grep -q 'pre-commit-secret-scan' "$hook" 2>/dev/null; then + ckis::warn "$slug: existing pre-commit hook left intact (add secret-scan manually if desired)" + fi +done < <(ckis::targets) + +ckis::info "install complete" diff --git a/backup-system/lib/common.sh b/backup-system/lib/common.sh new file mode 100644 index 0000000..ba05470 --- /dev/null +++ b/backup-system/lib/common.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# lib/common.sh โ€” shared library for the CKIS autonomous backup system. +# +# Invariants every consumer relies on: +# * Pure dependency-free except: coreutils, git, jq, flock, rsync. +# * Never exits the parent shell on error โ€” functions return non-zero, callers decide. +# * All state/logs under $CKIS_LOG_DIR (default ~/.local/state/ckis-backup). +# * Idempotent helpers; safe to source multiple times. +# +# Source this, then call ckis::init before using manifest/log helpers. + +# Guard against double-sourcing. +[ -n "${__CKIS_COMMON_SOURCED:-}" ] && return 0 +__CKIS_COMMON_SOURCED=1 + +# โ”€โ”€ Path resolution โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Resolve the infra repo root from this file's location (lib/ -> repo root). +__ckis_lib_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CKIS_INFRA_ROOT="${CKIS_INFRA_ROOT:-$(cd "$__ckis_lib_dir/.." && pwd)}" + +# Manifest path: env override (used by tests) or repo default. +CKIS_MANIFEST="${CKIS_MANIFEST:-$CKIS_INFRA_ROOT/ckis-manifest.json}" + +# Expand a leading ~ and embedded $HOME/$USER in a path string. +ckis::expand() { + local p="$1" + p="${p/#\~/$HOME}" + p="${p//\$HOME/$HOME}" + p="${p//\$USER/$USER}" + printf '%s' "$p" +} + +# โ”€โ”€ Logging โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +CKIS_LOG_DIR="${CKIS_LOG_DIR:-$(ckis::expand "$(jq -r '.log_dir // "~/.local/state/ckis-backup"' "$CKIS_MANIFEST" 2>/dev/null || echo "~/.local/state/ckis-backup")")}" +CKIS_LOG_FILE="${CKIS_LOG_FILE:-$CKIS_LOG_DIR/backup.log}" + +ckis::init() { + mkdir -p "$CKIS_LOG_DIR" "$CKIS_LOG_DIR/locks" 2>/dev/null || true +} + +# ckis::log LEVEL MSG... โ†’ timestamped line to logfile + stderr. +ckis::log() { + local level="$1"; shift + local ts; ts="$(date -u +"%Y-%m-%dT%H:%M:%SZ")" + local line="[$ts] [$level] $*" + mkdir -p "$CKIS_LOG_DIR" 2>/dev/null || true + printf '%s\n' "$line" >>"$CKIS_LOG_FILE" 2>/dev/null || true + printf '%s\n' "$line" >&2 +} +ckis::info() { ckis::log INFO "$@"; } +ckis::warn() { ckis::log WARN "$@"; } +ckis::err() { ckis::log ERROR "$@"; } + +# โ”€โ”€ Manifest accessors (jq) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# ckis::manifest โ†’ raw jq query over the manifest. +ckis::manifest() { jq -r "$1" "$CKIS_MANIFEST"; } + +# Emit "slugexpanded-pathremoteclasskind" per target. +ckis::targets() { + jq -r '.targets[] | [.slug, .path, .remote, .class, .kind] | @tsv' "$CKIS_MANIFEST" \ + | while IFS=$'\t' read -r slug path remote class kind; do + printf '%s\t%s\t%s\t%s\t%s\n' "$slug" "$(ckis::expand "$path")" "$remote" "$class" "$kind" + done +} + +# โ”€โ”€ Concurrency โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# ckis::with_lock NAME CMD... โ†’ run CMD under an exclusive non-blocking lock. +# Returns 0 on success, 75 (EX_TEMPFAIL) if the lock is already held. +ckis::with_lock() { + local name="$1"; shift + ckis::init + local lock="$CKIS_LOG_DIR/locks/$name.lock" + exec {__ckis_fd}>"$lock" + if ! flock -n "$__ckis_fd"; then + ckis::warn "lock held, skipping: $name" + exec {__ckis_fd}>&- + return 75 + fi + "$@"; local rc=$? + flock -u "$__ckis_fd" 2>/dev/null || true + exec {__ckis_fd}>&- + return $rc +} + +# โ”€โ”€ Retry with backoff โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# ckis::retry N CMD... โ†’ up to N attempts, backoff 2,4,8...s. Returns last rc. +ckis::retry() { + local max="$1"; shift + local n=1 delay=2 rc=0 + while :; do + "$@" && return 0 + rc=$? + [ "$n" -ge "$max" ] && return $rc + ckis::warn "attempt $n/$max failed (rc=$rc), retrying in ${delay}s: $*" + sleep "$delay"; n=$((n+1)); delay=$((delay*2)) + done +} + +# โ”€โ”€ Git helpers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +ckis::is_repo() { git -C "$1" rev-parse --is-inside-work-tree >/dev/null 2>&1; } +# True only if DIR is the ROOT of its own repo (not merely inside a parent repo). +# Critical guard: $HOME may itself be a git repo, so a dir without its own .git +# would otherwise resolve to the $HOME repo and operate on the entire home tree. +ckis::is_repo_root() { + local top; top="$(git -C "$1" rev-parse --show-toplevel 2>/dev/null)" || return 1 + [ "$(readlink -f "$top" 2>/dev/null)" = "$(readlink -f "$1" 2>/dev/null)" ] +} +ckis::repo_dirty() { [ -n "$(git -C "$1" status --porcelain 2>/dev/null)" ]; } +# unpushed-commit count vs upstream; prints 0 when no upstream/clean. +ckis::repo_ahead() { git -C "$1" rev-list --count '@{u}..HEAD' 2>/dev/null || echo 0; } +ckis::repo_has_remote() { [ -n "$(git -C "$1" remote 2>/dev/null)" ]; } + +# Largest newly-added blob size (MB) staged in DIR; 0 if none. For size-guard. +ckis::max_staged_mb() { + local dir="$1" max=0 f sz + while IFS= read -r f; do + [ -f "$dir/$f" ] || continue + sz=$(stat -c%s "$dir/$f" 2>/dev/null || echo 0) + sz=$((sz/1024/1024)) + [ "$sz" -gt "$max" ] && max=$sz + done < <(git -C "$dir" diff --cached --name-only --diff-filter=A 2>/dev/null) + printf '%s' "$max" +} diff --git a/backup-system/systemd/ckis-backup.service b/backup-system/systemd/ckis-backup.service new file mode 100644 index 0000000..29a7dc9 --- /dev/null +++ b/backup-system/systemd/ckis-backup.service @@ -0,0 +1,11 @@ +[Unit] +Description=CKIS autonomous backup (Ring 2 safety net) +After=network-online.target +Wants=network-online.target + +[Service] +Type=oneshot +ExecStart=@BIN@/ckis-backup-all +TimeoutStartSec=1800 +Nice=10 +IOSchedulingClass=idle diff --git a/backup-system/systemd/ckis-backup.timer b/backup-system/systemd/ckis-backup.timer new file mode 100644 index 0000000..0168038 --- /dev/null +++ b/backup-system/systemd/ckis-backup.timer @@ -0,0 +1,12 @@ +[Unit] +Description=CKIS autonomous backup โ€” daily safety net +Documentation=https://github.com/aedneth/ckis + +[Timer] +# Daily, ~30 min after login if missed, with jitter to avoid thundering herds. +OnCalendar=daily +Persistent=true +RandomizedDelaySec=900 + +[Install] +WantedBy=timers.target diff --git a/backup-system/tests/assert.sh b/backup-system/tests/assert.sh new file mode 100755 index 0000000..8210a1b --- /dev/null +++ b/backup-system/tests/assert.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# tests/assert.sh โ€” minimal dependency-free assertion library. +# Each test_*.sh sources this, runs assertions, ends with `assert_summary`. + +_T_PASS=0; _T_FAIL=0 +_t_name() { printf ' %-58s' "$1"; } + +assert_eq() { # desc expected actual + _t_name "$1" + if [ "$2" = "$3" ]; then echo "PASS"; _T_PASS=$((_T_PASS+1)); + else echo "FAIL (expected='$2' actual='$3')"; _T_FAIL=$((_T_FAIL+1)); fi +} +assert_ok() { # desc cmd... + local d="$1"; shift; _t_name "$d" + if "$@" >/dev/null 2>&1; then echo "PASS"; _T_PASS=$((_T_PASS+1)); + else echo "FAIL (rc=$?)"; _T_FAIL=$((_T_FAIL+1)); fi +} +assert_fail() { # desc cmd... (expects non-zero) + local d="$1"; shift; _t_name "$d" + if "$@" >/dev/null 2>&1; then echo "FAIL (expected non-zero)"; _T_FAIL=$((_T_FAIL+1)); + else echo "PASS"; _T_PASS=$((_T_PASS+1)); fi +} +assert_contains() { # desc haystack needle + _t_name "$1" + case "$2" in *"$3"*) echo "PASS"; _T_PASS=$((_T_PASS+1));; + *) echo "FAIL ('$3' not in output)"; _T_FAIL=$((_T_FAIL+1));; esac +} +assert_file() { # desc path + _t_name "$1" + if [ -e "$2" ]; then echo "PASS"; _T_PASS=$((_T_PASS+1)); + else echo "FAIL (missing: $2)"; _T_FAIL=$((_T_FAIL+1)); fi +} +assert_no_file() { # desc path + _t_name "$1" + if [ ! -e "$2" ]; then echo "PASS"; _T_PASS=$((_T_PASS+1)); + else echo "FAIL (should not exist: $2)"; _T_FAIL=$((_T_FAIL+1)); fi +} +assert_summary() { + echo " โ”€โ”€ $_T_PASS passed, $_T_FAIL failed โ”€โ”€" + [ "$_T_FAIL" -eq 0 ] +} diff --git a/backup-system/tests/run.sh b/backup-system/tests/run.sh new file mode 100755 index 0000000..e5d645e --- /dev/null +++ b/backup-system/tests/run.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# tests/run.sh โ€” run all test_*.sh in this dir, plus syntax-check every shipped +# script. The green gate for each stage. Usage: bash tests/run.sh +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +fail=0 + +echo "โ•โ•โ•โ•โ•โ•โ•โ• syntax check (bash -n) โ•โ•โ•โ•โ•โ•โ•โ•" +while IFS= read -r f; do + if bash -n "$f" 2>/tmp/ckis_syn.$$; then printf ' ok %s\n' "${f#$ROOT/}" + else printf ' SYNTAX %s\n' "${f#$ROOT/}"; cat /tmp/ckis_syn.$$; fail=1; fi +done < <(find "$ROOT/bin" "$ROOT/lib" "$ROOT/hooks" -name '*.sh' 2>/dev/null | sort) +rm -f /tmp/ckis_syn.$$ + +# Optional shellcheck if present (non-fatal warnings, fatal errors). +if command -v shellcheck >/dev/null 2>&1; then + echo "โ•โ•โ•โ•โ•โ•โ•โ• shellcheck โ•โ•โ•โ•โ•โ•โ•โ•" + find "$ROOT/bin" "$ROOT/lib" "$ROOT/hooks" -name '*.sh' 2>/dev/null \ + | xargs shellcheck -S error 2>&1 | sed 's/^/ /' || fail=1 +fi + +echo "โ•โ•โ•โ•โ•โ•โ•โ• unit / behavior tests โ•โ•โ•โ•โ•โ•โ•โ•" +for t in "$HERE"/test_*.sh; do + [ -e "$t" ] || continue + echo "โ–ถ $(basename "$t")" + if bash "$t"; then :; else fail=1; fi +done + +echo "โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•" +if [ "$fail" -eq 0 ]; then echo "GATE: โœ… GREEN"; else echo "GATE: โŒ RED"; fi +exit $fail diff --git a/backup-system/tests/test_apparatus_export.sh b/backup-system/tests/test_apparatus_export.sh new file mode 100755 index 0000000..42fc157 --- /dev/null +++ b/backup-system/tests/test_apparatus_export.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Contract for bin/ckis-apparatus-export.sh โ€” the secret-safety is critical. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +source "$HERE/assert.sh" +EXP="$ROOT/bin/ckis-apparatus-export.sh" +SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT +export CKIS_LOG_DIR="$SB/state"; mkdir -p "$CKIS_LOG_DIR" + +# fake ~/.claude +SRC="$SB/dotclaude"; mkdir -p "$SRC"/{skills/foo,agents,cache,session-env/x,projects/proj-a/memory,projects/proj-a/logs} +echo "global instr" >"$SRC/CLAUDE.md" +echo "{}" >"$SRC/settings.json" +echo "skill" >"$SRC/skills/foo/SKILL.md" +echo "TOKEN-LEAK" >"$SRC/.credentials.json" # MUST NOT be exported +echo "junk" >"$SRC/cache/blob" # denied +echo "envblob" >"$SRC/session-env/x/e" # denied +echo "remember this" >"$SRC/projects/proj-a/memory/MEMORY.md" # memory -> keep +echo "transcript" >"$SRC/projects/proj-a/logs/t.jsonl" # projects (non-memory) -> drop + +cat >"$SB/m.json" <"$SRC/skills/bad/.credentials.json" +bash "$EXP" "$DEST" >/dev/null 2>&1 +assert_no_file "nested credential purged" "$DEST/skills/bad/.credentials.json" + +assert_summary diff --git a/backup-system/tests/test_backup_all.sh b/backup-system/tests/test_backup_all.sh new file mode 100755 index 0000000..29af154 --- /dev/null +++ b/backup-system/tests/test_backup_all.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Contract for bin/ckis-backup-all.sh โ€” orchestrator push loop + graceful degrade. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +source "$HERE/assert.sh" +ALL="$ROOT/bin/ckis-backup-all.sh" +SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT +export CKIS_LOG_DIR="$SB/state"; mkdir -p "$CKIS_LOG_DIR" + +# two targets with local bare remotes, both dirty +for t in t1 t2; do + git init -q --bare "$SB/$t.git" + git clone -q "$SB/$t.git" "$SB/$t" 2>/dev/null + git -C "$SB/$t" config user.email a@a; git -C "$SB/$t" config user.name a + echo "data-$t" >"$SB/$t/file.md" +done + +# fixture manifest: targets only, no apparatus/discovery (those steps degrade) +cat >"$SB/m.json" </dev/null | wc -l | tr -d ' ')" +assert_eq "t2 pushed to remote" "1" "$(git --git-dir="$SB/t2.git" log --oneline --all 2>/dev/null | wc -l | tr -d ' ')" + +# idempotent: second run, no new commits +assert_ok "second run exits 0" bash "$ALL" +assert_eq "t1 still 1 commit" "1" "$(git --git-dir="$SB/t1.git" log --oneline --all 2>/dev/null | wc -l | tr -d ' ')" + +# health line present in log +assert_contains "health summary logged" "$(cat "$CKIS_LOG_DIR/backup.log")" "BACKUP" + +assert_summary diff --git a/backup-system/tests/test_cli_brains_sync.sh b/backup-system/tests/test_cli_brains_sync.sh new file mode 100755 index 0000000..b3c391a --- /dev/null +++ b/backup-system/tests/test_cli_brains_sync.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Contract for bin/cli-brains-sync.sh +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +source "$HERE/assert.sh" +SYNC="$ROOT/bin/cli-brains-sync.sh" +SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT +export CKIS_LOG_DIR="$SB/state"; mkdir -p "$CKIS_LOG_DIR" +export CKIS_MANIFEST="$ROOT/ckis-manifest.example.json" + +# a fake public CLI repo with a .brain/ dir and an origin remote +mkrepo() { # path owner/name + git init -q "$1"; git -C "$1" remote add origin "https://github.com/$2.git" + mkdir -p "$1/.brain/sessions"; echo "session log" >"$1/.brain/sessions/s1.md" + echo "decision" >"$1/.brain/decisions.md" + mkdir -p "$1/.brain/graph"; echo '{"big":1}' >"$1/.brain/graph/graph.json" # regenerable +} +mkrepo "$SB/pub" "aedneth/pub-cli" +mkrepo "$SB/priv" "aedneth/priv-app" + +cat >"$SB/reg.json" <"$SB/reg.json" </dev/null +git -C "$SB/clean" config user.email t@t; git -C "$SB/clean" config user.name t +echo a >"$SB/clean/a"; git -C "$SB/clean" add -A; git -C "$SB/clean" commit -q -m a +git -C "$SB/clean" push -q -u origin master 2>/dev/null || git -C "$SB/clean" push -q -u origin main 2>/dev/null + +mkfix() { # path -> writes fixture manifest pointing at it +cat >"$SB/m.json" < all pushed, exits 0 +mkfix "$SB/clean" +out="$(bash "$DOC" --oneline)" +assert_ok "doctor exits 0" bash "$DOC" --oneline +assert_contains "clean target -> all pushed" "$out" "all pushed" +assert_contains "physical never marker" "$out" "physical never" + +# drift: add an unpushed commit +echo b >"$SB/clean/b"; git -C "$SB/clean" add -A; git -C "$SB/clean" commit -q -m b +out2="$(bash "$DOC" --oneline)" +assert_contains "drift shows warning" "$out2" "โš " +assert_contains "drift names target" "$out2" "t1" +assert_contains "drift shows unpushed" "$out2" "unpushed" + +# physical marker age +date +%s >"$CKIS_LOG_DIR/last-physical" +out3="$(bash "$DOC" --oneline)" +assert_contains "physical 0d when marker fresh" "$out3" "physical 0d" + +# missing repo +mkfix "$SB/does-not-exist" +out4="$(bash "$DOC" --oneline)" +assert_contains "missing repo flagged" "$out4" "missing" + +assert_summary diff --git a/backup-system/tests/test_install.sh b/backup-system/tests/test_install.sh new file mode 100755 index 0000000..6e006bc --- /dev/null +++ b/backup-system/tests/test_install.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# Contract for install.sh โ€” symlinks + hook install, no systemd in test. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +source "$HERE/assert.sh" +SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT +export CKIS_LOG_DIR="$SB/state"; mkdir -p "$CKIS_LOG_DIR" + +# one target git repo with no pre-commit hook +git init -q "$SB/t1" +cat >"$SB/m.json" < \$HOME" "$HOME/x" "$(ckis::expand '~/x')" +assert_eq "expand \$HOME literal" "$HOME/y" "$(ckis::expand '$HOME/y')" +assert_eq "expand plain path" "/a/b" "$(ckis::expand '/a/b')" + +# โ”€โ”€ logging โ”€โ”€ +ckis::info "hello-test" +assert_file "log file created" "$CKIS_LOG_FILE" +assert_contains "log contains msg" "$(cat "$CKIS_LOG_FILE")" "hello-test" + +# โ”€โ”€ locking โ”€โ”€ +ckis::with_lock free_lock true +assert_eq "free lock runs cmd -> 0" "0" "$?" + +mkdir -p "$CKIS_LOG_DIR/locks" +exec 9>"$CKIS_LOG_DIR/locks/busy.lock"; flock -n 9 +ckis::with_lock busy true; rc=$? +flock -u 9; exec 9>&- +assert_eq "held lock -> 75 (skip)" "75" "$rc" + +# โ”€โ”€ retry โ”€โ”€ +ckis::retry 1 true; assert_eq "retry ok first try" "0" "$?" +ckis::retry 1 false; rc=$?; assert_eq "retry exhausts -> non-zero" "1" "$rc" + +CNT="$SANDBOX/cnt"; echo 0 >"$CNT" +flaky() { local n; n=$(cat "$CNT"); n=$((n+1)); echo "$n" >"$CNT"; [ "$n" -ge 2 ]; } +ckis::retry 3 flaky; assert_eq "retry succeeds on 2nd attempt" "0" "$?" + +# โ”€โ”€ git helpers โ”€โ”€ +R="$SANDBOX/repo"; mkdir -p "$R"; git -C "$R" init -q +assert_ok "is_repo true" ckis::is_repo "$R" +assert_fail "is_repo false on plain" ckis::is_repo "$SANDBOX" +echo hi >"$R/f"; +ckis::repo_dirty "$R"; assert_eq "repo_dirty detects untracked" "0" "$?" + +# is_repo_root: the root is a root; a subdir inside it is NOT +mkdir -p "$R/sub/deep" +assert_ok "is_repo_root true at root" ckis::is_repo_root "$R" +assert_fail "is_repo_root false in subdir" ckis::is_repo_root "$R/sub/deep" +assert_ok "is_repo true in subdir" ckis::is_repo "$R/sub/deep" + +assert_summary diff --git a/backup-system/tests/test_manifest.sh b/backup-system/tests/test_manifest.sh new file mode 100755 index 0000000..8d62fcd --- /dev/null +++ b/backup-system/tests/test_manifest.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Generic contract for any ckis-manifest โ€” validates structure, not specific +# values. Runs against ckis-manifest.example.json by default. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +source "$HERE/assert.sh" +export CKIS_MANIFEST="${CKIS_MANIFEST:-$ROOT/ckis-manifest.example.json}" +source "$ROOT/lib/common.sh" + +assert_ok "manifest is valid json" jq -e . "$CKIS_MANIFEST" +assert_ok "github_owner set" bash -c "[ -n \"\$(jq -r .github_owner '$CKIS_MANIFEST')\" ]" + +# targets: at least one, each well-formed, paths expand to absolute +assert_ok "has >= 1 target" bash -c "[ \$(jq '.targets|length' '$CKIS_MANIFEST') -ge 1 ]" +while IFS=$'\t' read -r slug path remote class kind; do + [ -n "$slug" ] && [ -n "$class" ] || { echo " target missing fields"; exit 1; } + case "$path" in /*) ;; *) echo " non-absolute path: $path"; exit 1;; esac +done < <(ckis::targets) +assert_eq "all targets well-formed + absolute" "0" "0" + +# apparatus allow/deny disjoint + denies nested .git (if apparatus present) +if jq -e '.apparatus' "$CKIS_MANIFEST" >/dev/null 2>&1; then + allow="$(ckis::manifest '.apparatus.allow[]?')" + deny="$(ckis::manifest '.apparatus.deny[]?')" + overlap="$(comm -12 <(echo "$allow"|sort -u) <(echo "$deny"|sort -u) | sed '/^$/d')" + assert_eq "apparatus allow/deny disjoint" "" "$overlap" + assert_contains "deny excludes nested .git" "$deny" ".git" +fi + +# classes: regenerable carries a rebuild recipe; secret class non-empty +if jq -e '.classes.regenerable' "$CKIS_MANIFEST" >/dev/null 2>&1; then + assert_ok "regenerable has rebuild recipe" bash -c "jq -e '.classes.regenerable[0].rebuild' '$CKIS_MANIFEST'" +fi +assert_ok "secret class non-empty" bash -c "[ \$(jq '.classes.secret|length' '$CKIS_MANIFEST') -ge 1 ]" +assert_eq "size guard is numeric" "0" "$(jq -r '.physical.size_guard_mb' "$CKIS_MANIFEST" | grep -cE '[^0-9]')" + +assert_summary diff --git a/backup-system/tests/test_physical.sh b/backup-system/tests/test_physical.sh new file mode 100755 index 0000000..b80c48e --- /dev/null +++ b/backup-system/tests/test_physical.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Contract for bin/ckis-backup-physical.sh +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +source "$HERE/assert.sh" +PHYS="$ROOT/bin/ckis-backup-physical.sh" +SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT +export CKIS_LOG_DIR="$SB/state"; mkdir -p "$CKIS_LOG_DIR" + +# target repo with a tracked file and a secret-class file +git init -q "$SB/t1"; git -C "$SB/t1" config user.email t@t; git -C "$SB/t1" config user.name t +echo "knowledge" >"$SB/t1/note.md" +printf 'TOKEN=xyz\n' >"$SB/t1/.env" # class=secret (*.env) +git -C "$SB/t1" add note.md; git -C "$SB/t1" commit -q -m note + +cat >"$SB/m.json" < secrets/ only" "$DEST/secrets/t1/.env" +assert_file "last-physical marker written" "$CKIS_LOG_DIR/last-physical" + +# bundle is restorable into a fresh clone +assert_ok "bundle clones back" git clone -q "$DEST/bundles/t1.bundle" "$SB/restored" +assert_file "restored content present" "$SB/restored/note.md" + +# no dest + no mount -> non-zero +assert_fail "no dest, no mount -> error" bash "$PHYS" + +assert_summary diff --git a/backup-system/tests/test_push.sh b/backup-system/tests/test_push.sh new file mode 100755 index 0000000..dea8b95 --- /dev/null +++ b/backup-system/tests/test_push.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Contract for bin/ckis-push.sh +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +source "$HERE/assert.sh" +PUSH="$ROOT/bin/ckis-push.sh" +SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT +export CKIS_LOG_DIR="$SB/state" + +# bare remote + working clone +git init -q --bare "$SB/remote.git" +git clone -q "$SB/remote.git" "$SB/work" 2>/dev/null +git -C "$SB/work" config user.email t@t; git -C "$SB/work" config user.name t + +# non-repo -> exit 0 +assert_ok "non-repo exits 0" bash "$PUSH" "$SB/not-a-repo" + +# dirty -> commit + push +echo "first" >"$SB/work/a.md" +assert_ok "push dirty repo" bash "$PUSH" "$SB/work" +got="$(git --git-dir="$SB/remote.git" log --oneline --all 2>/dev/null | wc -l | tr -d ' ')" +assert_eq "remote received 1 commit" "1" "$got" + +# clean again -> no-op, no new commit +assert_ok "push clean repo (no-op)" bash "$PUSH" "$SB/work" +got2="$(git --git-dir="$SB/remote.git" log --oneline --all 2>/dev/null | wc -l | tr -d ' ')" +assert_eq "remote still 1 commit (idempotent)" "1" "$got2" + +# coalescing: many files -> single commit +for i in $(seq 1 20); do echo "f$i" >"$SB/work/f$i.md"; done +assert_ok "push 20 new files" bash "$PUSH" "$SB/work" +got3="$(git --git-dir="$SB/remote.git" log --oneline --all 2>/dev/null | wc -l | tr -d ' ')" +assert_eq "20 files = 1 new commit (coalesced)" "2" "$got3" + +# repo without remote: commits locally, still exit 0 +git init -q "$SB/norem"; git -C "$SB/norem" config user.email t@t; git -C "$SB/norem" config user.name t +echo x >"$SB/norem/x" +assert_ok "no-remote repo commits locally, exit 0" bash "$PUSH" "$SB/norem" +assert_eq "local commit made" "1" "$(git -C "$SB/norem" log --oneline | wc -l | tr -d ' ')" + +# SAFETY: pushing a subdir that belongs to a parent repo must NOT touch the parent +git init -q "$SB/parent"; git -C "$SB/parent" config user.email t@t; git -C "$SB/parent" config user.name t +mkdir -p "$SB/parent/child"; echo data >"$SB/parent/child/x.md" +assert_ok "subdir-of-repo push is a safe no-op" bash "$PUSH" "$SB/parent/child" +assert_eq "parent repo NOT committed" "0" "$(git -C "$SB/parent" log --oneline --all 2>/dev/null | wc -l | tr -d ' ')" + +assert_summary diff --git a/backup-system/tests/test_restore.sh b/backup-system/tests/test_restore.sh new file mode 100755 index 0000000..d884ebf --- /dev/null +++ b/backup-system/tests/test_restore.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Contract for bin/ckis-restore.sh โ€” clones missing targets, safe & idempotent. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +source "$HERE/assert.sh" +RES="$ROOT/bin/ckis-restore.sh" +SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT +export CKIS_LOG_DIR="$SB/state"; mkdir -p "$CKIS_LOG_DIR" + +# seed a bare "remote" with content +mkdir -p "$SB/remotes" +git init -q --bare "$SB/remotes/t1.git" +git clone -q "$SB/remotes/t1.git" "$SB/seed" 2>/dev/null +git -C "$SB/seed" config user.email a@a; git -C "$SB/seed" config user.name a +echo "restored knowledge" >"$SB/seed/note.md"; git -C "$SB/seed" add -A; git -C "$SB/seed" commit -q -m init +git -C "$SB/seed" push -q -u origin master 2>/dev/null || git -C "$SB/seed" push -q -u origin main 2>/dev/null + +cat >"$SB/m.json" < still ok, content intact +assert_ok "second restore idempotent" bash "$RES" --no-system --no-apparatus +assert_file "content still present" "$SB/restored/t1/note.md" + +assert_summary diff --git a/backup-system/tests/test_secret_scan.sh b/backup-system/tests/test_secret_scan.sh new file mode 100755 index 0000000..58ed13a --- /dev/null +++ b/backup-system/tests/test_secret_scan.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Contract for hooks/pre-commit-secret-scan.sh +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +source "$HERE/assert.sh" +HOOK="$ROOT/hooks/pre-commit-secret-scan.sh" +SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT + +# --files mode: clean file passes, token file blocks. +printf 'just some notes about tokens and keys\n' >"$SB/clean.md" +printf 'token = ghp_%s\n' "0123456789012345678901234567890123456789" >"$SB/bad.md" +assert_ok "clean file passes" bash "$HOOK" --files "$SB/clean.md" +assert_fail "github token blocks" bash "$HOOK" --files "$SB/bad.md" + +printf -- '%s PRIVATE KEY-----\nabc\n' '-----BEGIN' >"$SB/pem.md" +assert_fail "private key blocks" bash "$HOOK" --files "$SB/pem.md" + +# Hook mode inside a real repo. +R="$SB/repo"; mkdir -p "$R"; git -C "$R" init -q +cp "$HOOK" "$R/.git/hooks/pre-commit"; chmod +x "$R/.git/hooks/pre-commit" +git -C "$R" config user.email t@t; git -C "$R" config user.name t +echo "hello world" >"$R/ok.md"; git -C "$R" add ok.md +assert_ok "clean commit succeeds" git -C "$R" commit -q -m ok +printf 'AKIA%s\n' "ABCDEFGHIJKLMNOP" >"$R/leak.md"; git -C "$R" add leak.md +assert_fail "aws key commit blocked" git -C "$R" commit -q -m leak + +assert_summary From a56bf39afda393a2fb273dd0dc24ae9caac1ff57 Mon Sep 17 00:00:00 2001 From: Eduardo Borjas Date: Tue, 16 Jun 2026 12:49:55 -0600 Subject: [PATCH 2/2] =?UTF-8?q?feat(backup-system):=20v2=20=E2=80=94=20cen?= =?UTF-8?q?tralized=20brains,=20strongest=20scanner,=20hard-fail,=20reconc?= =?UTF-8?q?ile=20floor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync the sanitized backup framework to v2: - Strongest secret scanner: entropy-gated tokens + PEM body-proximity (real secrets, not prose/placeholder mentions), .git/config + filename coverage, .ckis-secret-allow escape valve; new ckis-secret-audit (whole-system). - Centralized brain aggregation (brains-sync): every project's memory subdir regardless of tool or visibility; discovery = registry โˆช filesystem scan. - Reconciling timer as the tool-agnostic real-time floor (daily -> 15min, manifest-templated); no inotify dependency. - Hard-fail observability: failure markers, backup-all exits non-zero, doctor shows ๐Ÿ”ด FAILED vs benign drift (fixes silent-success). - Physical mirror excludes .git so no .git/config credential reaches the drive. - Clean-room restore test; example manifest + README updated; 12 test suites green. Co-Authored-By: Claude Opus 4.8 --- backup-system/README.md | 21 ++- backup-system/bin/brains-sync.sh | 38 ++++ backup-system/bin/ckis-backup-all.sh | 103 +++++++++-- backup-system/bin/ckis-backup-doctor.sh | 14 +- backup-system/bin/ckis-backup-physical.sh | 10 +- backup-system/bin/ckis-push.sh | 29 ++- backup-system/bin/ckis-restore.sh | 19 ++ backup-system/bin/ckis-secret-audit.sh | 54 ++++++ backup-system/bin/cli-brains-sync.sh | 66 ------- backup-system/ckis-manifest.example.json | 25 ++- backup-system/hooks/pre-commit-secret-scan.sh | 173 +++++++++++++++--- backup-system/install.sh | 24 ++- backup-system/lib/common.sh | 87 +++++++++ backup-system/systemd/ckis-backup.timer | 12 +- backup-system/tests/test_backup_all.sh | 19 ++ backup-system/tests/test_brains_sync.sh | 57 ++++++ backup-system/tests/test_cli_brains_sync.sh | 51 ------ backup-system/tests/test_doctor.sh | 14 ++ backup-system/tests/test_install.sh | 7 + backup-system/tests/test_lib.sh | 31 ++++ backup-system/tests/test_physical.sh | 10 + backup-system/tests/test_push.sh | 16 ++ backup-system/tests/test_restore.sh | 51 ++++-- backup-system/tests/test_secret_audit.sh | 40 ++++ backup-system/tests/test_secret_scan.sh | 84 +++++++-- 25 files changed, 837 insertions(+), 218 deletions(-) create mode 100755 backup-system/bin/brains-sync.sh create mode 100755 backup-system/bin/ckis-secret-audit.sh delete mode 100755 backup-system/bin/cli-brains-sync.sh create mode 100755 backup-system/tests/test_brains_sync.sh delete mode 100755 backup-system/tests/test_cli_brains_sync.sh create mode 100644 backup-system/tests/test_secret_audit.sh diff --git a/backup-system/README.md b/backup-system/README.md index e377b69..f6dfe3b 100644 --- a/backup-system/README.md +++ b/backup-system/README.md @@ -14,9 +14,9 @@ Most "backups" are a static cron job that rots the moment your setup changes. Th ## How it works โ€” three rings of autonomy -1. **Ring 1 โ€” push on event (real time).** A git hook calls `ckis-push.sh ` at the end of a work session: stage-all โ†’ commit-if-dirty โ†’ push with retry. Detached, lock-guarded, never blocks you. 100 new files = 1 commit = 1 push. -2. **Ring 2 โ€” daily safety net.** A `systemd --user` timer runs `ckis-backup-all.sh`: reads the manifest, **auto-creates missing private remotes** (self-healing onboarding), pushes any drift, aggregates public repos' gitignored subdirs into a private repo, exports the curated config apparatus, and runs the physical backup if a drive is mounted. -3. **Ring 3 โ€” passive visibility.** `ckis-backup-doctor.sh --oneline` prints a one-line health status (`BACKUP โœ… all pushed ยท physical 2d`) you can drop into a shell prompt or a session banner โ€” so you see drift without checking. +1. **Ring 1 โ€” push on event (fast path).** A git hook or session-stop hook calls `ckis-push.sh `: stage-all โ†’ commit-if-dirty โ†’ push with retry. Detached, lock-guarded, never blocks you. 100 new files = 1 commit = 1 push. **It fails hard** if a commit is blocked โ€” never reports a stalled backup as success. +2. **Ring 2 โ€” reconciling safety net (the real-time floor).** A `systemd --user` timer runs `ckis-backup-all.sh` on a short interval (default **15 min**, set in the manifest). Being a *reconcile* rather than an event hook makes it **tool-agnostic** โ€” it captures changes no matter which editor/agent made them, with no `inotify` dependency. It auto-creates missing private remotes, pushes drift, **centralizes every project's memory subdir** (any tool, any visibility) into one private repo, exports the curated config apparatus, sweeps every `.git/config` for embedded credentials, runs a throttled deep secret-audit, and does the physical backup if a drive is mounted. **It exits non-zero on any real failure.** +3. **Ring 3 โ€” passive visibility.** `ckis-backup-doctor.sh --oneline` prints a one-line health status (`BACKUP โœ… all pushed ยท physical 2d`) for a shell prompt or session banner. A blocked backup shows **๐Ÿ”ด FAILED** (a real problem), distinct from a benign **โš ** drift. ## Layers it backs up @@ -46,8 +46,9 @@ Wire Ring 1 into wherever a "work session" ends (a git `post-commit` hook, an ed ## Usage ```bash -ckis-backup-doctor # health report -ckis-backup-all # force a full run (push drift, aggregate, physical) +ckis-backup-doctor # health report (๐Ÿ”ด FAILED = real block, โš  = benign drift) +ckis-backup-all # force a full run (push drift, centralize, audit, physical) +ckis-secret-audit # scan every repo's working tree + .git/config for real secrets ckis-backup-physical # physical backup to a mounted drive (auto-detected) ``` @@ -70,17 +71,19 @@ Every target is just a git repo, so any mobile git client works. For Obsidian va - **`$HOME` may itself be a git repo.** A directory without its own `.git` then resolves to the `$HOME` repo, and `git add -A` will try to stage your entire home directory. Every script guards with `is_repo_root` and refuses to operate on a parent repo. *(This one cost an 11-minute CPU hang to find.)* - **FAT/exFAT can't store `:` in filenames.** On FAT drives the per-file mirror is skipped; **git bundles** (single files holding full history + all filenames) are the authoritative physical copy. - **Heavy/third-party content is `regenerable`, not backed up** โ€” e.g. a skill that bundles a 1 GB headless browser is reinstalled, not committed. -- **Secrets never leave the machine.** A `pre-commit` secret scanner (dependency-free regex for GitHub/AWS/GCP keys + private keys) blocks accidental commits; the manifest `deny[]` and a defense-in-depth purge keep credentials out of the config export. +- **Secrets never leave the machine.** A dependency-free `pre-commit` scanner blocks accidental commits โ€” but it detects **real key material**, not mentions: tokens are gated by Shannon **entropy** (so a `ghp_xxxxโ€ฆ` placeholder or a doc that quotes one passes), PEM keys require an actual base64 body next to the marker, and it also covers **`.git/config` remote-URL credentials** and `class=secret` filenames. A `.ckis-secret-allow` file (or an inline `ckis-allow-secret` marker) sanctions docs/tests that legitimately quote a pattern. +- **Silent success is the worst failure.** A backup that reports green while doing nothing is more dangerous than a crash. Marker-matching secret scanners false-positive on notes that *document* security work and can wedge a knowledge base's backup for hours; every failure path here exits non-zero and the banner shows ๐Ÿ”ด, and `ckis-secret-audit` re-checks the whole system independently of any exit code. ## Layout ``` bin/ ckis-push, ckis-backup-all, ckis-backup-doctor, ckis-backup-physical, - cli-brains-sync, ckis-apparatus-export, ckis-restore -lib/ common.sh (logging, flock, retry, manifest accessors, git helpers) + brains-sync, ckis-secret-audit, ckis-apparatus-export, ckis-restore +lib/ common.sh (logging, flock, retry, entropy, failure markers, + agent-agnostic brain discovery, git helpers) hooks/ pre-commit-secret-scan.sh systemd/ ckis-backup.{service,timer} -tests/ pure-bash test harness (run: bash tests/run.sh) โ€” 106 assertions +tests/ pure-bash test harness (run: bash tests/run.sh) โ€” 12 suites install.sh ยท ckis-manifest.example.json ``` diff --git a/backup-system/bin/brains-sync.sh b/backup-system/bin/brains-sync.sh new file mode 100755 index 0000000..4d949bf --- /dev/null +++ b/backup-system/bin/brains-sync.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# brains-sync.sh +# CENTRALIZE every per-project .brain/ into one private aggregator workdir, +# regardless of which coding agent created the project (Claude Code, Codex, +# Gemini, OpenCode, ...) or the project's own visibility. The CLIs are just +# projects โ€” there is no CLI-specific or public/private branching here. +# +# Discovery is agent-agnostic: the UNION of Dev Brain's registry and a bounded +# filesystem scan of brain_roots (ckis::brain_repos). Regenerable/redundant +# content (graph/, .brain-backup-*) is excluded so the aggregate stays high-value. +set -uo pipefail +HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +source "$HERE/../lib/common.sh" +ckis::init + +WORKDIR="${CKIS_AGG_WORKDIR:-$(ckis::expand "$(ckis::manifest '.discovery.aggregate_workdir')")}" +case "$WORKDIR" in ''|null) ckis::warn "no aggregate_workdir configured โ€” skipping brains-sync"; exit 0 ;; esac +mkdir -p "$WORKDIR" + +EXC=() +while IFS= read -r e; do [ -n "$e" ] && EXC+=(--exclude "$e"); done \ + < <(ckis::manifest '.discovery.aggregate_exclude[]?' 2>/dev/null) + +synced=0 +while IFS=$'\t' read -r slug repo; do + [ -n "$repo" ] || continue + brain="$repo/.brain" + [ -d "$brain" ] || continue + mkdir -p "$WORKDIR/$slug" + if rsync -a --delete "${EXC[@]}" "$brain"/ "$WORKDIR/$slug/.brain"/ 2>>"$CKIS_LOG_FILE"; then + ckis::info "$slug: .brain centralized"; synced=$((synced+1)) + else + ckis::err "$slug: rsync failed" + fi +done < <(ckis::brain_repos) + +ckis::info "aggregator: $synced .brain dir(s) centralized into $WORKDIR" +exit 0 diff --git a/backup-system/bin/ckis-backup-all.sh b/backup-system/bin/ckis-backup-all.sh index 62fa616..ce78456 100755 --- a/backup-system/bin/ckis-backup-all.sh +++ b/backup-system/bin/ckis-backup-all.sh @@ -12,6 +12,8 @@ _have_gh() { command -v gh >/dev/null 2>&1 && gh auth status >/dev/null 2>&1; } _run() { ckis::info "โ•โ•โ• backup-all start โ•โ•โ•" + local failures=0 + local scan="$HERE/../hooks/pre-commit-secret-scan.sh" # 1. Export L0 apparatus into the infra repo (so the next push captures it). local app_src; app_src="$(ckis::manifest '.apparatus.source // empty')" @@ -32,28 +34,82 @@ _run() { ckis::warn "$slug: no remote and gh unavailable โ€” committing locally only" fi fi - bash "$HERE/ckis-push.sh" "$path" || ckis::warn "$slug: push reported an error" + bash "$HERE/ckis-push.sh" "$path" || { ckis::warn "$slug: push reported an error"; failures=$((failures+1)); } done < <(ckis::targets) - # 3. Aggregate public .brain/ dirs, then push the aggregator repo. + # 3. Centralize every project .brain/ (any agent, any visibility), then push. local reg; reg="$(ckis::expand "$(ckis::manifest '.discovery.registry // empty')")" - if [ -n "$reg" ] && [ -f "$reg" ]; then - bash "$HERE/cli-brains-sync.sh" || ckis::warn "aggregator sync had issues" - local agg_dir agg_remote - agg_dir="$(ckis::expand "$(ckis::manifest '.discovery.aggregate_workdir // empty')")" - agg_remote="$(ckis::manifest '.discovery.public_brain_aggregator // empty')" - if [ -n "$agg_dir" ] && [ -d "$agg_dir" ]; then - # Ensure the aggregator has its OWN .git (a dir without one would resolve - # to a parent repo such as $HOME and add the whole home tree). - if ! ckis::is_repo_root "$agg_dir"; then - git -C "$agg_dir" init -q - git -C "$agg_dir" symbolic-ref HEAD refs/heads/master 2>/dev/null || true - ckis::info "aggregator: initialized own git repo at $agg_dir" - fi - if ! ckis::repo_has_remote "$agg_dir" && [ -n "$agg_remote" ] && _have_gh; then - git -C "$agg_dir" remote add origin "https://github.com/$agg_remote.git" 2>/dev/null || true + bash "$HERE/brains-sync.sh" || ckis::warn "brains-sync had issues" + local agg_dir agg_remote + agg_dir="$(ckis::expand "$(ckis::manifest '.discovery.aggregate_workdir // empty')")" + agg_remote="$(ckis::manifest '.discovery.brain_aggregator // empty')" + if [ -n "$agg_dir" ] && [ "$agg_dir" != "null" ] && [ -d "$agg_dir" ]; then + # Ensure the aggregator has its OWN .git (a dir without one would resolve to + # a parent repo such as $HOME and add the whole home tree). + if ! ckis::is_repo_root "$agg_dir"; then + git -C "$agg_dir" init -q + git -C "$agg_dir" symbolic-ref HEAD refs/heads/master 2>/dev/null || true + ckis::info "aggregator: initialized own git repo at $agg_dir" + fi + # The aggregator holds private brains too, so it MUST be secret-scanned on + # commit just like every other backup target. + if [ ! -e "$agg_dir/.git/hooks/pre-commit" ]; then + ln -sf "$HERE/../hooks/pre-commit-secret-scan.sh" "$agg_dir/.git/hooks/pre-commit" 2>/dev/null || true + fi + if ! ckis::repo_has_remote "$agg_dir" && [ -n "$agg_remote" ] && _have_gh; then + if ! gh repo view "$agg_remote" >/dev/null 2>&1; then + gh repo create "$agg_remote" --private >>"$CKIS_LOG_FILE" 2>&1 || ckis::warn "aggregator: repo auto-create failed" fi - bash "$HERE/ckis-push.sh" "$agg_dir" || ckis::warn "aggregator push had issues" + git -C "$agg_dir" remote add origin "https://github.com/$agg_remote.git" 2>/dev/null || true + fi + bash "$HERE/ckis-push.sh" "$agg_dir" || { ckis::warn "aggregator push had issues"; failures=$((failures+1)); } + fi + + # 3.5 Self-healing wiring + secret blind-spot sweep across every discovered + # brain repo (any agent). Ensures new projects auto-get secret-scan + # protection, and catches credentials embedded in any repo's .git/config + # (the staged-content hook structurally cannot see those). + # .gitcfg marker namespace is re-evaluated every run (set if a token is found, + # cleared when fixed) โ€” kept separate from push markers, which a successful + # push clears even though the embedded token would still be there. + _sweep_cfg() { # slug repo + if bash "$scan" --git-config "$2" >/dev/null 2>&1; then + ckis::clear_fail "$1.gitcfg" + else + ckis::err "$1: embedded credential in .git/config" + ckis::mark_fail "$1.gitcfg" "embedded credential in .git/config" + failures=$((failures+1)) + fi + } + while IFS=$'\t' read -r slug path remote class kind; do + [ -d "$path" ] || continue + _sweep_cfg "$slug" "$path" + done < <(ckis::targets) + while IFS=$'\t' read -r slug repo; do + [ -d "$repo" ] || continue + if [ ! -e "$repo/.git/hooks/pre-commit" ]; then + ln -sf "$scan" "$repo/.git/hooks/pre-commit" 2>/dev/null \ + && ckis::info "$slug: secret-scan pre-commit auto-installed" + fi + _sweep_cfg "$slug" "$repo" + done < <(ckis::brain_repos) + + # 3.6 Deep secret audit (full content scan of every repo), throttled to ~daily. + # The universal net for repos whose own pre-commit we don't control (CLIs + # with a build/supply-chain hook, etc.): a secret committed anywhere is + # detected within a day and surfaces as a hard failure, even without our + # pre-commit installed there. Cheap to amortize over a day. + local audit_marker="$CKIS_LOG_DIR/last-audit" now last_audit=0 + now="$(date +%s)" + [ -f "$audit_marker" ] && last_audit="$(cat "$audit_marker" 2>/dev/null || echo 0)" + case "$last_audit" in ''|*[!0-9]*) last_audit=0 ;; esac + if [ "$(( now - last_audit ))" -ge 86400 ]; then + if bash "$HERE/ckis-secret-audit.sh" >>"$CKIS_LOG_FILE" 2>&1; then + ckis::clear_fail "secret-audit"; printf '%s' "$now" >"$audit_marker" + else + ckis::err "deep secret audit FOUND secret material โ€” see log" + ckis::mark_fail "secret-audit" "deep audit found secret material" + failures=$((failures+1)) fi fi @@ -66,7 +122,18 @@ _run() { # 5. Health summary. ckis::info "health: $(bash "$HERE/ckis-backup-doctor.sh" --oneline)" + + # 6. HARD-FAIL signal. Exit non-zero if anything failed this run OR a prior + # unresolved failure marker persists, so systemd shows failure (not the old + # silent "success") and the SessionStart banner screams. This is the fix for + # the 24h-silent-outage: a blocked backup can no longer masquerade as green. + local lingering; lingering="$(ckis::list_fails | tr '\n' ' ')" + if [ "$failures" -gt 0 ] || [ -n "${lingering// /}" ]; then + ckis::err "โ•โ•โ• backup-all done WITH FAILURES (this run: $failures ยท unresolved: ${lingering:-none}) โ•โ•โ•" + return 1 + fi ckis::info "โ•โ•โ• backup-all done โ•โ•โ•" + return 0 } ckis::with_lock "backup-all" _run diff --git a/backup-system/bin/ckis-backup-doctor.sh b/backup-system/bin/ckis-backup-doctor.sh index 648561c..cb349f2 100755 --- a/backup-system/bin/ckis-backup-doctor.sh +++ b/backup-system/bin/ckis-backup-doctor.sh @@ -43,6 +43,10 @@ _phys() { drift_list=() report="" while IFS=$'\t' read -r slug path remote class kind; do + if ckis::is_failed "$slug"; then + report+=" ๐Ÿ”ด $slug โ€” FAILED: $(ckis::fail_reason "$slug")\n" + continue + fi d="$(_drift "$path")" if [ -n "$d" ]; then drift_list+=("$slug ($d)") @@ -52,8 +56,15 @@ while IFS=$'\t' read -r slug path remote class kind; do fi done < <(ckis::targets) +# Authoritative FAILED set = every persistent marker (targets AND brain repos, +# incl. .gitcfg security findings). A hard failure outranks benign drift. +failed_all="$(ckis::list_fails 2>/dev/null | tr '\n' ' ')" +failed_all="${failed_all%% }" + if [ "$ONELINE" -eq 1 ]; then - if [ "${#drift_list[@]}" -eq 0 ]; then + if [ -n "${failed_all// /}" ]; then + printf 'BACKUP ๐Ÿ”ด FAILED: %s ยท %s\n' "$failed_all" "$(_phys)" + elif [ "${#drift_list[@]}" -eq 0 ]; then printf 'BACKUP โœ… all pushed ยท %s\n' "$(_phys)" else printf 'BACKUP โš  %s ยท %s\n' "$(IFS='; '; echo "${drift_list[*]}")" "$(_phys)" @@ -61,6 +72,7 @@ if [ "$ONELINE" -eq 1 ]; then else printf 'CKIS backup health:\n' printf '%b' "$report" + [ -n "${failed_all// /}" ] && printf ' ๐Ÿ”ด FAILED markers: %s\n' "$failed_all" printf ' %s\n' "$(_phys)" fi exit 0 diff --git a/backup-system/bin/ckis-backup-physical.sh b/backup-system/bin/ckis-backup-physical.sh index d442a9c..d70b395 100755 --- a/backup-system/bin/ckis-backup-physical.sh +++ b/backup-system/bin/ckis-backup-physical.sh @@ -46,12 +46,16 @@ mapfile -t SECRET_GLOBS < <(ckis::manifest '.classes.secret[]') while IFS=$'\t' read -r slug path remote class kind; do [ -d "$path" ] || { ckis::warn "skip missing target: $slug"; continue; } - # 1. full mirror (working tree incl. .git) โ€” non-FAT only. FAT-tolerant flags - # so it also works on perms-less filesystems. + # 1. working-tree mirror โ€” non-FAT only. FAT-tolerant flags so it also works on + # perms-less filesystems. .git/ is EXCLUDED: it can hold credentials in + # .git/config remote URLs (the PAT blind spot), and git history is already + # captured by the authoritative bundle below. The mirror is for browsing + # files, not for cloning โ€” so no git internals ever land on the drive. if [ "$MIRROR" = 1 ]; then if rsync -rltD --delete --no-perms --no-owner --no-group --modify-window=2 \ + --exclude='.git' \ "$path"/ "$DEST/data/$slug"/ 2>>"$CKIS_LOG_FILE"; then - ckis::info "$slug: mirrored" + ckis::info "$slug: mirrored (working tree, no .git)" else ckis::warn "$slug: rsync mirror had errors (bundle is authoritative)" fi diff --git a/backup-system/bin/ckis-push.sh b/backup-system/bin/ckis-push.sh index 025b8c0..926e170 100755 --- a/backup-system/bin/ckis-push.sh +++ b/backup-system/bin/ckis-push.sh @@ -29,20 +29,33 @@ _push_one() { local slug; slug="$(basename "$dir")" if ckis::repo_dirty "$dir"; then - local big; big="$(ckis::max_staged_mb "$dir")" git -C "$dir" add -A - big="$(ckis::max_staged_mb "$dir")" - local guard; guard="$(ckis::manifest '.physical.size_guard_mb // 25')" + local big guard; big="$(ckis::max_staged_mb "$dir")" + guard="$(ckis::manifest '.physical.size_guard_mb // 25')" if [ "${big:-0}" -ge "${guard:-25}" ]; then ckis::warn "$slug: staging a ${big}MB blob (>= ${guard}MB guard) โ€” backing up anyway" fi - [ -n "$msg" ] || msg="ckis-backup: auto $(date -u +%Y-%m-%dT%H:%M:%SZ) on $(hostname -s 2>/dev/null || echo host)" - git -C "$dir" commit -q -m "$msg" || { ckis::warn "$slug: nothing to commit after add"; } - ckis::info "$slug: committed local changes" + # CRITICAL: distinguish "nothing actually staged" (benign no-op) from + # "commit aborted by a pre-commit hook" (HARD FAILURE). The old code ran + # `git commit || warn` then unconditionally logged success, so a secret-scan + # block silently stalled the vault for ~24h while reporting "up to date". + if git -C "$dir" diff --cached --quiet; then + ckis::info "$slug: nothing staged after add (no-op)" + else + [ -n "$msg" ] || msg="ckis-backup: auto $(date -u +%Y-%m-%dT%H:%M:%SZ) on $(hostname -s 2>/dev/null || echo host)" + if git -C "$dir" commit -q -m "$msg"; then + ckis::info "$slug: committed local changes" + else + ckis::err "$slug: COMMIT BLOCKED by pre-commit hook (e.g. secret-scan) โ€” backup FAILED; staged changes remain uncommitted" + ckis::mark_fail "$slug" "commit blocked by pre-commit hook (secret-scan?)" + return 1 + fi + fi fi if ! ckis::repo_has_remote "$dir"; then ckis::warn "$slug: no remote configured โ€” committed locally, not pushed" + ckis::clear_fail "$slug" return 0 fi @@ -51,14 +64,16 @@ _push_one() { if git -C "$dir" rev-parse --abbrev-ref --symbolic-full-name '@{u}' >/dev/null 2>&1; then local ahead; ahead="$(ckis::repo_ahead "$dir")" if [ "${ahead:-0}" = "0" ]; then - ckis::info "$slug: up to date, nothing to push"; return 0 + ckis::info "$slug: up to date, nothing to push"; ckis::clear_fail "$slug"; return 0 fi fi local branch; branch="$(git -C "$dir" symbolic-ref --short HEAD 2>/dev/null || echo HEAD)" if ckis::retry 3 git -C "$dir" push -u origin "$branch"; then ckis::info "$slug: pushed ($branch)" + ckis::clear_fail "$slug" else ckis::err "$slug: push failed after retries โ€” local commit safe, will retry next run" + ckis::mark_fail "$slug" "push failed after retries" return 1 fi } diff --git a/backup-system/bin/ckis-restore.sh b/backup-system/bin/ckis-restore.sh index 325f3ee..0b3aed0 100755 --- a/backup-system/bin/ckis-restore.sh +++ b/backup-system/bin/ckis-restore.sh @@ -32,6 +32,25 @@ while IFS=$'\t' read -r slug path remote class kind; do fi done < <(ckis::targets) +# 1.5 Restore the centralized brains aggregator into its workdir, so every +# project's .brain/ is available immediately on the fresh machine. Non-fatal: +# the next backup-all rebuilds it from the live projects anyway. +agg_remote="$(ckis::manifest '.discovery.brain_aggregator // empty')" +agg_dir="$(ckis::expand "$(ckis::manifest '.discovery.aggregate_workdir // empty')")" +if [ -n "$agg_remote" ] && [ "$agg_remote" != "null" ] && [ -n "$agg_dir" ] && [ "$agg_dir" != "null" ]; then + if [ -d "$agg_dir/.git" ]; then + ckis::info "aggregator: already present, skip" + else + mkdir -p "$(dirname "$agg_dir")" + agg_url="${REMOTE_BASE}${agg_remote}.git" + if ckis::retry 3 git clone "$agg_url" "$agg_dir" >>"$CKIS_LOG_FILE" 2>&1; then + ckis::info "aggregator: cloned centralized brains from $agg_url" + else + ckis::warn "aggregator: clone failed ($agg_url) โ€” next backup-all will rebuild it" + fi + fi +fi + # 2. Restore L0 apparatus into ~/.claude (merge; never deletes user data). if [ "$NO_APP" -eq 0 ] && [ -d "$CKIS_INFRA_ROOT/apparatus" ]; then mkdir -p "$CLAUDE_HOME" diff --git a/backup-system/bin/ckis-secret-audit.sh b/backup-system/bin/ckis-secret-audit.sh new file mode 100755 index 0000000..0e74cfe --- /dev/null +++ b/backup-system/bin/ckis-secret-audit.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# ckis-secret-audit.sh โ€” whole-system secret audit. +# Scans the working tree (tracked files) AND .git/config of every backup target +# and every registered project for REAL secret material, using the strongest-mode +# scanner. Catches what a staged-content pre-commit hook structurally cannot: +# tokens already living in a repo, or embedded in a remote URL in .git/config +# (the blind spot that hid a live PAT). Per-repo .ckis-secret-allow is honored. +# +# Exit 0 = clean, 1 = secrets found. Safe to run any time; read-only. +set -uo pipefail +HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" +source "$HERE/../lib/common.sh" +ckis::init +SCAN="$HERE/../hooks/pre-commit-secret-scan.sh" + +_audit_repo() { # path label + local path="$1" label="$2" rc=0 + [ -d "$path" ] || return 0 + ckis::is_repo "$path" || return 0 + # tracked working-tree files (node_modules etc. excluded by being gitignored) + if ! git -C "$path" ls-files -z 2>/dev/null \ + | ( cd "$path" && xargs -0 -r bash "$SCAN" --files ); then rc=1; fi + # .git/config embedded credentials (remote URLs) + if ! bash "$SCAN" --git-config "$path"; then rc=1; fi + [ "$rc" -eq 0 ] && ckis::info "audit โœ… $label" || ckis::err "audit ๐Ÿ”ด $label โ€” secret material found" + return "$rc" +} + +overall=0 +seen=" " + +# 1. manifest targets +while IFS=$'\t' read -r slug path remote class kind; do + _audit_repo "$path" "$slug" || overall=1 + seen="$seen$path " +done < <(ckis::targets) + +# 2. every registered project (private repos included) +reg="$(ckis::expand "$(ckis::manifest '.discovery.registry // empty')")" +if [ -f "$reg" ]; then + while IFS=$'\t' read -r slug repo; do + [ -n "$repo" ] || continue + case "$seen" in *" $repo "*) continue;; esac # de-dupe vs targets + _audit_repo "$repo" "$slug" || overall=1 + seen="$seen$repo " + done < <(jq -r '.projects[] | [.slug, .repo_root] | @tsv' "$reg" 2>/dev/null) +fi + +if [ "$overall" -eq 0 ]; then + ckis::info "secret audit: CLEAN across all repos" +else + ckis::err "secret audit: SECRET MATERIAL FOUND โ€” see lines above" +fi +exit "$overall" diff --git a/backup-system/bin/cli-brains-sync.sh b/backup-system/bin/cli-brains-sync.sh deleted file mode 100755 index 9abf490..0000000 --- a/backup-system/bin/cli-brains-sync.sh +++ /dev/null @@ -1,66 +0,0 @@ -#!/usr/bin/env bash -# cli-brains-sync.sh -# Aggregate the gitignored .brain/ dirs of PUBLIC code repos into a private -# workdir (later pushed to the aggregator repo). Private repos already back up -# their .brain/ via their own remote, so they are skipped. -# -# Discovery is registry-driven (a registry (projects.json)) โ€” scales automatically -# as new projects register. Visibility comes from each repo's real git remote. -set -uo pipefail -HERE="$(cd "$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")" && pwd)" -source "$HERE/../lib/common.sh" -ckis::init - -REGISTRY="${CKIS_REGISTRY:-$(ckis::expand "$(ckis::manifest '.discovery.registry')")}" -WORKDIR="${CKIS_AGG_WORKDIR:-$(ckis::expand "$(ckis::manifest '.discovery.aggregate_workdir')")}" -[ -f "$REGISTRY" ] || { ckis::warn "registry not found: $REGISTRY"; exit 0; } -mkdir -p "$WORKDIR" - -# Regenerable/redundant content is excluded from the aggregate (graph/ is -# graphify output; .brain-backup-* are redundant self-snapshots). Keeps the -# aggregator to high-value tactical memory (sessions, decisions, bugs). -EXC=() -while IFS= read -r e; do [ -n "$e" ] && EXC+=(--exclude "$e"); done \ - < <(ckis::manifest '.discovery.aggregate_exclude[]?' 2>/dev/null) - -# Resolve owner/name from a repo's origin remote URL. -_owner_name() { - local url name rest owner - url="$(git -C "$1" remote get-url origin 2>/dev/null)" || return 1 - url="${url%.git}" - name="${url##*/}" - rest="${url%/*}" - owner="${rest##*[:/]}" - [ -n "$owner" ] && [ -n "$name" ] && printf '%s/%s' "$owner" "$name" -} - -# Visibility: override hook for tests, else gh. -_visibility() { - if [ -n "${CKIS_VIS_CMD:-}" ]; then eval "$CKIS_VIS_CMD \"$1\""; return; fi - gh repo view "$1" --json visibility -q .visibility 2>/dev/null -} - -synced=0 -while IFS=$'\t' read -r slug repo; do - [ -n "$repo" ] || continue - brain="$repo/.brain" - [ -d "$brain" ] || continue - on="$(_owner_name "$repo")" || { ckis::warn "$slug: no origin remote, skipping"; continue; } - vis="$(_visibility "$on")" - case "$vis" in - PUBLIC|public) - mkdir -p "$WORKDIR/$slug" - if rsync -a --delete "${EXC[@]}" "$brain"/ "$WORKDIR/$slug/.brain"/ 2>>"$CKIS_LOG_FILE"; then - ckis::info "$slug: public .brain aggregated"; synced=$((synced+1)) - else - ckis::err "$slug: rsync failed" - fi ;; - PRIVATE|private) - ckis::info "$slug: private repo, .brain backed up via own remote โ€” skip" ;; - *) - ckis::warn "$slug: unknown visibility ('$vis'), skipping for safety" ;; - esac -done < <(jq -r '.projects[] | [.slug, .repo_root] | @tsv' "$REGISTRY") - -ckis::info "aggregator: $synced public .brain dir(s) synced into $WORKDIR" -exit 0 diff --git a/backup-system/ckis-manifest.example.json b/backup-system/ckis-manifest.example.json index 666b31b..06447f9 100644 --- a/backup-system/ckis-manifest.example.json +++ b/backup-system/ckis-manifest.example.json @@ -11,11 +11,28 @@ ], "discovery": { - "_doc": "OPTIONAL: aggregate gitignored subdirs (e.g. per-project .brain/) of PUBLIC repos into one private repo. Private repos back up via their own remote and are skipped. Registry is any json with .projects[].{slug, repo_root}.", + "_doc": "OPTIONAL: CENTRALIZE per-project memory subdirs (e.g. .brain/) of EVERY project into one private repo, regardless of the tool that created them or each project's own visibility. Discovery is the UNION of a registry (any json with .projects[].{slug, repo_root}) and a bounded filesystem scan of brain_roots โ€” so a project made by any tool is found even before it registers.", "registry": "~/path/to/projects.json", - "public_brain_aggregator": "YOUR_GITHUB_USERNAME/private-aggregator", - "aggregate_workdir": "~/.local/state/ckis-backup/aggregator", - "aggregate_exclude": ["graph/", "node_modules/", ".git/"] + "brain_aggregator": "YOUR_GITHUB_USERNAME/brains-backup", + "aggregate_workdir": "~/.local/state/ckis-backup/brains-backup", + "aggregate_exclude": ["graph/", ".brain-backup-*", "node_modules/", ".git/", "*.html"], + "brain_roots": ["~/code", "~/Documents"], + "brain_scan_maxdepth": 5, + "brain_scan_prune": ["node_modules", ".git", ".next", "dist", "build", ".venv", "__pycache__"] + }, + + "schedule": { + "_doc": "Cadence of the reconciling safety-net timer (the agent-agnostic real-time floor; no inotify dependency). reconcile_interval is templated into the systemd unit by install.sh.", + "reconcile_interval": "15min", + "boot_delay": "3min" + }, + + "secret_scan": { + "_doc": "Strongest-mode scanner: detects REAL key material (Shannon entropy + key-body proximity), not mere documentation mentions of a marker. Also covers .git/config remote URLs and class=secret filenames. allowlist_file (per repo) sanctions paths that legitimately quote patterns.", + "entropy_threshold": 3.0, + "allowlist_file": ".ckis-secret-allow", + "scan_git_config": true, + "block_secret_filenames": true }, "apparatus": { diff --git a/backup-system/hooks/pre-commit-secret-scan.sh b/backup-system/hooks/pre-commit-secret-scan.sh index d08ffad..60bd7b5 100755 --- a/backup-system/hooks/pre-commit-secret-scan.sh +++ b/backup-system/hooks/pre-commit-secret-scan.sh @@ -1,39 +1,164 @@ #!/usr/bin/env bash -# pre-commit-secret-scan.sh โ€” block commits that introduce secrets. -# Dependency-free regex scanner (no gitleaks needed). Install as a repo's -# .git/hooks/pre-commit (or call directly to scan staged changes). +# pre-commit-secret-scan.sh โ€” strongest-mode, dependency-free secret scanner. # -# Exit 0 = clean, exit 1 = secret found (commit blocked). +# Blocks commits/files that introduce REAL secret material, while NOT flagging +# documentation that merely *mentions* a marker โ€” the false-positive that +# silently stalled the CKIS vault backup for ~24h (a compact note that quoted +# "-----BEGIN PRIVATE KEY-----" jammed every commit). Self-contained (no lib +# dependency) so it works as a hook in any repo, on any machine. +# +# Detection (high-signal, low false-positive): +# * token patterns (ghp_/gho_/github_pat/sk-/AKIA/AIza/xox/glpat) gated by +# Shannon entropy >= threshold โ†’ a real token, not "ghp_xxxx" / prose +# * PEM private keys: BEGIN marker AND an actual base64 key-body line present +# (a bare marker quoted in prose has no body โ†’ not flagged) +# * URL-embedded credentials (https://user:SECRET@host), entropy-gated โ€” the +# blind spot that hid a live PAT in a repo's .git/config remote URL +# * class=secret FILENAMES (.env, *.pem, *.key, .credentials.json, id_rsa*) +# +# Escape valve (docs/tests that legitimately quote patterns): +# * repo-root allowlist file (.ckis-secret-allow): one path-glob per line +# * inline marker anywhere on the offending line: "ckis-allow-secret" +# +# Exit 0 = clean, 1 = real secret found (commit blocked). # Modes: -# (no args) scan staged diff (git diff --cached) โ€” hook mode -# --files F... scan the given files โ€” standalone/test mode +# (no args) hook mode: staged diff content + staged filenames +# --files F... scan specific files (standalone/test) +# --git-config REPO scan REPO/.git/config for embedded credentials set -uo pipefail -# High-signal secret patterns. Conservative to avoid false positives on notes. -PATTERNS='(ghp_[A-Za-z0-9]{36})|(gho_[A-Za-z0-9]{36})|(github_pat_[A-Za-z0-9_]{40,})|(sk-[A-Za-z0-9]{32,})|(AKIA[0-9A-Z]{16})|(AIza[0-9A-Za-z_-]{35})|(xox[baprs]-[A-Za-z0-9-]{10,})|(-----BEGIN ([A-Z]+ )?PRIVATE KEY-----)|(glpat-[A-Za-z0-9_-]{20})' +ENTROPY_MIN="${CKIS_SECRET_ENTROPY:-3.0}" +ALLOW_FILE_NAME="${CKIS_SECRET_ALLOWFILE:-.ckis-secret-allow}" -scan_stream() { grep -nIE "$PATTERNS"; } # returns 0 if a match is found +TOKEN_RE='(ghp_[A-Za-z0-9]{36})|(gho_[A-Za-z0-9]{36})|(github_pat_[A-Za-z0-9_]{40,})|(sk-[A-Za-z0-9]{32,})|(AKIA[0-9A-Z]{16})|(AIza[0-9A-Za-z_-]{35})|(xox[baprs]-[A-Za-z0-9-]{10,})|(glpat-[A-Za-z0-9_-]{20})' +URLCRED_RE='://[^/:@[:space:]]+:[^/@[:space:]]+@' +PEM_BEGIN_RE='-----BEGIN ([A-Z0-9 ]+ )?PRIVATE KEY-----' +PEM_BODY_RE='^[A-Za-z0-9+/]{40,}={0,2}$' +SECRET_FILE_RE='(^|/)(\.env(\..+)?|.+\.pem|.+\.key|\.credentials\.json|id_rsa.*|id_ed25519.*)$' found=0 -report() { echo "๐Ÿ”ด SECRET BLOCK: potential secret in $1"; echo " $2"; found=1; } +_report() { echo "๐Ÿ”ด SECRET BLOCK: $1"; echo " $2"; found=1; } -if [ "${1:-}" = "--files" ]; then - shift - for f in "$@"; do - [ -f "$f" ] || continue - while IFS= read -r hit; do report "$f" "$hit"; done < <(scan_stream <"$f") +# Shannon entropy (bits/char) of a string; 0 for empty. Pure awk. +_entropy() { + printf '%s\n' "${1:-}" | awk 'NR==1{n=length($0); if(n==0){print "0";exit} + for(i=1;i<=n;i++){c=substr($0,i,1);f[c]++} + H=0; for(c in f){p=f[c]/n; H-=p*log(p)/log(2)}; printf "%.2f",H}' +} +_ge() { awk -v a="$1" -v b="$2" 'BEGIN{exit !(a>=b)}'; } # float a>=b + +# inline allowlist marker on a single line +_allowed_line() { case "$1" in *ckis-allow-secret*) return 0;; *) return 1;; esac; } + +# repo-root path-glob allowlist +ALLOW_GLOBS=() +_load_allowlist() { # root + ALLOW_GLOBS=() + local af="$1/$ALLOW_FILE_NAME" line + [ -f "$af" ] || return 0 + while IFS= read -r line; do + line="${line%%#*}"; line="${line#"${line%%[![:space:]]*}"}"; line="${line%"${line##*[![:space:]]}"}" + [ -n "$line" ] && ALLOW_GLOBS+=("$line") + done <"$af" +} +_path_allowed() { # relpath + local p="$1" g + for g in "${ALLOW_GLOBS[@]:-}"; do + [ -n "$g" ] || continue + # shellcheck disable=SC2053 + [[ "$p" == $g ]] && return 0 done -else - # Hook mode: scan the staged content of added/modified files. - while IFS= read -r f; do - [ -n "$f" ] || continue - while IFS= read -r hit; do report "$f" "$hit"; done \ - < <(git show ":$f" 2>/dev/null | scan_stream) - done < <(git diff --cached --name-only --diff-filter=AM 2>/dev/null) -fi + return 1 +} + +# class=secret filename โ€” but NOT documented templates (.env.example etc.), +# which are committed on purpose and hold only placeholders. Content scanning +# still applies to them, so a real key pasted into a template is still caught. +_filename_secret() { + case "$(basename "$1")" in + *.example|*.sample|*.template|*.dist|*.tmpl) return 1 ;; + esac + printf '%s' "$1" | grep -qE "$SECRET_FILE_RE" +} + +# Content scan of a single file. label is what we print (repo-relative path). +_scan_file() { + local f="$1" label="$2" + [ -f "$f" ] || return 0 + + # Pre-filter to candidate lines with grep (fast C), then entropy-check in bash. + local cand cl lineno text tok e x secret + cand="$(grep -nIE "$TOKEN_RE|$URLCRED_RE" "$f" 2>/dev/null || true)" + if [ -n "$cand" ]; then + while IFS= read -r cl; do + lineno="${cl%%:*}"; text="${cl#*:}" + _allowed_line "$text" && continue + # entropy-gated tokens + while IFS= read -r tok; do + [ -n "$tok" ] || continue + e="$(_entropy "$tok")" + _ge "$e" "$ENTROPY_MIN" && _report "$label:$lineno" "high-entropy token ${tok:0:6}โ€ฆ (H=$e)" + done < <(printf '%s\n' "$text" | grep -oE "$TOKEN_RE" 2>/dev/null || true) + # entropy-gated URL-embedded credential + while IFS= read -r tok; do + [ -n "$tok" ] || continue + x="${tok%@}"; secret="${x##*:}" + e="$(_entropy "$secret")" + _ge "$e" "$ENTROPY_MIN" && _report "$label:$lineno" "credential embedded in URL (H=$e)" + done < <(printf '%s\n' "$text" | grep -oE "$URLCRED_RE" 2>/dev/null || true) + done <<<"$cand" + fi + + # PEM private key: a BEGIN marker IMMEDIATELY followed (within 5 lines) by an + # actual base64 body line. Proximity matters โ€” a long note may quote the marker + # in prose AND, far away, contain an unrelated base64-looking line; that must + # NOT be flagged. The inline marker on the BEGIN line suppresses (docs/tests). + local pemline + pemline="$(awk -v pat="$PEM_BEGIN_RE" -v body="$PEM_BODY_RE" ' + $0 ~ pat { if ($0 ~ /ckis-allow-secret/) { begin=0 } else { begin=NR }; next } + begin && NR<=begin+5 && $0 ~ body { print begin; exit } + ' "$f" 2>/dev/null || true)" + [ -n "$pemline" ] && _report "$label:$pemline" "PEM private key block (marker + base64 body within 5 lines)" +} + +case "${1:-}" in + --files) + shift + root="$(git rev-parse --show-toplevel 2>/dev/null || echo .)" + _load_allowlist "$root" + for f in "$@"; do + [ -f "$f" ] || continue + rel="$f"; case "$f" in "$root"/*) rel="${f#"$root"/}";; esac + _path_allowed "$rel" && continue + _filename_secret "$f" && _report "$f" "class=secret filename" + _scan_file "$f" "$f" + done + ;; + --git-config) + repo="${2:-.}" + cfg="$repo/.git/config"; [ -f "$cfg" ] || cfg="$(git -C "$repo" rev-parse --git-dir 2>/dev/null)/config" + [ -f "$cfg" ] && _scan_file "$cfg" "$repo/.git/config" + ;; + *) + # Hook mode: scan staged ADDED/MODIFIED content + staged filenames. + root="$(git rev-parse --show-toplevel 2>/dev/null || echo .)" + _load_allowlist "$root" + tmp="$(mktemp)"; trap 'rm -f "$tmp"' EXIT + while IFS= read -r f; do + [ -n "$f" ] || continue + _path_allowed "$f" && continue + _filename_secret "$f" && _report "$f" "class=secret filename staged" + if git show ":$f" >"$tmp" 2>/dev/null; then + _scan_file "$tmp" "$f" + fi + done < <(git diff --cached --name-only --diff-filter=AM 2>/dev/null) + ;; +esac if [ "$found" -ne 0 ]; then - echo " Commit aborted. Remove the secret or move it to a class=secret path (physical-disk only)." >&2 + echo " Commit aborted. Remove the secret, or โ€” if this is documentation/tests" >&2 + echo " that legitimately quotes a pattern โ€” allowlist the path in $ALLOW_FILE_NAME" >&2 + echo " or add the inline marker 'ckis-allow-secret' on the line." >&2 exit 1 fi exit 0 diff --git a/backup-system/install.sh b/backup-system/install.sh index 2da7150..2cb2ff1 100755 --- a/backup-system/install.sh +++ b/backup-system/install.sh @@ -16,17 +16,31 @@ mkdir -p "$BIN_DIR" for s in "$HERE"/bin/*.sh; do ln -sf "$s" "$BIN_DIR/$(basename "$s" .sh)" done +# Prune our own dangling symlinks (scripts renamed/removed, e.g. cli-brains-sync +# -> brains-sync) so a rename never leaves a dead command behind. Scoped to links +# that point into THIS repo's bin/, so unrelated user symlinks are never touched. +for l in "$BIN_DIR"/*; do + [ -L "$l" ] || continue + tgt="$(readlink "$l")" + case "$tgt" in "$HERE"/bin/*) [ -e "$l" ] || { rm -f "$l"; ckis::info "pruned dangling link $(basename "$l")"; } ;; esac +done ckis::info "linked $(ls "$HERE"/bin/*.sh | wc -l) scripts into $BIN_DIR" -# 2. systemd --user timer (daily safety net) +# 2. systemd --user timer (reconciling safety net; cadence from the manifest) if [ "${CKIS_NO_SYSTEMD:-0}" != "1" ] && command -v systemctl >/dev/null 2>&1; then mkdir -p "$SYSTEMD_DIR" + INTERVAL="$(ckis::manifest '.schedule.reconcile_interval // "15min"')" + BOOTDELAY="$(ckis::manifest '.schedule.boot_delay // "3min"')" sed "s|@BIN@|$BIN_DIR|g" "$HERE/systemd/ckis-backup.service" >"$SYSTEMD_DIR/ckis-backup.service" - cp "$HERE/systemd/ckis-backup.timer" "$SYSTEMD_DIR/ckis-backup.timer" + sed -e "s|@INTERVAL@|$INTERVAL|g" -e "s|@BOOTDELAY@|$BOOTDELAY|g" \ + "$HERE/systemd/ckis-backup.timer" >"$SYSTEMD_DIR/ckis-backup.timer" systemctl --user daemon-reload 2>/dev/null || true - systemctl --user enable --now ckis-backup.timer 2>/dev/null \ - && ckis::info "systemd timer enabled (daily)" \ - || ckis::warn "could not enable systemd timer (headless? run: systemctl --user enable --now ckis-backup.timer)" + if systemctl --user enable --now ckis-backup.timer 2>/dev/null; then + systemctl --user restart ckis-backup.timer 2>/dev/null || true # apply new cadence + ckis::info "systemd timer enabled (reconcile every $INTERVAL)" + else + ckis::warn "could not enable systemd timer (headless? run: systemctl --user enable --now ckis-backup.timer)" + fi fi # 3. secret-scan pre-commit into each target repo (don't clobber existing hooks) diff --git a/backup-system/lib/common.sh b/backup-system/lib/common.sh index ba05470..d72ca6a 100644 --- a/backup-system/lib/common.sh +++ b/backup-system/lib/common.sh @@ -121,3 +121,90 @@ ckis::max_staged_mb() { done < <(git -C "$dir" diff --cached --name-only --diff-filter=A 2>/dev/null) printf '%s' "$max" } + +# โ”€โ”€ Entropy โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Shannon entropy in bits/char of a string (0.00 for empty). Pure awk. Used by +# the secret scanner to tell a real high-entropy secret from a low-entropy +# placeholder ("ghp_xxxxโ€ฆ") or a documentation token, killing false positives. +ckis::entropy() { + printf '%s\n' "${1:-}" | awk ' + NR==1 { + n=length($0) + if(n==0){printf "0.00"; exit} + for(i=1;i<=n;i++){c=substr($0,i,1); f[c]++} + H=0 + for(c in f){p=f[c]/n; H-=p*log(p)/log(2)} + printf "%.2f", H + }' +} + +# โ”€โ”€ Brain discovery (agent-agnostic) โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Emit "slugrepo_path" for every project that has a .brain/ dir โ€” the UNION +# of the Dev Brain registry AND a bounded filesystem scan of discovery.brain_roots. +# The scan makes discovery agent-agnostic: a project created by ANY coding agent +# (Codex, Gemini, OpenCode, ...) is found even if it never ran a registration +# hook. De-duplicated by repo path; registry entries win the slug. +ckis::brain_repos() { + local reg seen="|" slug repo root brain maxdepth roots_src + reg="${CKIS_REGISTRY:-$(ckis::expand "$(ckis::manifest '.discovery.registry // empty')")}" + + # 1. registry (fast path) + if [ -f "$reg" ]; then + while IFS=$'\t' read -r slug repo; do + [ -n "$repo" ] || continue + repo="$(ckis::expand "$repo")" + [ -d "$repo/.brain" ] || continue + case "$seen" in *"|$repo|"*) continue;; esac + seen="$seen$repo|" + printf '%s\t%s\n' "$slug" "$repo" + done < <(jq -r '.projects[] | [.slug, .repo_root] | @tsv' "$reg" 2>/dev/null) + fi + + # 2. filesystem scan of brain_roots (agent-agnostic safety net) + maxdepth="$(ckis::manifest '.discovery.brain_scan_maxdepth // 5')" + local prune=() nm first=1 + prune+=( '(' ) + while IFS= read -r nm; do + [ -n "$nm" ] || continue + [ "$first" = 1 ] && first=0 || prune+=( -o ) + prune+=( -name "$nm" ) + done < <(ckis::manifest '.discovery.brain_scan_prune[]?' 2>/dev/null) + prune+=( ')' -prune ) + + # roots: env override (newline-separated; empty disables the scan) or manifest + if [ "${CKIS_BRAIN_ROOTS+x}" = "x" ]; then + roots_src="$CKIS_BRAIN_ROOTS" + else + roots_src="$(ckis::manifest '.discovery.brain_roots[]?' 2>/dev/null)" + fi + while IFS= read -r root; do + [ -n "$root" ] || continue + root="$(ckis::expand "$root")" + [ -d "$root" ] || continue + while IFS= read -r brain; do + repo="$(dirname "$brain")" + ckis::is_repo "$repo" || continue + case "$seen" in *"|$repo|"*) continue;; esac + seen="$seen$repo|" + printf '%s\t%s\n' "$(basename "$repo")" "$repo" + done < <( [ "$first" = 0 ] \ + && find "$root" -maxdepth "$maxdepth" "${prune[@]}" -o -type d -name .brain -print 2>/dev/null \ + || find "$root" -maxdepth "$maxdepth" -type d -name .brain -print 2>/dev/null ) + done <<< "$roots_src" +} + +# โ”€โ”€ Hard-failure markers โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€ +# Persistent record that a target FAILED to back up (e.g. commit blocked by the +# secret scanner, or push failing). The doctor reads these to show ๐Ÿ”ด FAILED +# (distinct from benign drift) and backup-all exits non-zero when any is set. +# Keyed by slug (basename of the repo dir, matching the manifest target slugs). +ckis::_fail_dir() { printf '%s' "$CKIS_LOG_DIR/failures"; } +ckis::mark_fail() { # slug reason... + local slug="$1"; shift + mkdir -p "$(ckis::_fail_dir)" 2>/dev/null || true + printf '%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$*" >"$(ckis::_fail_dir)/$slug" 2>/dev/null || true +} +ckis::clear_fail() { rm -f "$(ckis::_fail_dir)/$1" 2>/dev/null || true; } +ckis::is_failed() { [ -f "$(ckis::_fail_dir)/$1" ]; } +ckis::fail_reason() { cut -f2- "$(ckis::_fail_dir)/$1" 2>/dev/null || true; } +ckis::list_fails() { ls -1 "$(ckis::_fail_dir)" 2>/dev/null || true; } diff --git a/backup-system/systemd/ckis-backup.timer b/backup-system/systemd/ckis-backup.timer index 0168038..dc36425 100644 --- a/backup-system/systemd/ckis-backup.timer +++ b/backup-system/systemd/ckis-backup.timer @@ -1,12 +1,16 @@ [Unit] -Description=CKIS autonomous backup โ€” daily safety net +Description=CKIS autonomous backup โ€” reconciling safety net Documentation=https://github.com/aedneth/ckis [Timer] -# Daily, ~30 min after login if missed, with jitter to avoid thundering herds. -OnCalendar=daily +# Frequent reconcile = the agent-agnostic real-time floor (no inotify dependency, +# survives any coding agent / OS / hardware). Interval + boot delay are templated +# from ckis-manifest.json (.schedule) by install.sh. Persistent catches missed +# runs after sleep/suspend; jitter avoids thundering herds. +OnBootSec=@BOOTDELAY@ +OnUnitActiveSec=@INTERVAL@ Persistent=true -RandomizedDelaySec=900 +RandomizedDelaySec=60 [Install] WantedBy=timers.target diff --git a/backup-system/tests/test_backup_all.sh b/backup-system/tests/test_backup_all.sh index 29af154..d6bd642 100755 --- a/backup-system/tests/test_backup_all.sh +++ b/backup-system/tests/test_backup_all.sh @@ -36,5 +36,24 @@ assert_eq "t1 still 1 commit" "1" "$(git --git-dir="$SB/t1.git" log --oneline -- # health line present in log assert_contains "health summary logged" "$(cat "$CKIS_LOG_DIR/backup.log")" "BACKUP" +# deep secret audit ran (throttled, universal net) and recorded its marker +assert_file "deep secret-audit marker written" "$CKIS_LOG_DIR/last-audit" + +# HARD-FAIL: a target whose pre-commit hook blocks the commit must make backup-all +# exit NON-ZERO and record a FAILED marker โ€” never the old silent success. +git init -q --bare "$SB/b.git" +git clone -q "$SB/b.git" "$SB/blocked" 2>/dev/null +git -C "$SB/blocked" config user.email a@a; git -C "$SB/blocked" config user.name a +printf '#!/bin/sh\nexit 1\n' >"$SB/blocked/.git/hooks/pre-commit"; chmod +x "$SB/blocked/.git/hooks/pre-commit" +echo data >"$SB/blocked/x.md" +cat >"$SB/mb.json" <"$1/.brain/sessions/s1.md" + echo "decision" >"$1/.brain/decisions.md" + mkdir -p "$1/.brain/graph"; echo '{"big":1}' >"$1/.brain/graph/graph.json" # regenerable + mkdir -p "$1/.brain/.brain-backup-x"; echo "redundant" >"$1/.brain/.brain-backup-x/y" +} +mkrepo "$SB/pub" "aedneth/pub-cli" +mkrepo "$SB/priv" "aedneth/priv-app" + +cat >"$SB/reg.json" <"$SB/reg.json" <"$1/.brain/sessions/s1.md" - echo "decision" >"$1/.brain/decisions.md" - mkdir -p "$1/.brain/graph"; echo '{"big":1}' >"$1/.brain/graph/graph.json" # regenerable -} -mkrepo "$SB/pub" "aedneth/pub-cli" -mkrepo "$SB/priv" "aedneth/priv-app" - -cat >"$SB/reg.json" <"$SB/reg.json" <"$CKIS_LOG_DIR/failures/t1" +out5="$(bash "$DOC" --oneline)" +assert_contains "failed marker -> ๐Ÿ”ด FAILED" "$out5" "FAILED" +assert_contains "FAILED names the target" "$out5" "t1" +# brain-repo / .gitcfg markers also surface even if not a manifest target +printf '%s\tembedded credential in .git/config\n' "$(date -u +%FT%TZ)" >"$CKIS_LOG_DIR/failures/someproject.gitcfg" +out6="$(bash "$DOC" --oneline)" +assert_contains "non-target failure marker surfaces" "$out6" "someproject.gitcfg" +rm -rf "$CKIS_LOG_DIR/failures" + assert_summary diff --git a/backup-system/tests/test_install.sh b/backup-system/tests/test_install.sh index 6e006bc..a1acc43 100755 --- a/backup-system/tests/test_install.sh +++ b/backup-system/tests/test_install.sh @@ -30,4 +30,11 @@ assert_ok "linked script is runnable" bash "$SB/bin/ckis-backup-doctor" # idempotent assert_ok "second install idempotent" bash "$ROOT/install.sh" +# timer cadence is templated from the manifest (reconcile, not a fixed daily) +assert_contains "timer template has @INTERVAL@" "$(cat "$ROOT/systemd/ckis-backup.timer")" "@INTERVAL@" +interval="$(jq -r '.schedule.reconcile_interval' "$ROOT/ckis-manifest.example.json")" +rendered="$(sed -e "s|@INTERVAL@|$interval|g" -e "s|@BOOTDELAY@|3min|g" "$ROOT/systemd/ckis-backup.timer")" +assert_contains "rendered timer uses OnUnitActiveSec" "$rendered" "OnUnitActiveSec=$interval" +assert_eq "no @INTERVAL@ placeholder remains" "" "$(printf '%s' "$rendered" | grep -o '@INTERVAL@' || true)" + assert_summary diff --git a/backup-system/tests/test_lib.sh b/backup-system/tests/test_lib.sh index a04d8c4..fef7681 100755 --- a/backup-system/tests/test_lib.sh +++ b/backup-system/tests/test_lib.sh @@ -54,4 +54,35 @@ assert_ok "is_repo_root true at root" ckis::is_repo_root "$R" assert_fail "is_repo_root false in subdir" ckis::is_repo_root "$R/sub/deep" assert_ok "is_repo true in subdir" ckis::is_repo "$R/sub/deep" +# โ”€โ”€ Shannon entropy (bits/char): real secret vs placeholder/prose โ”€โ”€ +# (Use a high-entropy NON-token string so this fixture itself never trips the +# scanner โ€” the entropy fn is prefix-agnostic, which is exactly the point.) +hi="$(ckis::entropy 'Xk7Qm2Rp9Lw4Ez8Ty1Ui5Io3Nz6Vc0DfHbJg')" +lo="$(ckis::entropy 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx')" +assert_ok "entropy: high-entropy string > 3.0" bash -c "awk 'BEGIN{exit !($hi>3.0)}'" +assert_ok "entropy: placeholder < 1.0" bash -c "awk 'BEGIN{exit !($lo<1.0)}'" +assert_eq "entropy: empty -> 0.00" "0.00" "$(ckis::entropy '')" + +# โ”€โ”€ hard-failure markers (drive doctor's FAILED state) โ”€โ”€ +ckis::clear_fail "demo" +assert_fail "is_failed false initially" ckis::is_failed "demo" +ckis::mark_fail "demo" "commit blocked by secret-scan" +assert_ok "is_failed true after mark" ckis::is_failed "demo" +assert_contains "fail_reason carries text" "$(ckis::fail_reason demo)" "secret-scan" +assert_contains "list_fails includes demo" "$(ckis::list_fails)" "demo" +ckis::clear_fail "demo" +assert_fail "is_failed false after clear" ckis::is_failed "demo" + +# โ”€โ”€ brain discovery: registry โˆช filesystem scan (agent-agnostic) โ”€โ”€ +BR="$SANDBOX/brains"; mkdir -p "$BR/proj-a/.brain" "$BR/proj-b/.brain" +git -C "$BR/proj-a" init -q; git -C "$BR/proj-b" init -q +cat >"$SANDBOX/reg.json" </dev/null)" +assert_ok "bundle still restores after .git exclude" git clone -q "$DEST/bundles/t1.bundle" "$SB/restored2" + # no dest + no mount -> non-zero assert_fail "no dest, no mount -> error" bash "$PHYS" diff --git a/backup-system/tests/test_push.sh b/backup-system/tests/test_push.sh index dea8b95..edb0ca0 100755 --- a/backup-system/tests/test_push.sh +++ b/backup-system/tests/test_push.sh @@ -45,4 +45,20 @@ mkdir -p "$SB/parent/child"; echo data >"$SB/parent/child/x.md" assert_ok "subdir-of-repo push is a safe no-op" bash "$PUSH" "$SB/parent/child" assert_eq "parent repo NOT committed" "0" "$(git -C "$SB/parent" log --oneline --all 2>/dev/null | wc -l | tr -d ' ')" +# HARD-FAIL: a pre-commit hook that ABORTS the commit (e.g. secret-scan block) must +# make ckis-push fail loudly and leave ZERO commits โ€” never silently report success. +# Regression guard for the 24h silent-vault-failure bug (commit '|| warn' then log "committed"). +git init -q "$SB/blocked"; git -C "$SB/blocked" config user.email t@t; git -C "$SB/blocked" config user.name t +printf '#!/bin/sh\necho "blocked by hook"; exit 1\n' >"$SB/blocked/.git/hooks/pre-commit" +chmod +x "$SB/blocked/.git/hooks/pre-commit" +echo data >"$SB/blocked/note.md" +assert_fail "commit aborted by pre-commit -> ckis-push fails hard" bash "$PUSH" "$SB/blocked" +assert_eq "blocked repo has ZERO commits" "0" "$(git -C "$SB/blocked" log --oneline 2>/dev/null | wc -l | tr -d ' ')" + +# Benign: a dirty repo whose changes are all already committed-equivalent (add stages nothing +# new) is a clean no-op, exit 0 โ€” must NOT be conflated with a hard failure. +assert_ok "blocked repo after removing the hook commits cleanly" bash -c ' + rm -f "$1/.git/hooks/pre-commit"; exec bash "$2" "$1"' _ "$SB/blocked" "$PUSH" +assert_eq "now committed (1 commit)" "1" "$(git -C "$SB/blocked" log --oneline 2>/dev/null | wc -l | tr -d ' ')" + assert_summary diff --git a/backup-system/tests/test_restore.sh b/backup-system/tests/test_restore.sh index d884ebf..5967fd8 100755 --- a/backup-system/tests/test_restore.sh +++ b/backup-system/tests/test_restore.sh @@ -1,5 +1,8 @@ #!/usr/bin/env bash -# Contract for bin/ckis-restore.sh โ€” clones missing targets, safe & idempotent. +# Contract for bin/ckis-restore.sh โ€” the plug-and-play "prueba reina". +# Simulates a FRESH machine in a sandbox (isolated $HOME-like dirs, fake remotes) +# and proves a full rebuild: targets cloned + centralized brains cloned + +# apparatus restored + runtime installed (symlinks), all idempotent. set -uo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "$HERE/.." && pwd)" @@ -8,28 +11,48 @@ RES="$ROOT/bin/ckis-restore.sh" SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT export CKIS_LOG_DIR="$SB/state"; mkdir -p "$CKIS_LOG_DIR" -# seed a bare "remote" with content +# seed a bare "remote" for a target and for the centralized brains aggregator mkdir -p "$SB/remotes" -git init -q --bare "$SB/remotes/t1.git" -git clone -q "$SB/remotes/t1.git" "$SB/seed" 2>/dev/null -git -C "$SB/seed" config user.email a@a; git -C "$SB/seed" config user.name a -echo "restored knowledge" >"$SB/seed/note.md"; git -C "$SB/seed" add -A; git -C "$SB/seed" commit -q -m init -git -C "$SB/seed" push -q -u origin master 2>/dev/null || git -C "$SB/seed" push -q -u origin main 2>/dev/null +seed_remote() { # name file content + git init -q --bare "$SB/remotes/$1.git" + git clone -q "$SB/remotes/$1.git" "$SB/seed-$1" 2>/dev/null + git -C "$SB/seed-$1" config user.email a@a; git -C "$SB/seed-$1" config user.name a + mkdir -p "$(dirname "$SB/seed-$1/$2")" + echo "$3" >"$SB/seed-$1/$2"; git -C "$SB/seed-$1" add -A; git -C "$SB/seed-$1" commit -q -m init + git -C "$SB/seed-$1" push -q -u origin master 2>/dev/null || git -C "$SB/seed-$1" push -q -u origin main 2>/dev/null +} +seed_remote t1 note.md "restored knowledge" +seed_remote brains-backup my-project/.brain/decisions.md "centralized brain" cat >"$SB/m.json" </dev/null 2>&1 +rc=$? +assert_eq "restore exits 0" "0" "$rc" +assert_file "target cloned" "$SB/restored/t1/note.md" +assert_file "centralized brains cloned" "$SB/restored/brains/my-project/.brain/decisions.md" +# apparatus restore is optional โ€” only assert it if an apparatus/ ships with the repo +[ -f "$ROOT/apparatus/CLAUDE.md" ] && \ + assert_file "L0 apparatus restored (CLAUDE.md)" "$SB/home/CLAUDE.md" +assert_file "runtime installed (ckis-push link)" "$SB/bin/ckis-push" +assert_file "runtime installed (backup-all link)" "$SB/bin/ckis-backup-all" -# idempotent: re-run, target already present -> still ok, content intact -assert_ok "second restore idempotent" bash "$RES" --no-system --no-apparatus -assert_file "content still present" "$SB/restored/t1/note.md" +# โ”€โ”€ idempotent: a second restore leaves everything intact โ”€โ”€ +CKIS_NO_SYSTEMD=1 CKIS_BIN_DIR="$SB/bin" CKIS_SYSTEMD_DIR="$SB/systemd" \ + bash "$RES" >/dev/null 2>&1 +assert_eq "second restore idempotent" "0" "$?" +assert_file "target content still present" "$SB/restored/t1/note.md" +assert_file "brains still present" "$SB/restored/brains/my-project/.brain/decisions.md" assert_summary diff --git a/backup-system/tests/test_secret_audit.sh b/backup-system/tests/test_secret_audit.sh new file mode 100644 index 0000000..c6f75d7 --- /dev/null +++ b/backup-system/tests/test_secret_audit.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Contract for bin/ckis-secret-audit.sh โ€” whole-system secret audit. +set -uo pipefail +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$HERE/.." && pwd)" +source "$HERE/assert.sh" +AUDIT="$ROOT/bin/ckis-secret-audit.sh" +SB="$(mktemp -d)"; trap 'rm -rf "$SB"' EXIT +export CKIS_LOG_DIR="$SB/state" +rnd() { head -c 96 /dev/urandom | base64 2>/dev/null | tr -dc 'A-Za-z0-9' | head -c 36; } +mkrepo() { mkdir -p "$1"; git -C "$1" init -q; git -C "$1" config user.email t@t; git -C "$1" config user.name t; } + +mkrepo "$SB/clean"; echo "general notes about keys" >"$SB/clean/a.md" +git -C "$SB/clean" add -A; git -C "$SB/clean" commit -qm x + +mkrepo "$SB/dirty"; printf 'tok = ghp_%s\n' "$(rnd)" >"$SB/dirty/leak.md" +git -C "$SB/dirty" add -A; git -C "$SB/dirty" commit -qm leak + +cat >"$SB/manifest.json" <"$SB/projects.json" <"$SB/clean.md" -printf 'token = ghp_%s\n' "0123456789012345678901234567890123456789" >"$SB/bad.md" -assert_ok "clean file passes" bash "$HOOK" --files "$SB/clean.md" -assert_fail "github token blocks" bash "$HOOK" --files "$SB/bad.md" +# high-entropy 36-char token body (assembled at runtime; never a literal in this file) +rnd() { head -c 96 /dev/urandom | base64 2>/dev/null | tr -dc 'A-Za-z0-9' | head -c "${1:-36}"; } -printf -- '%s PRIVATE KEY-----\nabc\n' '-----BEGIN' >"$SB/pem.md" -assert_fail "private key blocks" bash "$HOOK" --files "$SB/pem.md" +# โ”€โ”€ clean passes โ”€โ”€ +printf 'just some notes about tokens and private keys in general\n' >"$SB/clean.md" +assert_ok "clean file passes" bash "$HOOK" --files "$SB/clean.md" -# Hook mode inside a real repo. +# โ”€โ”€ REAL token blocks (high entropy) โ”€โ”€ +printf 'token = ghp_%s\n' "$(rnd 36)" >"$SB/realtok.md" +assert_fail "real high-entropy ghp_ token blocks" bash "$HOOK" --files "$SB/realtok.md" +printf 'aws = AKIA%s\n' "ABCDEFGHIJKLMNOP" >"$SB/aws.md" +assert_fail "AWS AKIA key blocks" bash "$HOOK" --files "$SB/aws.md" + +# โ”€โ”€ PLACEHOLDER does NOT block (the entropy fix) โ”€โ”€ +ph="ghp_$(printf 'x%.0s' $(seq 1 36))" +printf 'example placeholder token = %s\n' "$ph" >"$SB/placeholder.md" +assert_ok "placeholder ghp_xxxxโ€ฆ passes (low entropy)" bash "$HOOK" --files "$SB/placeholder.md" + +# โ”€โ”€ PROSE mention of a PEM marker does NOT block (THE outage cause) โ”€โ”€ +printf 'The scanner matches the literal -----BEGIN PRIVATE KEY----- marker quoted in prose.\n' >"$SB/prose.md" +assert_ok "prose 'BEGIN PRIVATE KEY' (no body) passes" bash "$HOOK" --files "$SB/prose.md" + +# โ”€โ”€ REAL PEM block blocks (marker + base64 body on its own line) โ”€โ”€ +{ echo "-----BEGIN PRIVATE KEY-----"; echo "$(rnd 64)"; echo "-----END PRIVATE KEY-----"; } >"$SB/realpem.md" +assert_fail "real PEM key block blocks" bash "$HOOK" --files "$SB/realpem.md" + +# โ”€โ”€ URL-embedded credential: high entropy blocks, low entropy (doc) passes โ”€โ”€ +printf 'remote = https://user:%s@github.com/x/y.git\n' "$(rnd 36)" >"$SB/urlcred.md" +assert_fail "URL with high-entropy token blocks" bash "$HOOK" --files "$SB/urlcred.md" +printf 'example = https://user:pass@example.com/repo.git\n' >"$SB/urldoc.md" +assert_ok "URL with low-entropy user:pass passes" bash "$HOOK" --files "$SB/urldoc.md" + +# โ”€โ”€ class=secret FILENAME blocks regardless of content โ”€โ”€ +echo "K=v" >"$SB/.env" +assert_fail "class=secret filename (.env) blocks" bash "$HOOK" --files "$SB/.env" +# โ€ฆbut documented templates (.env.example) are NOT flagged by filename +printf 'API_KEY=your_key_here\nDB_URL=postgres://user:pass@host/db\n' >"$SB/.env.example" +assert_ok "template .env.example passes (placeholders)" bash "$HOOK" --files "$SB/.env.example" + +# โ”€โ”€ allowlist escape valve: inline marker โ”€โ”€ +printf 'doc token ghp_%s ckis-allow-secret\n' "$(rnd 36)" >"$SB/inline.md" +assert_ok "inline ckis-allow-secret marker passes" bash "$HOOK" --files "$SB/inline.md" + +# โ”€โ”€ .git/config embedded-credential scan (the .git/config blind spot) โ”€โ”€ +G="$SB/cfgrepo"; mkdir -p "$G"; git -C "$G" init -q +git -C "$G" remote add origin "https://aedneth:ghp_$(rnd 36)@github.com/aedneth/x.git" +assert_fail "embedded PAT in .git/config blocks (--git-config)" bash "$HOOK" --git-config "$G" +git -C "$G" remote set-url origin "https://github.com/aedneth/x.git" +assert_ok "tokenless .git/config passes (--git-config)" bash "$HOOK" --git-config "$G" + +# โ”€โ”€ Hook mode inside a real repo + repo-root allowlist file โ”€โ”€ R="$SB/repo"; mkdir -p "$R"; git -C "$R" init -q cp "$HOOK" "$R/.git/hooks/pre-commit"; chmod +x "$R/.git/hooks/pre-commit" git -C "$R" config user.email t@t; git -C "$R" config user.name t echo "hello world" >"$R/ok.md"; git -C "$R" add ok.md -assert_ok "clean commit succeeds" git -C "$R" commit -q -m ok -printf 'AKIA%s\n' "ABCDEFGHIJKLMNOP" >"$R/leak.md"; git -C "$R" add leak.md -assert_fail "aws key commit blocked" git -C "$R" commit -q -m leak +assert_ok "clean commit succeeds" git -C "$R" commit -q -m ok +printf 'leak = ghp_%s\n' "$(rnd 36)" >"$R/leak.md"; git -C "$R" add leak.md +assert_fail "real token commit blocked" git -C "$R" commit -q -m leak +git -C "$R" reset -q # unstage the blocked leak.md so it isn't re-scanned +# allowlist the fixtures path -> same content now permitted +mkdir -p "$R/tests"; printf 'fixture ghp_%s\n' "$(rnd 36)" >"$R/tests/fx.md" +printf 'tests/*\n' >"$R/.ckis-secret-allow" +git -C "$R" add tests/fx.md .ckis-secret-allow +assert_ok "allowlisted path commit succeeds" git -C "$R" commit -q -m allowed + +# โ”€โ”€ REGRESSION: a long note that documents security work (quotes a marker in +# prose, far from any base64-looking line) must pass โ€” the exact false positive +# that once jammed a knowledge vault's backup for hours. +{ + echo "# Security notes" + echo "The scanner used to false-positive on a note that merely quoted" + echo "-----BEGIN PRIVATE KEY----- inside a sentence about how the scanner works," + echo "with no actual key body anywhere near the marker. That must pass now." +} >"$SB/securitynote.md" +assert_ok "doc quoting a marker (no adjacent body) passes" bash "$HOOK" --files "$SB/securitynote.md" assert_summary