From 7db44e5652d20f4c197f6f1e914ea7633c6f5704 Mon Sep 17 00:00:00 2001 From: Jacob Magar Date: Sat, 18 Jul 2026 02:09:46 -0400 Subject: [PATCH 1/3] fix: remediate comprehensive repository review --- .github/dependabot.yml | 4 + .github/workflows/ci.yml | 81 + .github/workflows/openwiki-update.yml | 142 +- .github/workflows/release-please.yml | 10 +- .github/workflows/release.yml | 121 + CHANGELOG.md | 148 +- CLAUDE.md | 21 +- INCREMENTAL_CATALOG_UPDATE.md | 7 +- Makefile | 57 +- README.md | 1422 +---------- cmd/nugs/api_client.go | 54 +- cmd/nugs/batch.go | 3 +- cmd/nugs/cache.go | 3 +- cmd/nugs/cancel_unix.go | 3 +- cmd/nugs/cancel_windows.go | 3 +- cmd/nugs/catalog_analysis_mediaaware.go | 3 +- cmd/nugs/catalog_autorefresh.go | 3 +- cmd/nugs/catalog_handlers.go | 4 +- cmd/nugs/cli_integration_test.go | 66 + cmd/nugs/completions.go | 3 +- cmd/nugs/config.go | 3 +- cmd/nugs/config_ffmpeg_test.go | 19 +- cmd/nugs/detach_common.go | 3 +- cmd/nugs/detach_unix.go | 3 +- cmd/nugs/detach_windows.go | 3 +- cmd/nugs/download.go | 3 +- cmd/nugs/filelock.go | 3 +- cmd/nugs/format.go | 39 +- cmd/nugs/helpers.go | 10 +- cmd/nugs/hotkey_input.go | 3 +- cmd/nugs/list_commands.go | 3 +- cmd/nugs/main.go | 280 +- cmd/nugs/main_flow_test.go | 68 + cmd/nugs/model_aliases.go | 108 +- cmd/nugs/output.go | 3 +- cmd/nugs/rclone.go | 3 +- cmd/nugs/runtime_status.go | 13 +- cmd/nugs/signal_persistence.go | 3 +- cmd/nugs/structs.go | 27 +- cmd/nugs/url_parser.go | 3 +- cmd/nugs/version.go | 55 + cmd/nugs/version_test.go | 35 + cmd/nugs/video.go | 3 +- cmd/nugs/watch.go | 34 +- docs/ARCHITECTURE.md | 23 +- docs/COMMANDS.md | 18 +- docs/CONFIG.md | 1119 ++------ docs/QUICK_REFERENCE.md | 207 +- docs/nugs-api-endpoints.md | 22 +- docs/nugs-client-README.md | 345 --- go.mod | 2 +- internal/api/apilog.go | 130 +- internal/api/apilog_test.go | 51 + internal/api/circuit.go | 20 +- internal/api/circuit_test.go | 46 + internal/api/client.go | 195 +- internal/cache/artist_cache.go | 70 +- internal/cache/atomic.go | 48 + internal/cache/cache.go | 196 +- internal/cache/containers_index_test.go | 6 +- internal/cache/filelock_windows.go | 50 +- internal/cache/full_catalog.go | 80 + internal/cache/regression_test.go | 5 +- internal/cache/remediation_test.go | 71 + internal/catalog/catalog_update_test.go | 5 +- internal/catalog/handlers.go | 49 +- internal/catalog/handlers_coverage_test.go | 17 +- internal/catalog/helpers.go | 15 +- internal/catalog/media_filter.go | 15 +- internal/catalog/request_budget_test.go | 48 + internal/catalog/watch.go | 36 +- internal/catalog/watch_systemd.go | 92 +- internal/catalog/watch_test.go | 84 +- internal/config/atomic_test.go | 52 + internal/config/config.go | 109 +- internal/download/audio.go | 116 +- internal/download/batch.go | 14 +- internal/download/deps.go | 9 + internal/download/deps_test.go | 12 + internal/download/precalculate_test.go | 37 + internal/download/regression_test.go | 29 +- internal/download/video.go | 74 +- internal/helpers/errors.go | 12 +- internal/helpers/paths.go | 33 +- internal/helpers/paths_security_test.go | 40 + internal/list/list.go | 19 +- internal/model/constants.go | 2 + internal/model/progress_box.go | 3 +- internal/model/progress_box_test.go | 13 + internal/model/types.go | 28 - internal/notify/gotify.go | 42 +- internal/notify/gotify_security_test.go | 41 + internal/rclone/concurrency_test.go | 73 + internal/rclone/rclone.go | 30 +- internal/rclone/storage_adapter.go | 72 +- internal/rclone/storage_adapter_test.go | 2 +- internal/runtime/detach_unix.go | 2 +- internal/runtime/status.go | 22 +- scripts/check-docs.py | 72 + tools/openwiki/package-lock.json | 2683 ++++++++++++++++++++ tools/openwiki/package.json | 11 + 101 files changed, 5614 insertions(+), 3988 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 cmd/nugs/cli_integration_test.go create mode 100644 cmd/nugs/main_flow_test.go create mode 100644 cmd/nugs/version.go create mode 100644 cmd/nugs/version_test.go delete mode 100644 docs/nugs-client-README.md create mode 100644 internal/api/apilog_test.go create mode 100644 internal/api/circuit_test.go create mode 100644 internal/cache/atomic.go create mode 100644 internal/cache/full_catalog.go create mode 100644 internal/cache/remediation_test.go create mode 100644 internal/catalog/request_budget_test.go create mode 100644 internal/config/atomic_test.go create mode 100644 internal/download/precalculate_test.go create mode 100644 internal/helpers/paths_security_test.go create mode 100644 internal/notify/gotify_security_test.go create mode 100644 internal/rclone/concurrency_test.go create mode 100644 scripts/check-docs.py create mode 100644 tools/openwiki/package-lock.json create mode 100644 tools/openwiki/package.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e41bb07..d6edcfe 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,6 +8,8 @@ updates: prefix: deps prefix-development: deps include: scope + cooldown: + default-days: 7 - package-ecosystem: gomod directory: / schedule: @@ -16,3 +18,5 @@ updates: prefix: deps prefix-development: deps include: scope + cooldown: + default-days: 7 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5ca3c53 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,81 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +concurrency: + group: ci-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + quality: + name: Format, test, analyze, and document + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: true + - name: Formatting + run: test -z "$(gofmt -l cmd internal)" + - name: Module integrity + run: | + go mod tidy -diff + go mod verify + - name: Vet + run: go vet ./... + - name: Race tests + run: go test -race -count=1 ./... + - name: Documentation contracts + run: python3 scripts/check-docs.py + - name: Static analysis + run: go run honnef.co/go/tools/cmd/staticcheck@2025.1.1 ./... + - name: Vulnerability analysis + run: go run golang.org/x/vuln/cmd/govulncheck@v1.1.4 ./... + + build: + name: Build ${{ matrix.target }} + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + include: + - target: linux-amd64 + goos: linux + goarch: amd64 + - target: linux-arm64 + goos: linux + goarch: arm64 + - target: darwin-amd64 + goos: darwin + goarch: amd64 + - target: darwin-arm64 + goos: darwin + goarch: arm64 + - target: windows-amd64 + goos: windows + goarch: amd64 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: true + - name: Cross-build + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + BUILD_TARGET: ${{ matrix.target }} + run: go build -trimpath -o "${RUNNER_TEMP}/nugs-${BUILD_TARGET}" ./cmd/nugs diff --git a/.github/workflows/openwiki-update.yml b/.github/workflows/openwiki-update.yml index 6e55482..e393b55 100644 --- a/.github/workflows/openwiki-update.yml +++ b/.github/workflows/openwiki-update.yml @@ -3,76 +3,66 @@ name: OpenWiki Update on: workflow_dispatch: schedule: - # GitHub schedules use UTC; 08:00 UTC is midnight PST. - cron: "0 8 * * *" permissions: - contents: write - pull-requests: write + contents: read + +concurrency: + group: openwiki-update + cancel-in-progress: false jobs: - update: + generate: + name: Generate documentation (secret-bearing, read-only) runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read steps: - name: Check out repository - uses: actions/checkout@v4 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 with: - persist-credentials: true + persist-credentials: false - name: Set up Node.js - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4 with: node-version: "22" + cache: npm + cache-dependency-path: tools/openwiki/package-lock.json + + - name: Install integrity-locked OpenWiki + run: npm ci --ignore-scripts --prefix tools/openwiki - name: Connect Tailscale - uses: tailscale/github-action@v4 + uses: tailscale/github-action@306e68a486fd2350f2bfc3b19fcd143891a4a2d8 # v4 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} tags: tag:ci ping: 100.120.242.29 - - name: Install OpenWiki - run: npm install --global openwiki - - name: Preflight OpenWiki API env: OPENAI_COMPATIBLE_API_KEY: ${{ secrets.OPENAI_COMPATIBLE_API_KEY }} OPENAI_COMPATIBLE_BASE_URL: http://100.120.242.29:8317/v1 run: | set -euo pipefail - - for env_file in "$HOME/docs/.env" /home/jmagar/docs/.env; do - if [[ -f "$env_file" ]]; then - set -a - source "$env_file" - set +a - break - fi - done - export OPENAI_COMPATIBLE_BASE_URL="http://100.120.242.29:8317/v1" - - if [[ -z "${OPENAI_COMPATIBLE_API_KEY:-}" ]]; then - echo "::error::OPENAI_COMPATIBLE_API_KEY is required for OpenWiki updates" - exit 1 - fi - + test -n "${OPENAI_COMPATIBLE_API_KEY:-}" response="$(mktemp)" - status="$( - curl --silent --show-error --location \ - --output "$response" \ - --write-out "%{http_code}" \ - --header "Authorization: Bearer ${OPENAI_COMPATIBLE_API_KEY}" \ - "${OPENAI_COMPATIBLE_BASE_URL%/}/models" || true - )" + trap 'rm -f "$response"' EXIT + status="$(curl --silent --show-error --location --max-time 30 --connect-timeout 10 \ + --output "$response" --write-out "%{http_code}" \ + --header "Authorization: Bearer ${OPENAI_COMPATIBLE_API_KEY}" \ + "${OPENAI_COMPATIBLE_BASE_URL%/}/models")" if [[ "$status" != "200" ]]; then - echo "::error::OpenWiki API preflight failed for ${OPENAI_COMPATIBLE_BASE_URL%/}/models with HTTP ${status}" + echo "::error::OpenWiki API preflight failed with HTTP ${status}" head -c 300 "$response" - echo exit 1 fi - - name: Run OpenWiki + - name: Generate OpenWiki content env: OPENWIKI_PROVIDER: openai-compatible OPENAI_COMPATIBLE_API_KEY: ${{ secrets.OPENAI_COMPATIBLE_API_KEY }} @@ -80,33 +70,61 @@ jobs: OPENWIKI_MODEL_ID: gpt-5.3-codex-spark run: | set -euo pipefail + timeout --signal=TERM --kill-after=30s 20m \ + tools/openwiki/node_modules/.bin/openwiki --update --print + test -d openwiki + test -n "$(find openwiki -type f -print -quit)" - for env_file in "$HOME/docs/.env" /home/jmagar/docs/.env; do - if [[ -f "$env_file" ]]; then - set -a - source "$env_file" - set +a - break - fi - done - export OPENWIKI_PROVIDER="${OPENWIKI_PROVIDER:-openai-compatible}" - export OPENAI_COMPATIBLE_BASE_URL="http://100.120.242.29:8317/v1" - export OPENWIKI_MODEL_ID="${OPENWIKI_MODEL_ID:-gpt-5.3-codex-spark}" + - name: Upload generated documentation + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: openwiki-generated + path: openwiki + if-no-files-found: error + retention-days: 1 - if [[ -z "${OPENAI_COMPATIBLE_API_KEY:-}" ]]; then - echo "::error::OPENAI_COMPATIBLE_API_KEY is required for OpenWiki updates" - exit 1 - fi + publish: + name: Publish pull request (write-capable, no service secrets) + needs: generate + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: write # Push the generated documentation branch. + pull-requests: write # Create or update the generated documentation PR. + steps: + - name: Check out repository + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false - openwiki --update --print - - name: Create OpenWiki update pull request - uses: peter-evans/create-pull-request@v7 + - name: Download generated documentation + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: - add-paths: openwiki - branch: openwiki/update - commit-message: "docs: update OpenWiki" - title: "docs: update OpenWiki" - body: | - Automated OpenWiki documentation update. + name: openwiki-generated + path: openwiki - This PR was generated by the scheduled OpenWiki workflow. + - name: Create or update OpenWiki pull request + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + git config user.name github-actions[bot] + git config user.email 41898282+github-actions[bot]@users.noreply.github.com + git fetch origin '+refs/heads/openwiki/update:refs/remotes/origin/openwiki/update' || true + git switch -C openwiki/update + git add -- openwiki + if git diff --cached --quiet; then + echo "No OpenWiki changes" + exit 0 + fi + git commit -m "docs: update OpenWiki" + gh auth setup-git + git push --force-with-lease origin openwiki/update + if gh pr view openwiki/update >/dev/null 2>&1; then + gh pr edit openwiki/update --title "docs: update OpenWiki" \ + --body "Automated update generated in a separate read-only, secret-bearing job." + else + gh pr create --base main --head openwiki/update \ + --title "docs: update OpenWiki" \ + --body "Automated update generated in a separate read-only, secret-bearing job." + fi diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index baf92f1..68f5496 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -15,17 +15,19 @@ concurrency: jobs: release-please: + name: Prepare or publish release runs-on: ubuntu-latest + timeout-minutes: 15 permissions: - contents: write - pull-requests: write - issues: write + contents: write # Update release branches and publish tags/releases. + pull-requests: write # Create and maintain the release PR. + issues: write # Label issues linked to a release. steps: - name: Require release-please token env: RELEASE_PLEASE_TOKEN: ${{ secrets.RELEASE_PLEASE_TOKEN }} run: test -n "$RELEASE_PLEASE_TOKEN" - - uses: googleapis/release-please-action@8b8fd2cc23b2e18957157a9d923d75aa0c6f6ad5 + - uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4 id: release with: token: ${{ secrets.RELEASE_PLEASE_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..b003e65 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,121 @@ +name: Release assets + +on: + release: + types: [published] + +permissions: + contents: read + +concurrency: + group: release-assets-${{ github.event.release.tag_name }} + cancel-in-progress: false + +jobs: + build: + name: Build and smoke-test ${{ matrix.target }} + runs-on: ${{ matrix.runner }} + timeout-minutes: 20 + strategy: + fail-fast: true + matrix: + include: + - target: linux-amd64 + runner: ubuntu-latest + goos: linux + goarch: amd64 + binary: nugs + smoke: true + - target: linux-arm64 + runner: ubuntu-latest + goos: linux + goarch: arm64 + binary: nugs + smoke: false + - target: darwin-amd64 + runner: macos-15-intel + goos: darwin + goarch: amd64 + binary: nugs + smoke: true + - target: darwin-arm64 + runner: macos-14 + goos: darwin + goarch: arm64 + binary: nugs + smoke: true + - target: windows-amd64 + runner: windows-latest + goos: windows + goarch: amd64 + binary: nugs.exe + smoke: true + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + persist-credentials: false + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5 + with: + go-version-file: go.mod + cache: false + - name: Build release binary + shell: bash + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + VERSION: ${{ github.event.release.tag_name }} + COMMIT: ${{ github.sha }} + RELEASE_BINARY: ${{ matrix.binary }} + run: | + set -euo pipefail + mkdir -p dist/package + build_date="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + go build -trimpath \ + -ldflags "-s -w -X main.version=${VERSION} -X main.commit=${COMMIT} -X main.buildDate=${build_date}" \ + -o "dist/package/${RELEASE_BINARY}" ./cmd/nugs + - name: Smoke-test native binary + if: matrix.smoke + shell: bash + env: + RELEASE_BINARY: ${{ matrix.binary }} + run: '"dist/package/${RELEASE_BINARY}" version' + - name: Create archive + shell: bash + env: + RELEASE_TAG: ${{ github.event.release.tag_name }} + RELEASE_TARGET: ${{ matrix.target }} + RELEASE_BINARY: ${{ matrix.binary }} + run: tar -C dist/package -czf "dist/nugs-${RELEASE_TAG}-${RELEASE_TARGET}.tar.gz" "${RELEASE_BINARY}" + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: release-${{ matrix.target }} + path: dist/*.tar.gz + if-no-files-found: error + retention-days: 7 + + publish: + name: Publish checksummed assets and provenance + needs: build + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: write # Attach checksums and archives to the release. + id-token: write # Authenticate GitHub's keyless provenance signer. + attestations: write # Publish the provenance attestation. + steps: + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + pattern: release-* + path: dist + merge-multiple: true + - name: Create checksums + run: cd dist && sha256sum -- *.tar.gz > checksums.txt + - name: Attest release archives + uses: actions/attest-build-provenance@e8998f949152b193b063cb0ec769d69d929409be # v2 + with: + subject-path: dist/*.tar.gz + - name: Upload release assets + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.event.release.tag_name }} + run: gh release upload "${RELEASE_TAG}" dist/*.tar.gz dist/checksums.txt --clobber diff --git a/CHANGELOG.md b/CHANGELOG.md index 6d7396d..ac3c289 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,145 +1,19 @@ # Changelog -All notable changes to this project will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), -and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - -## [0.0.2](https://github.com/jmagar/nugs-cli/compare/v0.0.1...v0.0.2) (2026-07-11) +Notable tagged releases are documented here. Release Please maintains this file; +the manifest and Git tags are the release-order source of truth. +## [0.0.2](https://github.com/jmagar/nugs-cli/releases/tag/v0.0.2) (2026-07-11) ### Fixed -* **ci:** switch OpenWiki to local openai-compatible proxy ([6774b71](https://github.com/jmagar/nugs-cli/commit/6774b71a281c493f5caca1364c3f6487453be4e0)) - -## [v1.0.0] - 2026-02-24 - -The first release of **nugs-cli** — a complete rewrite and fork of [Sorrow446/Nugs-Downloader](https://github.com/Sorrow446/Nugs-Downloader). This release transforms a single-file Go script into a production-grade CLI with 14 internal packages, a catalog system, watch automation, and cloud upload integration. - -### ⚠️ Breaking Changes - -- **`rclonePath` no longer affects local downloads**: `rclonePath` now ONLY controls remote upload destinations. Use `outPath` for local download paths. - - Migration: Set `outPath` to your desired local directory explicitly -- **Module renamed**: `github.com/Sorrow446/Nugs-Downloader` → `github.com/jmagar/nugs-cli` -- **Go 1.24+ required**: Minimum Go version bumped from 1.16 to 1.24 -- **Binary location changed**: Now installs to `~/.local/bin/nugs` via `make build` - -### ✨ Features - -**Catalog System** -- Local catalog caching of the entire Nugs.net library (~31,000 shows) with four index files for fast lookups (4e49848) -- Gap detection: find missing shows in your collection per artist (694718e) -- Catalog stats with date ranges and visual status indicators (4fe4889, e4088d3) -- New shows indicator: see what was added since your last catalog update (4e49848) -- Auto-refresh system with configurable daily/weekly schedule and timezone support - -**Watch & Automation** -- `nugs watch [artistID]` — poll for new shows, auto-download or notify (ede09c0) -- Systemd integration: `nugs watch install` generates `.service` + `.timer` units (ede09c0) -- Per-artist Gotify notifications during multi-artist watch runs (bb99245) - -**Notifications** -- Gotify push notifications for download completions, catalog updates, and errors (ede09c0) -- Configurable via `gotifyURL`, `gotifyToken`, `gotifyPriority` - -**API Resilience** -- Rate limiting, circuit breaker, and automatic retry with exponential backoff (974d89e) -- `context.Context` threading through all API functions for proper cancellation (e6dcd6c) - -**Media Type System** -- Audio/video/both filtering on all catalog commands (dfa1a5d) -- Media type detection with emoji indicators: 🎵 audio, 🎬 video, 📹 both (0587602) -- `nugs gaps 1125 video` — video-only gap detection and fill (31fb22f) -- Separate `defaultOutputs` config for audio vs video paths (5900d9c) - -**Upload Progress** -- Real-time rclone upload progress with percent, speed, and ETA in the progress box (acd09fd, 8bea651) -- Upload verification before local file deletion (0ffdf11) - -**UI Overhaul** -- Unicode formatting with color-coded output throughout the CLI (30358ef, c978771) -- Welcome screen showing latest shows from Nugs.net (0c106c1) -- JSON output mode for all list commands with multiple detail levels (c47f5b6) -- Responsive table rendering with proper ANSI handling (59d15ed) - -**Video Downloads** -- Enhanced video format detection with fallback chains (cd6a23f) -- Video downloads from artist URLs (a7ce96c) -- Skip video-only albums option (4385434) -- `--skip-chapters` option for video downloads (360a68c) - -**Shell Completions** -- `nugs completion ` for bash, zsh, fish, and PowerShell (245960f) - -**Developer Experience** -- Comprehensive command documentation (cd6a23f) -- `make build` installs to `~/.local/bin/nugs` (fdfd20f) -- Race detector in test suite (28cbc82) - -### 🐛 Bug Fixes - -- Fix deadlocks in upload progress callbacks (29bee36, 5fabd62) -- Fix progress box phase state machine — reject unknown transitions, reset between albums (c032303) -- Dynamically update ShowTotal during progress tracking (1112f6b) -- Preserve JSON output when remote scan fails (e50245a) -- Fix availType to fetch downloadable shows correctly (911c7f0) -- Harden gap detection and upload verification (0ffdf11) -- Mutex protection for batch state (06446e4) -- Replace string error matching with `errors.Is` (9a49654) -- Redact filesystem paths in diagnostics (eb33400) -- Fix rclone progress parsing and progress box rendering (1d81be9) -- Handle ANSI color codes properly in table truncation (59d15ed) -- Fix livestream URL parsing (1a2e465, d27dbef, 24d4104) -- Fix API stream URL resolution (c1aeab2) - -### 🔧 Refactoring - -- **Complete monorepo restructuring**: Decomposed single-file monolith into 14 internal packages with strict dependency tiers (03ae139 → dbd27b5) - - `internal/model` — Pure type definitions (Tier 0) - - `internal/helpers` — Utility functions (Tier 1) - - `internal/ui` — Theme, formatting, output (Tier 1) - - `internal/api` — HTTP client, URL parser (Tier 1) - - `internal/cache` — File locking, catalog cache (Tier 1) - - `internal/config` — Config management (Tier 2) - - `internal/rclone` — Cloud storage integration (Tier 2) - - `internal/runtime` — Session and platform code (Tier 2) - - `internal/catalog` — Catalog operations, watch, gap detection (Tier 3) - - `internal/download` — Audio, video, batch engines (Tier 3) - - `internal/list` — Artist/show list output (Tier 3) - - `internal/notify` — Gotify push alerts (Tier 3) - - `internal/completion` — Shell completions (Tier 3) - - `internal/testutil` — Shared test helpers (Tier 0) -- Storage adapter pattern for download backends (d02bc65) -- Rename `ApiMethod` → `APIMethod` per Go conventions (75cbd27) -- `context.Context` threading through rclone, list, and all packages (adb540b, c2798ee) - -### 🧪 Testing - -- Comprehensive unit tests for upload progress (48df4dc) -- Watch tests: 12 watch tests + 7 notification cases (ede09c0) -- Per-artist notification tests (bb99245) -- Edge cases for `IsShowDownloadable` (a9b3894) -- Shared test utilities: `WithTempHome`, `CaptureStdout`, `ChdirTemp` (internal/testutil) - -### 📚 Documentation - -- Full CLAUDE.md development guide with architecture overview (1adfb2b) -- ARCHITECTURE.md — package structure, dependencies, design patterns -- CONFIG.md — complete configuration reference (all 23 fields) -- COMMANDS.md — comprehensive command examples -- Migration guide for `rclonePath` behavior change - -### 🔨 Maintenance - -- `.gitignore` to protect credentials and build artifacts (0385bad) -- Applied `gofmt -s` formatting across codebase (58cec8a) -- Addressed 80+ PR review comments across all packages - -### 👥 Contributors +- **ci:** switch OpenWiki to the configured OpenAI-compatible proxy + ([6774b71](https://github.com/jmagar/nugs-cli/commit/6774b71a281c493f5caca1364c3f6487453be4e0)) -- @jmagar (94 commits) -- @Sorrow446 (20 commits — original upstream) -- @twalker1998 (4 commits — skip video-only albums) -- @marksibert (1 commit — livestream URL fix) +## Pre-release history -[v1.0.0]: https://github.com/jmagar/nugs-cli/releases/tag/v1.0.0 +Work before the Release Please manifest was bootstrapped was not published as a +`v1.0.0` release. Historical commits remain available in Git, but are not listed +as tagged releases here. Current command and configuration contracts are in +[README.md](README.md), [docs/COMMANDS.md](docs/COMMANDS.md), and +[docs/CONFIG.md](docs/CONFIG.md). diff --git a/CLAUDE.md b/CLAUDE.md index f8fb558..0a9575d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,7 +5,7 @@ This is the main development guide for contributors. For specialized topics, see ## 📚 Documentation Index - **[ARCHITECTURE.md](./docs/ARCHITECTURE.md)** - Package structure, dependencies, design patterns -- **[CONFIG.md](./docs/CONFIG.md)** - Complete configuration reference (all 23 fields) +- **[CONFIG.md](./docs/CONFIG.md)** - Complete configuration reference - **[README.md](./README.md)** - User documentation and command reference - **[COMMANDS.md](./docs/COMMANDS.md)** - Comprehensive command examples @@ -26,7 +26,7 @@ This is the main development guide for contributors. For specialized topics, see ## Quick Start **Prerequisites:** -- Go 1.24+ (required — `go.mod` specifies `go 1.24`) +- Go 1.25.12+ (required — `go.mod` specifies the patched minimum toolchain) - FFmpeg (for video downloads) - Make (required for building) @@ -98,7 +98,7 @@ Root: Command Orchestration (cmd/nugs/main.go) - File locking for concurrent safety **Auto-Refresh** -- Configurable schedule (daily/weekly) +- Configurable schedule (hourly/daily/weekly) - Timezone-aware refresh times - Automatic updates at startup if needed @@ -277,7 +277,7 @@ ffmpeg -hide_banner \ **Sequential track downloads** (NOT parallel): - Prevents API rate limits - Single progress box per album -- **Exception:** Size pre-calculation uses 8 concurrent HEAD requests (60s timeout) +- **Exception:** Size pre-calculation uses 8 concurrent HEAD requests (5s per request/track, 60s overall maximum) ### Error Handling @@ -317,7 +317,7 @@ Run `go test ./... -cover` to get the latest coverage report. **Key areas needing coverage:** - `internal/api` - Authentication, rate limiting, circuit breaker -- `internal/config` - Config loading, all 23 fields +- `internal/config` - Config loading and all JSON-tagged fields - `internal/download` - Download logic, FFmpeg integration - `internal/notify` - Gotify client (new) @@ -367,7 +367,7 @@ func TestParseMediaModifier(t *testing.T) { 1. **API client tests** with `httptest.Server` (mock responses) 2. **Cache concurrency tests** (file locking, corruption recovery) -3. **Config validation tests** (all 23 fields) +3. **Config validation tests** (all JSON-tagged fields) 4. **Download flow tests** with small test files (<1MB) ### Manual Testing @@ -532,13 +532,13 @@ make build Key capabilities implemented as of the current branch (see `git log` for full history): **Watch / Catalog Monitoring** -- `nugs watch [artistID]` — Poll for new shows and auto-download or notify -- Systemd integration: `nugs watch install` generates a `.service` + `.timer` unit +- `nugs watch add/remove/list` manages watched artists; `nugs watch check` downloads new shows +- `nugs watch enable/disable` manages the systemd service and timer - Files: `internal/catalog/watch.go`, `watch_systemd.go`, `watch_systemd_other.go` **Gotify Push Notifications** - Download completions, catalog updates, and errors trigger Gotify push alerts -- Config fields: `gotifyURL`, `gotifyToken`, `gotifyPriority` +- Config fields: `gotifyUrl`, `gotifyToken`; priorities are application-selected - File: `internal/notify/gotify.go` **API Resilience** @@ -547,7 +547,8 @@ Key capabilities implemented as of the current branch (see `git log` for full hi - File: `internal/api/` **Media Type Filtering** -- All catalog commands accept `audio`, `video`, or `both` modifiers +- Artist list, gaps, coverage, watch-check, and download commands accept media modifiers where documented +- `nugs latest` cannot filter by media because its source omits product details - Emoji indicators in output: 🎵 audio, 🎬 video, 📹 both - `nugs gaps 1125 video` — video-only gap detection and fill diff --git a/INCREMENTAL_CATALOG_UPDATE.md b/INCREMENTAL_CATALOG_UPDATE.md index 5804299..0512387 100644 --- a/INCREMENTAL_CATALOG_UPDATE.md +++ b/INCREMENTAL_CATALOG_UPDATE.md @@ -1,4 +1,9 @@ - Plan: Full Catalog Crawl + Incremental Updates +# Historical proposal: Full Catalog Crawl + Incremental Updates + +> Status: design proposal, not a current command or implementation reference. +> In particular, do not assume `nugs catalog update full` or the proposed +> `full-catalog-*.json` files exist until the feature lands and this document is +> promoted into the user documentation. Context diff --git a/Makefile b/Makefile index 32fb79c..dcb3bbb 100644 --- a/Makefile +++ b/Makefile @@ -1,23 +1,58 @@ +VERSION ?= $(shell git describe --tags --always --dirty) +COMMIT ?= $(shell git rev-parse --short=12 HEAD) +BUILD_DATE ?= $(shell date -u +%Y-%m-%dT%H:%M:%SZ) +INSTALL_DIR ?= $(HOME)/.local/lib/nugs BINARY := $(HOME)/.local/bin/nugs +BINARY_DIR := $(dir $(BINARY)) +VERSIONED_BINARY := $(INSTALL_DIR)/nugs-$(VERSION) +LDFLAGS := -s -w -X main.version=$(VERSION) -X main.commit=$(COMMIT) -X main.buildDate=$(BUILD_DATE) SOURCES := $(shell find cmd internal -name '*.go') go.mod go.sum -.PHONY: build clean install test +.PHONY: build clean install test docs-check fmt-check module-check vet staticcheck vulncheck cross-build verify -# Build only if source files are newer than the binary -build: $(BINARY) +build: $(VERSIONED_BINARY) + @mkdir -p $(BINARY_DIR) + @ln -sfn $(VERSIONED_BINARY) $(BINARY) + @echo "installed: $(BINARY) -> $(VERSIONED_BINARY)" -$(BINARY): $(SOURCES) - @mkdir -p $(HOME)/.local/bin - @echo "Building nugs..." - @go build -o $(BINARY) ./cmd/nugs - @echo "done: $(BINARY)" +$(VERSIONED_BINARY): $(SOURCES) + @mkdir -p $(INSTALL_DIR) + @echo "Building nugs $(VERSION)..." + @go build -trimpath -ldflags '$(LDFLAGS)' -o $(VERSIONED_BINARY) ./cmd/nugs -# Clean removes binary from ~/.local/bin clean: - @rm -f $(BINARY) ./nugs ./nugs-cli + @rm -f $(BINARY) $(VERSIONED_BINARY) ./nugs ./nugs-cli test: @go test -race -count=1 ./... -# Install is the same as build (already in user's PATH) +docs-check: + @python3 scripts/check-docs.py + +fmt-check: + @test -z "$$(gofmt -l cmd internal)" + +module-check: + @go mod tidy -diff + @go mod verify + +vet: + @go vet ./... + +staticcheck: + @go run honnef.co/go/tools/cmd/staticcheck@2025.1.1 ./... + +vulncheck: + @go run golang.org/x/vuln/cmd/govulncheck@v1.1.4 ./... + +cross-build: + @tmp="$$(mktemp -d)"; trap 'rm -rf "$$tmp"' EXIT; \ + GOOS=linux GOARCH=amd64 go build -trimpath -o "$$tmp/nugs-linux-amd64" ./cmd/nugs; \ + GOOS=linux GOARCH=arm64 go build -trimpath -o "$$tmp/nugs-linux-arm64" ./cmd/nugs; \ + GOOS=darwin GOARCH=amd64 go build -trimpath -o "$$tmp/nugs-darwin-amd64" ./cmd/nugs; \ + GOOS=darwin GOARCH=arm64 go build -trimpath -o "$$tmp/nugs-darwin-arm64" ./cmd/nugs; \ + GOOS=windows GOARCH=amd64 go build -trimpath -o "$$tmp/nugs-windows-amd64.exe" ./cmd/nugs + +verify: fmt-check module-check vet test docs-check staticcheck vulncheck cross-build + install: build diff --git a/README.md b/README.md index 74f10cf..c11428d 100644 --- a/README.md +++ b/README.md @@ -1,84 +1,37 @@ # Nugs CLI -Download your favorite live shows from Nugs.net, browse 13,000+ concerts offline, and never miss a new release. +`nugs` is a local Go CLI for downloading media that your Nugs.net account can +access, browsing the catalog, tracking collection gaps, and automating artist +watch lists. It is not an MCP server or a Python/npm package. -Built for Deadheads, jam band fans, and anyone who wants their live music collection organized and accessible. +## Requirements -[![Release](https://img.shields.io/github/v/release/jmagar/nugs-cli)](https://github.com/jmagar/nugs-cli/releases) -[![Go Version](https://img.shields.io/badge/Go-1.16+-blue.svg)](https://golang.org) -[![Platform](https://img.shields.io/badge/platform-linux%20%7C%20macos%20%7C%20windows-lightgrey)](https://github.com/jmagar/nugs-cli/releases) - -## Contents - -- [Naming](#naming) -- [Capabilities And Boundaries](#capabilities-and-boundaries) -- [Install](#install) -- [Quickstart](#quickstart) -- [Client Configuration](#client-configuration) -- [Runtime Surfaces](#runtime-surfaces) -- [MCP Tool Reference](#mcp-tool-reference) -- [CLI Reference](#cli-reference) -- [Authentication](#authentication) -- [Safety And Trust Model](#safety-and-trust-model) -- [Architecture](#architecture) -- [Distribution Contract](#distribution-contract) -- [Development](#development) -- [Verification](#verification) -- [Deployment](#deployment) -- [Troubleshooting](#troubleshooting) -- [Documentation](#documentation) -- [Features](#features) -- [Installation](#installation) -- [Quick Start](#quick-start) -- [Configuration](#configuration) -- [Supported Media Types](#supported-media-types) -- [Usage](#usage) -- [Advanced Features](#advanced-features) -- [FFmpeg Setup](#ffmpeg-setup) -- [Command Reference](#command-reference) -- [Examples](#examples) -- [Related Servers](#related-servers) -- [License](#license) - -## Naming - -The repository and release identity are still `nugs-cli`; the installed binary -is `nugs`. This is a Go CLI for Nugs.net downloads and catalog management, not -an RMCP server package. There is no `nugs-rmcp` npm package and no `npx -y` -launcher path today. - -## Capabilities And Boundaries - -Nugs CLI downloads and organizes Nugs.net media you already have legal account -access to. It caches the catalog locally, supports audio and video formats, -tracks collection gaps, and can automate watch/update flows. - -**Not for:** bypassing Nugs.net access controls, sharing credentials with -agents, redistributing downloaded media, or exposing a remote automation server. -This is a local operator CLI. - -MCP callers never provide credentials, tokens, keys, or secrets as action -arguments. Nugs credentials belong in the local config file only; this repo does -not ship an MCP tool surface. +- Go 1.25.12 or newer when building from source +- Make +- FFmpeg for video and HLS conversion +- Optional: rclone for remote uploads ## Install -Download the latest release binary from -[GitHub Releases](https://github.com/jmagar/nugs-cli/releases), or build from -source: +Download a platform archive from +[GitHub Releases](https://github.com/jmagar/nugs-cli/releases), verify it against +`checksums.txt`, and place `nugs` (`nugs.exe` on Windows) on `PATH`. + +To build from source: ```bash git clone https://github.com/jmagar/nugs-cli.git cd nugs-cli make build +nugs version ``` -There is no npm launcher, so `npx -y nugs-cli` is intentionally not a supported -install path. +`make build` installs a versioned binary under `~/.local/lib/nugs/` and updates +the stable `~/.local/bin/nugs` symlink. -## Quickstart +## Quick start -The first-screen 30-second path is: +Run the interactive setup rather than hand-authoring an incomplete config: ```bash nugs @@ -87,1291 +40,186 @@ nugs list nugs grab 23329 ``` -Running `nugs` with no arguments creates or checks config and shows recent -catalog additions. +The setup prompts for account credentials, audio/video quality, local output +paths, FFmpeg, and optional rclone settings. The config is written with mode +`0600` on Unix. Password input is hidden. + +For Apple or Google accounts, use a session token; see +[Token authentication](docs/token.md). -## Client Configuration +## Configuration -Configuration is local file based. Nugs CLI checks these paths in order and -uses the first one found: +The first existing file wins; files are not merged: 1. `./config.json` 2. `~/.nugs/config.json` 3. `~/.config/nugs/config.json` -Use file permissions to protect credentials: - -```bash -chmod 600 ~/.nugs/config.json -``` - -## Runtime Surfaces - -| Surface | Status | Purpose | -|---|---:|---| -| CLI | Required | `nugs` local command for downloads, catalog browsing, watch automation, and config | -| Config file | Required | Local credentials and output preferences | -| MCP | Not shipped | No `tools/call` endpoint or MCP server is included | -| REST/Web | Not shipped | The project does not expose a daemon or browser UI | - -## MCP Tool Reference - -No MCP tool is shipped. Do not configure an MCP client with `nugs` today. - -For comparison, a future MCP wrapper would use a normal `tools/call` shape and -would still keep credentials out of call arguments: - -```json -{ - "jsonrpc": "2.0", - "id": 1, - "method": "tools/call", - "params": { - "name": "nugs", - "arguments": { - "action": "list", - "artist": "Billy Strings" - } - } -} -``` - -That example is illustrative only; the supported surface is the CLI below. - -## CLI Reference - -Common commands: - -```bash -nugs -nugs list -nugs list 1125 -nugs grab 23329 -nugs update -nugs gaps -nugs watch -``` - -The detailed command catalog remains in [Command Reference](#command-reference) -and `docs/COMMANDS.md`. - -## Authentication - -Authentication is handled by Nugs.net credentials in the local config file. -Config values include `email`, `password`, and optional `token` for Apple/Google -account flows. Credentials are never passed on the command line or through MCP -arguments. - -## Safety And Trust Model - -Keep `config.json` private, keep downloads local unless you explicitly configure -rclone, and only download media your subscription grants you access to. The CLI -does not run a network listener; automation should be local systemd/watch -automation unless you wrap it yourself. - -## Architecture - -The Go CLI loads config, authenticates to Nugs.net, maintains a local catalog -cache, resolves shows/artists/media formats, downloads media, and optionally -hands completed files to rclone or notification hooks. The catalog cache is -rebuildable from upstream data. - -## Distribution Contract - -The source of truth for release identity is the GitHub Release tag plus the Go -module and release artifacts built from that tag. Distribution/version -invariants: - -- Release binaries must be built from the tagged source. -- The installed binary name remains `nugs`. -- There is no npm package or MCP registry entry to version. -- Generated catalog/cache files are runtime data and should not be committed. -- Curated docs should point at source-of-truth implementation and command docs - instead of duplicating every flag in multiple places. - -## Development - -```bash -make build -go test ./... -``` - -For development setup, architecture details, and coding guidelines, see -[CLAUDE.md](CLAUDE.md). - -## Verification - -Before landing README changes: - -```bash -python3 /home/jmagar/workspace/soma/scripts/check-readme-guide.py README.md -go test ./... -``` - -## Deployment - -Nugs CLI is deployed by installing the `nugs` binary on the machine that owns the -download directory. Watch automation can run via systemd timers; rclone upload is -optional and controlled by local config. - -## Documentation - -This README is curated for first-run orientation. Source-of-truth docs and code -are split as follows: - -- `docs/COMMANDS.md` for command behavior and live API-backed command notes. -- `docs/QUICK_REFERENCE.md` for compact operator examples. -- `docs/nugs-api-endpoints.md` for upstream endpoint research. -- `docs/nugs-client-README.md` for client/library context. -- `cmd/` and `internal/` for implementation source of truth. - ---- - -## Features - -### Media Download -- Multiple formats: ALAC, FLAC, MQA, 360 Reality Audio, AAC -- Video support: Up to 4K with chapter markers -- Batch downloads: Process multiple albums, playlists, or text files -- Artist shortcuts: Download entire catalogs or latest releases - -### Catalog Management -- Offline browsing: Cache entire Nugs.net catalog (13,000+ shows) -- Fast search: Instant statistics and show discovery -- Gap detection: Find missing shows in your collection -- Auto-refresh: Automatic daily/weekly catalog updates -- JSON output: Script-friendly structured data - -### Discovery & Analysis -- List artists: Browse all 300+ artists -- View shows: See complete discography for any artist -- Statistics: Top artists, date ranges, collection coverage -- Latest additions: Track new releases - -### Power User Features -- **Rclone integration**: Auto-upload to cloud storage -- **Watch automation**: Monitor artists, auto-download new shows via systemd timer -- **Push notifications**: Gotify alerts when new shows are downloaded -- **Flexible configuration**: CLI args override config file -- **File locking**: Prevents corruption from concurrent writes - ---- - -## Installation - -### Pre-built Binaries - -Download the latest release for your platform: -- [Windows, Linux, macOS binaries](https://github.com/jmagar/nugs-cli/releases) - -### Building from Source - -**Requirements:** Go 1.16+ and Make - -```bash -git clone https://github.com/jmagar/nugs-cli.git -cd nugs-cli -make build -``` - -The binary installs to `~/.local/bin/nugs` (already in your PATH). - -> **📖 Developers:** See [CLAUDE.md](CLAUDE.md) for detailed build instructions and development guide. - ---- - -## Quick Start - -Alright, let's get you downloading shows in about 2 minutes. - -**Step 1: Run initial setup** (displays welcome screen and creates config): - -```bash -nugs -``` - -**Step 2: Configure your credentials** in one of these locations: -- `./config.json` (current directory) -- `~/.nugs/config.json` (recommended) -- `~/.config/nugs/config.json` (XDG standard) - -```json -{ - "email": "your-email@example.com", - "password": "your-password", - "outPath": "/path/to/downloads", - "format": 2 -} -``` - -> **🔒 Security:** After creating your config, set secure permissions: `chmod 600 ~/.nugs/config.json` -> **💡 Tip:** Most folks use `format: 2` (FLAC) for the best quality-to-size ratio. - -**Step 3: Download your first show**: - -```bash -nugs grab 23329 -``` - -**Step 4: Browse the catalog**: - -```bash -nugs list -nugs list 1125 # Billy Strings -``` - -> **ℹ️ Note:** Running `nugs` with no arguments shows the latest 15 catalog additions - a quick way to discover new releases! - ---- - -## Configuration - -Now that you've seen what Nugs CLI can do, let's customize it for your workflow. - -The config file can be placed in one of three locations (checked in this order, first found wins — no merging): -1. `./config.json` - Current directory -2. `~/.nugs/config.json` - Recommended location -3. `~/.config/nugs/config.json` - XDG standard location - -On first run, you'll be prompted to create it. - -### Core Settings - -| Option | Description | Values | -|--------|-------------|--------| -| `email` | Your Nugs.net email | string | -| `password` | Your Nugs.net password | string | -| `format` | Audio download quality | `1` = 16-bit/44.1kHz ALAC
`2` = 16-bit/44.1kHz FLAC
`3` = 24-bit/48kHz MQA
`4` = 360 Reality Audio
`5` = 150 Kbps AAC | -| `videoFormat` | Video download quality | `1` = 480p
`2` = 720p
`3` = 1080p
`4` = 1440p
`5` = 4K/best available | -| `defaultOutputs` | Default media type preference | `audio` (default) - Prefer audio downloads
`video` - Prefer video downloads
`both` - Download both formats when available | -| `outPath` | Download destination | Path (created if doesn't exist) | -| `videoOutPath` | Video download destination (defaults to `outPath`) | Path | - -### Advanced Settings - -| Option | Description | Default | -|--------|-------------|---------| -| `token` | Auth token for Apple/Google accounts ([guide](token.md)) | - | -| `useFfmpegEnvVar` | Use FFmpeg from PATH vs local dir | `false` | -| `ffmpegNameStr` | Custom FFmpeg binary name or absolute path | `"ffmpeg"` | -| `skipChapters` | Skip embedding chapter markers in video files | `false` | -| `skipSizePreCalculation` | Skip pre-download size calculation (faster start, no ETA) | `false` | -| `forceVideo` | **Deprecated** — use `defaultOutputs: "video"` instead | `false` | -| `skipVideos` | **Deprecated** — use `defaultOutputs: "audio"` instead | `false` | - -### Rclone Settings - -| Option | Description | -|--------|-------------| -| `rcloneEnabled` | Enable auto-upload to cloud storage | -| `rcloneRemote` | Rclone remote name (e.g., `gdrive`) | -| `rclonePath` | Path on remote (e.g., `/Music/Nugs`) | -| `rcloneVideoPath` | Remote path for videos (defaults to `rclonePath`) | -| `deleteAfterUpload` | Delete local files after successful upload | -| `rcloneTransfers` | Number of parallel transfers (default: 4) | - -> **Migration note (2026-02-05):** -> `rclonePath` is now remote-only. Local download detection and storage always use `outPath`. -> If you previously relied on `rclonePath` for local folder checks, move/update your `outPath` -> to the local download root. - -### Catalog Auto-Refresh - -| Option | Description | Default | -|--------|-------------|---------| -| `catalogAutoRefresh` | Enable automatic catalog updates | `true` | -| `catalogRefreshTime` | Time to refresh (24-hour format) | `"05:00"` | -| `catalogRefreshTimezone` | Timezone for refresh time | `"America/New_York"` | -| `catalogRefreshInterval` | Refresh frequency | `"daily"` | - -### Watch & Notifications - -| Option | Description | Default | -|--------|-------------|---------| -| `watchedArtists` | Artist IDs to monitor for new shows | `[]` | -| `watchInterval` | How often to check for new shows (Go duration: `30m`, `1h`, `6h`) | `"1h"` | -| `gotifyUrl` | Gotify server URL for push notifications | `""` | -| `gotifyToken` | Gotify app token | `""` | - -Auto-refresh is **enabled by default** to keep your catalog up-to-date. Configure via commands: +See [Configuration reference](docs/CONFIG.md) for every supported field, +defaults, validation, watch settings, and Gotify settings. Never commit a real +config file or pass credentials as command arguments. -```bash -nugs refresh set # Change schedule (time/timezone/interval) -nugs refresh disable # Disable auto-refresh -nugs refresh enable # Re-enable with current settings -``` - ---- - -## Supported Media Types - -| Type | URL Example | Shorthand | -|------|-------------|-----------| -| **Album** | `https://play.nugs.net/release/23329` | `23329` | -| **Artist (all)** | `https://play.nugs.net/#/artist/461` | `461 full` | -| **Artist (latest)** | `https://play.nugs.net/#/artist/461/latest` | `grab 461 latest` | -| **Catalog Playlist** | `https://2nu.gs/3PmqXLW` | - | -| **User Playlist** | `https://play.nugs.net/#/playlists/playlist/1215400` | - | -| **Video** | `https://play.nugs.net/#/videos/artist/1045/.../27323` | - | -| **Livestream (exclusive)** | `https://play.nugs.net/watch/livestreams/exclusive/30119` | - | -| **Livestream (purchased)** | `https://www.nugs.net/on/demandware.store/...` | - | -| **Webcast** | `https://play.nugs.net/#/my-webcasts/5826189-30369-0-624602` | - | - -> **⚠️ Windows Users:** Wrap URLs containing special characters in quotes to avoid shell parsing issues. - ---- - -## Usage +## Commands -### Downloading Content - -Let's get to the good stuff - actually downloading music. - -**Download single album:** +### Download ```bash nugs grab 23329 -nugs grab https://play.nugs.net/release/23329 -``` - -**Download multiple albums:** - -```bash -nugs grab 23329 23790 24105 -``` - -**Download from text file:** - -```bash -nugs /path/to/urls.txt -``` - -**Download artist's latest shows:** - -```bash -nugs grab 1125 latest # Billy Strings (respects defaultOutputs) -nugs grab 1125 latest video # Latest videos only -nugs grab 461 latest # Grateful Dead -``` - -**Download entire artist catalog:** - -```bash -nugs 1125 full # Billy Strings - all shows (respects defaultOutputs) -nugs 1125 full video # Billy Strings - all videos only -nugs 461 full both # Grateful Dead - download both audio and video +nugs grab 23329 video +nugs grab 23329 both +nugs 1125 latest audio +nugs 1125 full +nugs -f 2 23329 +nugs -F 3 23329 ``` -**Download both formats:** +Supported inputs include numeric release IDs and the documented Nugs release, +artist, playlist, video, livestream, and webcast URL forms. See +[Command reference](docs/COMMANDS.md) for the exact accepted patterns. -```bash -nugs grab 23329 both # Download both audio and video for show 23329 -``` - -**Override quality settings:** - -```bash -nugs grab -f 3 23329 # MQA quality -nugs -F 5 video-url # 4K video -nugs grab -o /mnt/storage/music 23329 # Custom output path -``` - -**Hotkeys during an active download:** +During an attached interactive download: | Key | Action | -|-----|--------| -| `Shift+P` | Pause / resume current download | -| `Shift+C` | Cancel current download | - ---- +|---|---| +| `Shift+P` | Pause or resume | +| `Shift+C` | Cancel | +| `Ctrl+C` | Interrupt | -### Browse & List - -**List all artists:** +### Browse ```bash -nugs list # All artists with media indicators (🎵 🎬 📹) -nugs list audio # Only artists with audio shows -nugs list video # Only artists with video shows -nugs list both # Only artists with both formats -``` - -**View artist's shows:** - -```bash -nugs list 1125 # Billy Strings (all shows with 🎵 🎬 📹 indicators) -nugs list 1125 audio # Billy Strings audio shows only -nugs list 1125 video # Billy Strings video shows only -nugs list 1125 both # Billy Strings shows with both formats -nugs list 461 # Grateful Dead -``` - -**Media Type Indicators:** -- 🎵 Audio -- 🎬 Video -- 📹 Both audio and video - -**Filter and search shows:** - -```bash -# Filter shows by venue -nugs list 461 "Red Rocks" - -# Find artists with 100+ shows -nugs list ">100" -nugs list "<=50" -nugs list "=25" - -# Get artist's latest 5 shows +nugs list +nugs list audio +nugs list 1125 +nugs list 1125 video nugs list 1125 latest 5 +nugs list 1125 "Red Rocks" +nugs list '>100' ``` -> **💡 Advanced users:** All commands support `--json ` output for piping to `jq` or other tools. - ---- - -### Catalog Management - -Want to browse Nugs.net's 13,000+ shows offline? The catalog system has you covered. Update the catalog once, then query it instantly without hitting Nugs.net servers. +Quote comparison filters so the shell does not interpret `>` or `<` as +redirection. -#### Update Catalog - -**Fetch latest catalog** (updates cache with current Nugs.net catalog): +### Catalog ```bash nugs update -``` - -Output: - -```text -✓ Catalog updated successfully - Total shows: 13,253 - Update time: 1 seconds - Cache location: /home/user/.cache/nugs -``` - -#### Cache Status - -**View cache information:** - -```bash nugs cache -``` - -Output: - -```text -Catalog Cache Status: - - Location: /home/user/.cache/nugs - Last Updated: 2026-02-05 14:30:00 (2 hours ago) - Total Shows: 13,253 - Artists: 335 unique - Cache Size: 7.4 MB - Version: v1.0.0 -``` - -#### Statistics - -**View catalog statistics:** - -```bash nugs stats -``` - -Output: - -```text -Catalog Statistics: - - Total Shows: 13,253 - Total Artists: 335 unique - Date Range: 1965-01-01 to 2026-02-04 - -Top 10 Artists by Show Count: - - ID Artist Shows - ────────────────────────────────────────── - 1125 Billy Strings 430 - 22 Umphrey's McGee 415 - 1084 Spafford 411 - ... -``` - -#### Latest Additions - -**View recently added shows:** - -```bash -nugs latest # Default: 15 shows (both formats with emoji) -nugs latest video # Latest video releases only -nugs latest 50 # Last 50 shows (both formats) -nugs latest 50 audio # Last 50 audio releases -``` - -Output: - -```text -Latest 15 Shows in Catalog: - - Artist Date Title Media - 1. 🎬 Daniel Donato 02/03/26 02/03/26 Missoula, MT Video - 2. 🎵 String Cheese... 07/18/00 07/18/00 Mt. Shasta, CA Audio - 3. 📹 Dizgo 01/30/26 01/30/26 Columbus, OH Both - ... -``` - -#### Gap Detection - -**Find missing shows in your collection:** - -```bash -nugs gaps 1125 # Billy Strings (respects defaultOutputs) -nugs gaps 1125 audio # Audio gaps only -nugs gaps 1125 video # Video gaps only -nugs gaps 1125 both # Both audio AND video gaps (stricter - shows you have either audio OR video but not both) -nugs gaps 1125 461 1045 # Multiple artists at once -``` - -Output: - -```text - ID Date Title Media - ──────────────────────────────────────────────────────────────────── - 46385 12/14/25 12/14/25 ACL Live Austin, TX 🎵 - 46380 12/13/25 12/13/25 The Criterion Oklahoma City 🎬 - ... -``` - -**Get IDs only for piping:** - -```bash -nugs gaps 1125 --ids-only -nugs gaps 1125 video --ids-only # Video gaps only -# Output: 46385 -# 46380 -# 46375 - -# Download all audio gaps -nugs gaps 1125 audio --ids-only | xargs -n1 nugs grab - -# Download all video gaps -nugs gaps 1125 video --ids-only | xargs -I {} nugs grab {} video - -# Download first 10 gaps -nugs gaps 1125 --ids-only | head -10 | xargs -n1 nugs grab -``` - -**Auto-download all missing shows:** - -```bash -nugs gaps 1045 fill # Fill gaps (respects defaultOutputs) -nugs gaps 1045 fill video # Fill all video gaps -nugs gaps 1045 fill both # Fill all shows where you're missing either format -``` - -Output: - -```text -Filling Gaps: Phish (Video) - - Total Missing: 234 shows - -⬇ Downloading 1/234: 2025-12-14 - 12/14/25 Madison Square Garden... 🎬 -⬇ Downloading 2/234: 2025-12-13 - 12/13/25 Madison Square Garden... 🎬 -... - -Download Summary: - Total Attempted: 234 - Successfully Downloaded: 232 - Failed: 2 -``` - -#### Coverage Statistics - -**Check download coverage for artists:** - -```bash -# Single artist (respects defaultOutputs) +nugs latest +nugs latest 50 +nugs gaps 1125 +nugs gaps 1125 video --ids-only +nugs gaps 1125 audio fill nugs coverage 1125 - -# Coverage for specific media type -nugs coverage 1125 video # Video coverage only -nugs coverage 1125 audio # Audio coverage only -nugs coverage 1125 both # Both formats coverage - -# Multiple artists -nugs coverage 1125 461 1045 - -# All artists with downloads (auto-detects from output directory) nugs coverage -nugs coverage video # Video coverage for all artists -``` - -Output: - -```text -Download Coverage Statistics (Video) - - Artist ID Artist Name Downloaded Total Coverage Media - ────────────────────────────────────────────────────────────────────────────── - 1125 Billy Strings 12 156 7.7% 🎬 - 461 Phish 45 234 19.2% 🎬 - 1045 Widespread Panic 78 445 17.5% 🎬 -``` - ---- - -### JSON Output - -All catalog commands support `--json ` for machine-readable output. Advanced users can pipe JSON to `jq` or other tools for custom filtering. - -**Levels:** -- `minimal` - Essential fields only -- `standard` - Adds location details -- `extended` - All metadata -- `raw` - Unmodified API response - -**Example:** - -```bash -# Get cache status as JSON -nugs cache --json standard - -# Pipe to jq for custom filtering -nugs stats --json standard | jq '.topArtists[:3]' -``` - ---- - -## Advanced Features - -### Auto-Refresh - -The catalog cache automatically updates on a schedule. **Auto-refresh is enabled by default** (5am EST, daily). - -**Configure custom schedule:** - -```bash -nugs refresh set -# Enter refresh time: 03:00 -# Enter timezone: America/Los_Angeles -# Enter interval: weekly -``` - -**Disable auto-refresh:** - -```bash -nugs refresh disable -``` - -**Re-enable auto-refresh:** - -```bash nugs refresh enable +nugs refresh disable +nugs refresh set ``` -> **🔄 Auto-refresh triggers at startup when:** -> - Auto-refresh is enabled -> - Current time is past the configured refresh time -> - Cache hasn't been updated within the refresh interval - -### Gap Detection - -Find shows you haven't downloaded yet: - -**Basic gap detection:** - -```bash -nugs gaps 1125 # Shows you're missing -``` - -**Integration with downloads:** - -```bash -# Download all missing shows -nugs gaps 1125 --ids-only | xargs -n1 nugs grab - -# Download 5 most recent gaps -nugs gaps 1125 --ids-only | head -5 | xargs -n1 nugs grab - -# Download in parallel (3 concurrent) -nugs gaps 1125 --ids-only | xargs -P 3 -n1 nugs grab - -# Save gaps to file for later -nugs gaps 1125 --ids-only > billy-gaps.txt -``` - -**Check multiple artists:** - -```bash -# All at once -nugs gaps 1125 461 1045 - -# Or use a loop for more control -for artist in 1125 461 1045; do - echo "Gaps for artist $artist:" - nugs gaps $artist --ids-only | wc -l -done -``` - -### Rclone Integration - -Automatically upload downloads to cloud storage (Google Drive, Dropbox, etc.): - -**1. Install and configure rclone:** - -```bash -# Install rclone -curl https://rclone.org/install.sh | sudo bash - -# Configure a remote (follow interactive prompts) -rclone config -``` - -**2. Enable in Nugs config** (`~/.nugs/config.json`): - -```json -{ - "rcloneEnabled": true, - "rcloneRemote": "gdrive", - "rclonePath": "/Music/Nugs", - "deleteAfterUpload": false, - "rcloneTransfers": 4 -} -``` - -**3. Download and upload automatically:** - -```bash -nugs grab 23329 # Downloads and uploads to gdrive:/Music/Nugs/ -``` - -> **📍 Smart Gap Detection:** Gap detection checks both local storage AND your rclone remote, so you won't accidentally re-download shows that are already in the cloud. +`nugs latest` does **not** support `audio`, `video`, or `both` filters because +the latest-catalog response lacks product details. Use `nugs list +` for media-filtered browsing. ---- +With no artist IDs, `nugs coverage` scans artists discovered in configured local +and remote download folders. It does not query account subscriptions. -### Watch Command - -Monitor artists and automatically download new shows on a schedule. Uses a systemd user timer (Linux) — no root required. - -**Manage your watch list:** - -```bash -nugs watch add 1125 # Watch Billy Strings -nugs watch add 461 # Watch Grateful Dead -nugs watch list # Show watched artists with names -nugs watch remove 461 # Stop watching an artist -``` - -**One-shot check** (updates catalog and downloads gaps for all watched artists): +### Watch automation ```bash +nugs watch add 1125 +nugs watch remove 1125 +nugs watch list nugs watch check +nugs watch check video +nugs watch enable +nugs watch disable ``` -**Automated scheduling** (systemd timer, fires on boot + every `watchInterval`): - -```bash -nugs watch enable # Write unit files, enable and start timer -nugs watch disable # Stop, disable, remove unit files -``` - -**With Gotify push notifications** (add to `~/.nugs/config.json`): - -```json -{ - "watchedArtists": ["1125", "461"], - "watchInterval": "1h", - "gotifyUrl": "http://your-gotify-server:8080", - "gotifyToken": "your-app-token" -} -``` - -Notification behavior: -- **New downloads**: single summary notification (priority 5) — `"3 new show(s) downloaded"` -- **Errors only**: error notification (priority 7) with failure details -- **Nothing new**: silent — no hourly noise when everything is up-to-date - -View timer logs: - -```bash -journalctl --user -u nugs-watch.service -f -systemctl --user list-timers -``` - ---- - -## FFmpeg Setup - -FFmpeg is required for: -- **Video downloads** (TS → MP4 conversion) -- **HLS-only audio tracks** (M3U8 → audio file) - -### Installation - -**Linux:** - -```bash -sudo apt install ffmpeg -``` - -**macOS:** +`watch enable` installs and enables the Linux systemd timer; `watch disable` +removes it. `watch check` updates the catalog and fills gaps for watched artists. +Gotify notifications use `gotifyUrl` and `gotifyToken`; notification priorities +are selected by the application, not by a config field. -```bash -brew install ffmpeg -``` - -**Windows:** -1. Download from [FFmpeg Builds](https://github.com/BtbN/FFmpeg-Builds/releases) -2. Extract and add to PATH, or place in Nugs directory - -**Termux (Android):** - -```bash -pkg install ffmpeg -``` - -### Configuration - -**Use FFmpeg from PATH** (recommended): - -```json -{ - "useFfmpegEnvVar": true -} -``` - -**Use FFmpeg from script directory** (if you can't install system-wide): - -```json -{ - "useFfmpegEnvVar": false -} -``` - -Just drop the `ffmpeg` binary in the same directory as `nugs`. - ---- - -## Command Reference - -### Download Commands +### Runtime control ```bash -nugs ... # Download one or more albums -nugs grab latest # Download artist's latest shows -nugs full # Download artist's entire catalog -nugs -f # Override audio format -nugs -F # Override video format -nugs -o # Custom output directory +nugs status +nugs cancel +nugs completion bash +nugs completion zsh +nugs completion fish +nugs completion powershell ``` -### List Commands +### Structured output -```bash -nugs list # List all artists with media indicators -nugs list audio # List only artists with audio -nugs list video # List only artists with video -nugs list >100 # Filter artists by show count -nugs list <=50 # Operators: >, <, >=, <=, = -nugs list # List all artist's shows with 🎵🎬📹 -nugs list audio # List artist's audio shows only -nugs list video # List artist's video shows only -nugs list both # Shows with both formats -nugs list latest # List latest N shows -nugs list "venue" # Filter shows by venue name -nugs list --json # JSON output -nugs list --json # JSON output -``` +Commands that expose structured output accept `--json` with `minimal`, +`standard`, `extended`, or `raw` where documented. Do not assume mutation-only +commands support JSON; consult [Command reference](docs/COMMANDS.md). -**Examples:** +## Media formats -```bash -# List latest 5 shows from Billy Strings -nugs list 1125 latest 5 +Audio `-f` values: -# List Billy Strings video shows only -nugs list 1125 video +| Value | Format | +|---:|---| +| 1 | 16-bit 44.1 kHz ALAC | +| 2 | 16-bit 44.1 kHz FLAC | +| 3 | 24-bit 48 kHz MQA | +| 4 | 360 Reality Audio | +| 5 | 150 Kbps AAC | -# Find all Grateful Dead shows at Red Rocks -nugs list 461 "Red Rocks" +Video `-F` values: -# Find all Billy Strings video shows at Ryman -nugs list 1125 video "ryman" +| Value | Resolution | +|---:|---| +| 1 | 480p | +| 2 | 720p | +| 3 | 1080p | +| 4 | 1440p | +| 5 | 4K / best available | -# List artists with video content -nugs list video -``` - -### Catalog Commands +## Rclone safety -Short-form aliases (recommended): +- `outPath` and `videoOutPath` are local download directories. +- `rclonePath` and `rcloneVideoPath` are remote paths. +- When `deleteAfterUpload` is enabled, local deletion occurs only after upload + verification succeeds. -```bash -nugs update # Update catalog cache -nugs cache # View cache status -nugs stats # View statistics -nugs latest [limit] [audio|video|both] # View latest additions -nugs gaps [audio|video|both] # List missing shows -nugs gaps [...] --ids-only # IDs only (for piping) -nugs gaps [audio|video|both] fill # Auto-download missing shows (auth required) -nugs coverage [ids...] [audio|video|both] # Coverage stats -nugs refresh enable # Enable auto-refresh -nugs refresh disable # Disable auto-refresh -nugs refresh set # Configure auto-refresh (interactive) -``` +See [Configuration migrations](docs/CONFIG.md#migrations) if upgrading from a +version where `rclonePath` affected local paths. -> All short forms are equivalent to `nugs catalog ` (e.g. `nugs update` = `nugs catalog update`, `nugs refresh set` = `nugs catalog config set`). - -### Watch Commands - -```bash -nugs watch add # Add artist to watch list -nugs watch remove # Remove artist from watch list -nugs watch list # Show watched artists with names -nugs watch check [audio|video] # Update catalog + fill gaps for all watched artists (auth required) -nugs watch enable # Write systemd unit files and enable timer (Linux) -nugs watch disable # Stop timer and remove unit files -``` - -### Session Commands - -```bash -nugs status # Show status of any active download (PID, progress, current item) -nugs cancel # Cancel the active download session -``` - -### Global Options - -```bash --f, --format <1-5> Audio format (1=ALAC 2=FLAC 3=MQA 4=360RA 5=AAC) --F <1-5> Video format (1=480p 2=720p 3=1080p 4=1440p 5=4K) --o Output directory (overrides outPath in config) ---skip-chapters Skip chapter markers in video files ---json JSON output: minimal | standard | extended | raw ---help, -h Show help -``` - -> `--force-video` and `--skip-videos` are deprecated. Use `defaultOutputs: "video"` or `defaultOutputs: "audio"` in config instead. - -### Shell Completions - -Enable tab completion for commands, flags, and arguments in your shell: - -**Bash:** -```bash -# Install system-wide -sudo nugs completion bash > /etc/bash_completion.d/nugs - -# Or user-only -nugs completion bash > ~/.bash_completion.d/nugs -source ~/.bashrc -``` - -**Zsh (vanilla):** -```bash -# Create completion directory if needed -mkdir -p ~/.zsh/completion - -# Install completion -nugs completion zsh > ~/.zsh/completion/_nugs - -# Add to ~/.zshrc if not already present -echo 'fpath=(~/.zsh/completion $fpath)' >> ~/.zshrc -echo 'autoload -Uz compinit && compinit' >> ~/.zshrc - -# Reload shell -source ~/.zshrc -``` - -**Zsh (oh-my-zsh):** -```bash -# Install to oh-my-zsh custom directory -mkdir -p ~/.oh-my-zsh/custom/completions -nugs completion zsh > ~/.oh-my-zsh/custom/completions/_nugs - -# Add to your .zshrc BEFORE oh-my-zsh.sh is sourced -# (Add this line before the "source $ZSH/oh-my-zsh.sh" line) -fpath=($ZSH/custom/completions $fpath) - -# Reload shell -exec zsh -``` - -**Fish:** -```bash -# Install completion -nugs completion fish > ~/.config/fish/completions/nugs.fish - -# Reload completions -source ~/.config/fish/completions/nugs.fish -``` - -**PowerShell:** -```powershell -# Add to PowerShell profile -nugs completion powershell >> $PROFILE - -# Reload profile -. $PROFILE -``` - -**Completions include:** -- Top-level commands: `list`, `catalog`, `watch`, `status`, `cancel`, `completion`, `help` -- Short aliases: `update`, `cache`, `stats`, `latest`, `gaps`, `coverage`, `refresh`, `grab` -- Catalog subcommands: `update`, `cache`, `stats`, `latest`, `gaps`, `coverage`, `config`, `list` -- Watch subcommands: `add`, `remove`, `list`, `check`, `enable`, `disable` -- Config subcommands: `enable`, `disable`, `set` -- Media modifiers: `audio`, `video`, `both` -- Flags: `-f`, `-F`, `-o`, `--json`, `--skip-chapters` -- Format values: `1-5` for audio/video formats -- JSON levels: `minimal`, `standard`, `extended`, `raw` -- Shell types: `bash`, `zsh`, `fish`, `powershell` - ---- - -## Examples - -### Example 1: Download Billy Strings Shows - -```bash -# Latest shows only -nugs grab 1125 latest - -# Entire catalog (all 430+ shows) -nugs 1125 full - -# Specific show -nugs grab 23329 -``` - -### Example 2: Find and Download Missing Dead & Company Shows - -```bash -# See what you're missing -nugs gaps 1045 - -# Download all gaps -nugs gaps 1045 --ids-only | xargs -n1 nugs grab - -# Or just the 10 most recent -nugs gaps 1045 --ids-only | head -10 | xargs -n1 nugs grab -``` - -### Example 3: Batch Download from File - -```bash -# Create a file with show IDs -cat > shows.txt << EOF -23329 -23790 -24105 -EOF - -# Download all -nugs shows.txt -``` - -### Example 4: Find All Red Rocks Shows - -```bash -# Filter shows by venue (case-insensitive) -nugs list 1125 "Red Rocks" - -# Advanced: Use JSON output with jq -nugs list 1125 --json standard | \ - jq -r '.shows[] | select(.venue | contains("Red Rocks")) | .containerID' | \ - xargs -n1 nugs grab -``` - -### Example 5: Daily Catalog Update Script - -```bash -#!/bin/bash -# daily-catalog-update.sh - -# Update catalog -nugs update - -# View the stats -nugs stats -``` - -### Example 6: Check Collection Coverage - -```bash -# Get coverage stats for your favorite artists -nugs coverage - -# Check video coverage -nugs coverage video - -# Or check individual artists -nugs gaps 1125 # Shows coverage percentage -nugs gaps 461 -nugs gaps 1045 -``` - -### Example 7: Video-First Workflows - -**Configure for video preference:** - -```json -{ - "defaultOutputs": "video", - "videoFormat": 5, - "outPath": "/mnt/storage/nugs" -} -``` - -**Browse and download videos:** - -```bash -# Find artists with video content -nugs list video - -# View Billy Strings videos -nugs list 1125 video - -# Download latest videos -nugs grab 1125 latest video - -# Fill all video gaps -nugs gaps 1125 video fill - -# Check video coverage -nugs coverage 1125 video -``` - -### Example 8: Comprehensive Collection (Both Formats) - -**Download both audio and video:** - -```bash -# Single show - both formats -nugs grab 46201 both - -# Artist's latest - both formats -nugs grab 1125 latest both - -# Fill gaps for both formats -nugs gaps 1125 both fill - -# Find shows where you're missing either format -nugs gaps 1125 both -``` - -**Check what you have:** +## Development ```bash -# Overall coverage (audio) -nugs coverage 1125 audio - -# Video coverage -nugs coverage 1125 video - -# Shows with both formats downloaded -nugs coverage 1125 both +make test +make verify ``` ---- - -## Troubleshooting - -### Common Issues - -> **❌ Error:** "No cache found - run 'nugs update' first" -> -> **✅ Solution:** Run `nugs update` to download the catalog cache. - -> **❌ Error:** FFmpeg not found -> -> **✅ Solution:** -> - Install FFmpeg (see [FFmpeg Setup](#ffmpeg-setup)) -> - Or set `useFfmpegEnvVar: false` in config and place FFmpeg in the same directory as `nugs` - -> **❌ Error:** Auth failed / Invalid credentials -> -> **✅ Solution:** -> - Double-check your email/password in config.json -> - For Apple/Google accounts, use token authentication ([guide](token.md)) - -> **❌ Error:** "No audio available" -> -> **✅ Solution:** -> - The show might be video-only - try `--force-video` -> - Or it might not be available on your subscription tier - -> **❌ Error:** Rclone upload fails -> -> **✅ Solution:** -> - Verify rclone is installed: `rclone version` -> - Test your remote: `rclone ls :` -> - Check your config.json paths - -> **⚠️ Issue:** Gap detection shows wrong results -> -> **✅ Solution:** -> - Make sure `outPath` in config matches your actual download location -> - Run `nugs update` to refresh the catalog -> - Verify you haven't manually moved or renamed files - -### Getting Help - -1. Check this README first -2. Look at [Issues](https://github.com/jmagar/nugs-cli/issues) -3. Open a new issue with: - - Your OS and Go version - - Command you ran - - Full error message - - Relevant config (redact credentials) - ---- - -## Disclaimer - -- I will not be responsible for how you use Nugs CLI -- Nugs brand and name is the registered trademark of its respective owner -- Nugs CLI has no partnership, sponsorship or endorsement with Nugs.net -- This tool is for personal use only - respect copyright and terms of service -- Only download content you have legal access to through your subscription - ---- - -## Contributing - -Contributions welcome! Please open an issue first to discuss proposed changes. - -For development setup, architecture details, and coding guidelines, see [CLAUDE.md](CLAUDE.md). - -## Credits +`make verify` checks formatting, module integrity, vet, race tests, documentation +links/config-field coverage, static analysis, vulnerability analysis, and +supported cross-builds. CI runs the same contracts. -Originally forked from [Sorrow446/Nugs-Downloader](https://github.com/Sorrow446/Nugs-Downloader) +The source of truth for contributor architecture and workflow is +[CLAUDE.md](CLAUDE.md). Detailed references: -Catalog caching, auto-refresh, gap detection, and modern improvements by [jmagar](https://github.com/jmagar) +- [Commands](docs/COMMANDS.md) +- [Configuration](docs/CONFIG.md) +- [Architecture](docs/ARCHITECTURE.md) +- [Upstream API notes](docs/nugs-api-endpoints.md) +- [CLI quick reference](docs/QUICK_REFERENCE.md) -## License +## Release integrity -No standalone license file is present in this checkout. See the repository -history and upstream fork lineage before redistributing binaries or modified -source. +Release automation publishes archives for Linux, macOS, and Windows, a +`checksums.txt` file, and GitHub build provenance. CI smoke-tests each native +binary before publication. The `nugs version` output includes version, commit, +and build date for rollback and incident diagnosis. -## Related Servers +## Safety and legal use -- `unifi-rmcp / rustifi` - UniFi controller REST API bridge. -- `tailscale-rmcp / rustscale` - Tailscale API bridge for devices, users, and tailnet operations. -- `unraid-rmcp / unrust` - Unraid GraphQL bridge for NAS and server management. -- `apprise-rmcp` - Apprise notification fan-out bridge for many delivery backends. -- `gotify-rmcp` - Gotify push notification bridge for sends, messages, apps, and clients. -- `arcane-rmcp` - Arcane Docker management bridge for containers and related resources. -- `yarr-rmcp` - Media-stack bridge for Sonarr, Radarr, Prowlarr, Plex, and related services. -- `ytdl-mcp` - Media download and metadata workflow server. -- `synapse` - Local Synapse workflow server for scout and flux actions. -- `cortex` - Syslog and homelab log aggregation MCP server. -- `axon` - RAG, crawl, scrape, extract, and semantic search project. -- `lab` - Homelab control plane and Labby gateway project. -- `lumen` - Local semantic code search MCP server. -- `agentcast` - Agent transcript and activity publishing project. -- `soma` - RMCP scaffold/runtime template for new provider-backed servers. +Use the CLI only for media your account is authorized to access. Do not share +credentials, tokens, signed media URLs, or downloaded content. This project is +not affiliated with Nugs.net. diff --git a/cmd/nugs/api_client.go b/cmd/nugs/api_client.go index 7fe49c4..1944fc2 100644 --- a/cmd/nugs/api_client.go +++ b/cmd/nugs/api_client.go @@ -1,12 +1,10 @@ package main -// API client wrappers delegating to internal/api during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Root orchestration adapters that combine API and cache behavior. import ( "context" "fmt" - "net/http" "os" "strings" "time" @@ -16,26 +14,6 @@ import ( "github.com/jmagar/nugs-cli/internal/model" ) -const ( - devKey = api.DevKey - clientId = api.ClientID - layout = api.Layout - userAgent = api.UserAgent - userAgentTwo = api.UserAgentTwo - authUrl = api.AuthURL - streamApiBase = api.StreamAPIBase - subInfoUrl = api.SubInfoURL - userInfoUrl = api.UserInfoURL - playerUrl = api.PlayerURL -) - -var ( - jar = api.Jar - client = api.Client -) - -var qualityMap = api.QualityMap - func auth(ctx context.Context, email, pwd string) (string, error) { return api.Auth(ctx, email, pwd) } @@ -49,31 +27,6 @@ func getPlan(subInfo *SubInfo) (string, bool) { return api.GetPlan(subInfo) } func extractLegToken(tokenStr string) (string, string, error) { return api.ExtractLegToken(tokenStr) } -func getAlbumMeta(ctx context.Context, albumId string) (*AlbumMeta, error) { - return api.GetAlbumMeta(ctx, albumId) -} -func getPlistMeta(ctx context.Context, plistId, email, legacyToken string, cat bool) (*PlistMeta, error) { - return api.GetPlistMeta(ctx, plistId, email, legacyToken, cat) -} -func getLatestCatalog(ctx context.Context) (*LatestCatalogResp, error) { - return api.GetLatestCatalog(ctx) -} -func getArtistMeta(ctx context.Context, artistId string) ([]*ArtistMeta, error) { - return api.GetArtistMeta(ctx, artistId) -} -func getArtistList(ctx context.Context) (*ArtistListResp, error) { - return api.GetArtistList(ctx) -} -func getPurchasedManUrl(ctx context.Context, skuID int, showID, userID, uguID string) (string, error) { - return api.GetPurchasedManURL(ctx, skuID, showID, userID, uguID) -} -func getStreamMeta(ctx context.Context, trackId, skuId, format int, streamParams *StreamParams) (string, error) { - return api.GetStreamMeta(ctx, trackId, skuId, format, streamParams) -} -func queryQuality(streamUrl string) *Quality { return api.QueryQuality(streamUrl) } -func getTrackQual(quals []*Quality, wantFmt int) *Quality { - return api.GetTrackQual(quals, wantFmt) -} // preorderCacheTTL is the reduced TTL applied when cached metadata contains any // non-AVAILABLE shows (e.g. PREORDER). Keeps the window short so a show that @@ -105,7 +58,7 @@ func getArtistMetaCached(ctx context.Context, artistID string, ttl time.Duration // availType=2 returns only PREORDER shows (upcoming, not yet downloadable). // Using availType=2 causes IsShowDownloadable to filter everything out (all shows // are PREORDER), making gaps detection always report "no missing shows." - const availType = 1 // Fetch AVAILABLE catalog — the only value that returns downloadable shows + const availType = model.AvailableCatalogView // AVAILABLE is the only downloadable catalog view. cachedPages, cachedAt, readErr := readArtistMetaCache(artistID) if readErr == nil && len(cachedPages) > 0 { @@ -157,6 +110,3 @@ func getArtistListCached(ctx context.Context, ttl time.Duration) (*model.ArtistL } return nil, fetchErr } - -// Re-export for callers that reference the client directly. -func getHTTPClient() *http.Client { return api.Client } diff --git a/cmd/nugs/batch.go b/cmd/nugs/batch.go index a86673c..f7c8b1e 100644 --- a/cmd/nugs/batch.go +++ b/cmd/nugs/batch.go @@ -1,7 +1,6 @@ package main -// Batch wrappers delegating to internal/download during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for batch downloads. import ( "context" diff --git a/cmd/nugs/cache.go b/cmd/nugs/cache.go index 3dcf696..760878e 100644 --- a/cmd/nugs/cache.go +++ b/cmd/nugs/cache.go @@ -1,7 +1,6 @@ package main -// Cache wrappers delegating to internal/cache during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for cache operations. import ( "time" diff --git a/cmd/nugs/cancel_unix.go b/cmd/nugs/cancel_unix.go index ea43fc5..bb325fc 100644 --- a/cmd/nugs/cancel_unix.go +++ b/cmd/nugs/cancel_unix.go @@ -2,8 +2,7 @@ package main -// Cancel unix wrappers delegating to internal/runtime during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Unix cancellation adapters for internal runtime control. import "github.com/jmagar/nugs-cli/internal/runtime" diff --git a/cmd/nugs/cancel_windows.go b/cmd/nugs/cancel_windows.go index 8e503dc..75cb5eb 100644 --- a/cmd/nugs/cancel_windows.go +++ b/cmd/nugs/cancel_windows.go @@ -2,8 +2,7 @@ package main -// Cancel windows wrappers delegating to internal/runtime during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Windows cancellation adapters for internal runtime control. import "github.com/jmagar/nugs-cli/internal/runtime" diff --git a/cmd/nugs/catalog_analysis_mediaaware.go b/cmd/nugs/catalog_analysis_mediaaware.go index bb15e61..6ed6cc8 100644 --- a/cmd/nugs/catalog_analysis_mediaaware.go +++ b/cmd/nugs/catalog_analysis_mediaaware.go @@ -1,7 +1,6 @@ package main -// Media-aware catalog analysis wrappers delegating to internal/catalog during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for media-aware catalog analysis. import "github.com/jmagar/nugs-cli/internal/catalog" diff --git a/cmd/nugs/catalog_autorefresh.go b/cmd/nugs/catalog_autorefresh.go index 66e9096..b2f893a 100644 --- a/cmd/nugs/catalog_autorefresh.go +++ b/cmd/nugs/catalog_autorefresh.go @@ -1,7 +1,6 @@ package main -// Auto-refresh wrappers delegating to internal/catalog during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for catalog auto-refresh. import ( "context" diff --git a/cmd/nugs/catalog_handlers.go b/cmd/nugs/catalog_handlers.go index 6a8d85c..84ac047 100644 --- a/cmd/nugs/catalog_handlers.go +++ b/cmd/nugs/catalog_handlers.go @@ -1,7 +1,6 @@ package main -// Catalog handler wrappers delegating to internal/catalog during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for catalog operations. import ( "context" @@ -69,4 +68,3 @@ func catalogCoverage(ctx context.Context, artistIds []string, cfg *Config, jsonL func catalogList(ctx context.Context, artistIds []string, cfg *Config, jsonLevel string, mediaFilter MediaType) error { return catalog.CatalogList(ctx, artistIds, cfg, jsonLevel, mediaFilter, buildCatalogDeps()) } - diff --git a/cmd/nugs/cli_integration_test.go b/cmd/nugs/cli_integration_test.go new file mode 100644 index 0000000..62ea0f5 --- /dev/null +++ b/cmd/nugs/cli_integration_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "bytes" + "errors" + "os" + "os/exec" + "strings" + "testing" +) + +const cliHelperEnv = "NUGS_CLI_INTEGRATION_HELPER" + +func TestCLIProcessHelper(t *testing.T) { + if os.Getenv(cliHelperEnv) != "1" { + return + } + for i, arg := range os.Args { + if arg == "--" { + os.Args = append([]string{"nugs"}, os.Args[i+1:]...) + os.Exit(realMain()) + } + } + os.Exit(2) +} + +func runCLIProcess(t *testing.T, args ...string) (string, string, int) { + t.Helper() + cmdArgs := append([]string{"-test.run=^TestCLIProcessHelper$", "--"}, args...) + cmd := exec.Command(os.Args[0], cmdArgs...) + cmd.Env = append(os.Environ(), cliHelperEnv+"=1", "HOME="+t.TempDir()) + cmd.Dir = t.TempDir() + var stdout, stderr bytes.Buffer + cmd.Stdout = &stdout + cmd.Stderr = &stderr + err := cmd.Run() + if err == nil { + return stdout.String(), stderr.String(), 0 + } + var exitErr *exec.ExitError + if !errors.As(err, &exitErr) { + t.Fatalf("run CLI helper: %v", err) + } + return stdout.String(), stderr.String(), exitErr.ExitCode() +} + +func TestCLIProcessContracts(t *testing.T) { + t.Run("version bypasses configuration and authentication", func(t *testing.T) { + t.Parallel() + stdout, stderr, code := runCLIProcess(t, "version") + if code != 0 || stderr != "" || !strings.Contains(stdout, "\nnugs ") { + t.Fatalf("version = code %d stdout %q stderr %q", code, stdout, stderr) + } + }) + + t.Run("unknown input fails without success output", func(t *testing.T) { + t.Parallel() + stdout, stderr, code := runCLIProcess(t, "not-a-nugs-url") + if code == 0 { + t.Fatalf("unknown input exit code = 0, stdout %q stderr %q", stdout, stderr) + } + if strings.Contains(strings.ToLower(stdout), "complete") { + t.Fatalf("failure emitted completion output: %q", stdout) + } + }) +} diff --git a/cmd/nugs/completions.go b/cmd/nugs/completions.go index 0b160e8..3b2f457 100644 --- a/cmd/nugs/completions.go +++ b/cmd/nugs/completions.go @@ -1,7 +1,6 @@ package main -// Shell completion wrappers delegating to internal/completion during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapter for shell completion generation. import "github.com/jmagar/nugs-cli/internal/completion" diff --git a/cmd/nugs/config.go b/cmd/nugs/config.go index b27aff9..b7109bb 100644 --- a/cmd/nugs/config.go +++ b/cmd/nugs/config.go @@ -1,7 +1,6 @@ package main -// Config wrappers delegating to internal/config during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for configuration loading. import "github.com/jmagar/nugs-cli/internal/config" diff --git a/cmd/nugs/config_ffmpeg_test.go b/cmd/nugs/config_ffmpeg_test.go index 9150e6a..0114c15 100644 --- a/cmd/nugs/config_ffmpeg_test.go +++ b/cmd/nugs/config_ffmpeg_test.go @@ -8,7 +8,7 @@ import ( "github.com/jmagar/nugs-cli/internal/testutil" ) -func TestResolveFfmpegBinary_LocalPreferred(t *testing.T) { +func TestResolveFfmpegBinary_DoesNotImplicitlyUseCurrentDirectory(t *testing.T) { tmp := testutil.ChdirTemp(t) local := filepath.Join(tmp, "ffmpeg") testutil.WriteExecutable(t, local) @@ -16,12 +16,23 @@ func TestResolveFfmpegBinary_LocalPreferred(t *testing.T) { t.Setenv("PATH", "") cfg := &Config{UseFfmpegEnvVar: false} - got, err := resolveFfmpegBinary(cfg) + if _, err := resolveFfmpegBinary(cfg); err == nil { + t.Fatal("resolveFfmpegBinary implicitly trusted ./ffmpeg") + } +} + +func TestResolveFfmpegBinary_ExplicitAbsolutePath(t *testing.T) { + tmp := t.TempDir() + local := filepath.Join(tmp, "ffmpeg") + testutil.WriteExecutable(t, local) + t.Setenv("PATH", "") + + got, err := resolveFfmpegBinary(&Config{FfmpegNameStr: local}) if err != nil { t.Fatalf("resolveFfmpegBinary returned error: %v", err) } - if got != "./ffmpeg" { - t.Fatalf("expected ./ffmpeg, got %q", got) + if got != local { + t.Fatalf("got %q, want %q", got, local) } } diff --git a/cmd/nugs/detach_common.go b/cmd/nugs/detach_common.go index 8020eda..b780a63 100644 --- a/cmd/nugs/detach_common.go +++ b/cmd/nugs/detach_common.go @@ -1,7 +1,6 @@ package main -// Detach wrappers delegating to internal/runtime during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for detached runtime execution. import "github.com/jmagar/nugs-cli/internal/runtime" diff --git a/cmd/nugs/detach_unix.go b/cmd/nugs/detach_unix.go index 8d2a281..2b7f440 100644 --- a/cmd/nugs/detach_unix.go +++ b/cmd/nugs/detach_unix.go @@ -2,8 +2,7 @@ package main -// Detach unix wrappers delegating to internal/runtime during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Unix adapters for detached runtime execution. import "github.com/jmagar/nugs-cli/internal/runtime" diff --git a/cmd/nugs/detach_windows.go b/cmd/nugs/detach_windows.go index e0dd337..c4d618b 100644 --- a/cmd/nugs/detach_windows.go +++ b/cmd/nugs/detach_windows.go @@ -2,8 +2,7 @@ package main -// Detach windows wrappers delegating to internal/runtime during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Windows adapters for detached runtime execution. import "github.com/jmagar/nugs-cli/internal/runtime" diff --git a/cmd/nugs/download.go b/cmd/nugs/download.go index d805841..3bca805 100644 --- a/cmd/nugs/download.go +++ b/cmd/nugs/download.go @@ -1,7 +1,6 @@ package main -// Download wrappers delegating to internal/download during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for the internal download engine. import ( "context" diff --git a/cmd/nugs/filelock.go b/cmd/nugs/filelock.go index a266bc0..12287d2 100644 --- a/cmd/nugs/filelock.go +++ b/cmd/nugs/filelock.go @@ -1,7 +1,6 @@ package main -// Filelock wrappers delegating to internal/cache during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command aliases for cache locking. import "github.com/jmagar/nugs-cli/internal/cache" diff --git a/cmd/nugs/format.go b/cmd/nugs/format.go index c6aa9f2..6fd3771 100644 --- a/cmd/nugs/format.go +++ b/cmd/nugs/format.go @@ -1,10 +1,6 @@ package main -// Format wrappers delegating to internal/ui during migration. -// Migration plan: Phase 12 will move renderProgressBox and renderCompletionSummary -// into internal/ui, eliminating the aliases below. These functions remain here -// because they depend on root-level runtime callbacks (updateRuntimeProgress) -// and package-level color variables from structs.go. +// Command-specific progress rendering with runtime status callbacks. import ( "fmt" @@ -16,49 +12,24 @@ import ( "github.com/jmagar/nugs-cli/internal/ui" ) -const defaultProgressRenderInterval = model.DefaultProgressRenderInterval - -// Box drawing character wrappers var ( - boxTopLeft = ui.BoxTopLeft - boxTopRight = ui.BoxTopRight - boxBottomLeft = ui.BoxBottomLeft - boxBottomRight = ui.BoxBottomRight - boxVertical = ui.BoxVertical - boxHorizontal = ui.BoxHorizontal - boxTeeLeft = ui.BoxTeeLeft - boxTeeRight = ui.BoxTeeRight - boxTeeTop = ui.BoxTeeTop - boxTeeBottom = ui.BoxTeeBottom - boxCross = ui.BoxCross + boxVertical = ui.BoxVertical + boxHorizontal = ui.BoxHorizontal + boxTeeLeft = ui.BoxTeeLeft + boxTeeRight = ui.BoxTeeRight boxDoubleHorizontal = ui.BoxDoubleHorizontal boxDoubleTopLeft = ui.BoxDoubleTopLeft boxDoubleTopRight = ui.BoxDoubleTopRight boxDoubleBottomLeft = ui.BoxDoubleBottomLeft boxDoubleBottomRight = ui.BoxDoubleBottomRight - - bulletSquare = ui.BulletSquare - bulletCircle = ui.BulletCircle - bulletArrow = ui.BulletArrow - bulletDiamond = ui.BulletDiamond ) -// ansiRegex delegates to ui.AnsiRegex -var ansiRegex = ui.AnsiRegex - func getTermWidth() int { return ui.GetTermWidth() } -func stripAnsiCodes(s string) string { return ui.StripAnsiCodes(s) } -func visibleLength(s string) int { return ui.VisibleLength(s) } func truncateWithEllipsis(s string, maxLen int) string { return ui.TruncateWithEllipsis(s, maxLen) } func padRight(s string, width int) string { return ui.PadRight(s, width) } -func padCenter(s string, width int) string { return ui.PadCenter(s, width) } -func printHeader(title string) { ui.PrintHeader(title) } func printSection(title string) { ui.PrintSection(title) } -func printList(items []string, color string) { ui.PrintList(items, color) } func printKeyValue(key, value, valueColor string) { ui.PrintKeyValue(key, value, valueColor) } -func printDivider() { ui.PrintDivider() } -func printBox(text string, borderColor string) { ui.PrintBox(text, borderColor) } // Table type aliases type TableColumn = ui.TableColumn diff --git a/cmd/nugs/helpers.go b/cmd/nugs/helpers.go index b89f9df..e6fc7a6 100644 --- a/cmd/nugs/helpers.go +++ b/cmd/nugs/helpers.go @@ -1,14 +1,12 @@ package main -// Thin wrappers delegating to internal/helpers during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for internal helper operations. import "github.com/jmagar/nugs-cli/internal/helpers" -func handleErr(errText string, err error, _panic bool) { helpers.HandleErr(errText, err, _panic) } -func makeDirs(path string) error { return helpers.MakeDirs(path) } -func sanitise(filename string) string { return helpers.Sanitise(filename) } -func getVideoOutPath(cfg *Config) string { return helpers.GetVideoOutPath(cfg) } +func makeDirs(path string) error { return helpers.MakeDirs(path) } +func sanitise(filename string) string { return helpers.Sanitise(filename) } +func getVideoOutPath(cfg *Config) string { return helpers.GetVideoOutPath(cfg) } func getRcloneBasePath(cfg *Config, isVideo bool) string { return helpers.GetRcloneBasePath(cfg, isVideo) } diff --git a/cmd/nugs/hotkey_input.go b/cmd/nugs/hotkey_input.go index ec4e565..c5f3393 100644 --- a/cmd/nugs/hotkey_input.go +++ b/cmd/nugs/hotkey_input.go @@ -1,7 +1,6 @@ package main -// Hotkey input wrapper delegating to internal/runtime during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapter for runtime hotkey input. import "github.com/jmagar/nugs-cli/internal/runtime" diff --git a/cmd/nugs/list_commands.go b/cmd/nugs/list_commands.go index 648566b..957758f 100644 --- a/cmd/nugs/list_commands.go +++ b/cmd/nugs/list_commands.go @@ -1,7 +1,6 @@ package main -// List command wrappers delegating to internal/list during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for listing operations. import ( "context" diff --git a/cmd/nugs/main.go b/cmd/nugs/main.go index 72bc538..cdfa2b8 100644 --- a/cmd/nugs/main.go +++ b/cmd/nugs/main.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "os" "os/signal" @@ -20,6 +21,9 @@ func init() { if slices.Contains(os.Args, "--json") { return } + if isVersionRequest(os.Args[1:]) { + return + } // Suppress banner for completion command to avoid breaking shell completion parsing if len(os.Args) > 1 && os.Args[1] == "completion" { return @@ -33,13 +37,28 @@ func init() { } func main() { - cfg, jsonLevel := bootstrap() - run(cfg, jsonLevel) + os.Exit(realMain()) +} + +func realMain() int { + if isVersionRequest(os.Args[1:]) { + printVersion(os.Stdout) + return 0 + } + cfg, jsonLevel, err := bootstrap() + if err == nil { + err = run(cfg, jsonLevel) + } + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + return 0 } // bootstrap handles early setup: session persistence, working directory, // config file detection, JSON flag parsing, and config/arg parsing. -func bootstrap() (*Config, string) { +func bootstrap() (*Config, string, error) { setupSessionPersistence() configExists := false @@ -59,7 +78,7 @@ func bootstrap() (*Config, string) { if !configExists { err := promptForConfig() if err != nil { - handleErr("Failed to create config.", err, true) + return nil, "", fmt.Errorf("failed to create config: %w", err) } } @@ -74,9 +93,7 @@ func bootstrap() (*Config, string) { for i := 0; i < len(os.Args); i++ { if os.Args[i] == "--json" { if i+1 >= len(os.Args) { - fmt.Println("Error: --json flag requires a level argument (minimal, standard, extended, raw)") - printInfo("Usage: nugs list artists --json ") - os.Exit(1) + return nil, "", errors.New("--json flag requires a level argument (minimal, standard, extended, raw)") } jsonLevel = os.Args[i+1] os.Args = append(os.Args[:i], os.Args[i+2:]...) @@ -86,13 +103,12 @@ func bootstrap() (*Config, string) { // Validate json level if jsonLevel != "" && jsonLevel != JSONLevelMinimal && jsonLevel != JSONLevelStandard && jsonLevel != JSONLevelExtended && jsonLevel != JSONLevelRaw { - fmt.Printf("Invalid JSON level: %s. Valid options: %s, %s, %s, %s\n", jsonLevel, JSONLevelMinimal, JSONLevelStandard, JSONLevelExtended, JSONLevelRaw) - os.Exit(1) + return nil, "", fmt.Errorf("invalid JSON level %q: valid options are %s, %s, %s, %s", jsonLevel, JSONLevelMinimal, JSONLevelStandard, JSONLevelExtended, JSONLevelRaw) } cfg, err := parseCfg() if err != nil { - handleErr("Failed to parse config/args.", err, true) + return nil, "", fmt.Errorf("failed to parse config/args: %w", err) } cfg.Urls = normalizeCliAliases(cfg.Urls) printActiveRuntimeHint(os.Getpid(), cfg.Urls) @@ -106,48 +122,43 @@ func bootstrap() (*Config, string) { } } - return cfg, jsonLevel + return cfg, jsonLevel, nil } // run is the main orchestration function: handles early-exit commands (detach, status, // cancel, completion), runtime tracking, environment setup, command routing (list, // catalog, artist shortcuts), authentication, and dispatching URL downloads. -func run(cfg *Config, jsonLevel string) { +func run(cfg *Config, jsonLevel string) (runErr error) { if maybeDetachAndExit(os.Args[1:], cfg.Urls) { - return + return nil } if len(cfg.Urls) == 1 && cfg.Urls[0] == "status" { printRuntimeStatus() - return + return nil } if len(cfg.Urls) == 1 && cfg.Urls[0] == "cancel" { status, err := readRuntimeStatus() if err != nil { printWarning("No active crawl status found") - return + return nil } if status.State != "running" { printWarning(fmt.Sprintf("No running crawl found (state: %s)", status.State)) - return + return nil } if err := requestRuntimeCancel(); err != nil { - handleErr("Failed to request crawl cancellation.", err, false) - return + return fmt.Errorf("failed to request crawl cancellation: %w", err) } _ = cancelProcessByPID(status.PID) printSuccess(fmt.Sprintf("Cancellation requested for crawl pid=%d", status.PID)) - return + return nil } // Completion command - generate shell completion scripts if len(cfg.Urls) > 0 && cfg.Urls[0] == "completion" { - err := completionCommand(cfg.Urls) - if err != nil { - handleErr("Completion command failed.", err, true) - } - return + return completionCommand(cfg.Urls) } trackRuntime := !isReadOnlyCommand(cfg.Urls) @@ -163,6 +174,10 @@ func run(cfg *Config, jsonLevel string) { finalizeRuntimeStatus("cancelled") return } + if runErr != nil { + finalizeRuntimeStatus("failed") + return + } finalizeRuntimeStatus("completed") }() stopHotkeys := startCrawlHotkeysIfNeeded(cfg.Urls) @@ -181,7 +196,7 @@ func run(cfg *Config, jsonLevel string) { if cfg.RcloneEnabled { err = checkRcloneAvailable(jsonLevel != "") if err != nil { - handleErr("Rclone check failed.", err, true) + return fmt.Errorf("rclone check failed: %w", err) } } printStartupEnvironment(cfg, jsonLevel) @@ -192,52 +207,71 @@ func run(cfg *Config, jsonLevel string) { if err != nil { fmt.Printf("Error displaying welcome screen: %v\n", err) } - return + return err } // Route list and catalog commands (pre-auth) - if handleListCommand(ctx, cfg, jsonLevel) { - return + if handled, err := handleListCommand(ctx, cfg, jsonLevel); handled { + return err } - if handleCatalogCommand(ctx, cfg, jsonLevel) { - return + if handled, err := handleCatalogCommand(ctx, cfg, jsonLevel); handled { + return err } - if handleWatchCommand(ctx, cfg, jsonLevel) { - return + if handled, err := handleWatchCommand(ctx, cfg, jsonLevel); handled { + return err } // Handle " latest/full" shorthand if len(cfg.Urls) == 2 || len(cfg.Urls) == 3 { - if handled := handleArtistShorthand(cfg); handled { - return + if handled, err := handleArtistShorthand(cfg); handled { + return err } } - // Authenticate - var token string - err = makeDirs(cfg.OutPath) + streamParams, legacyToken, uguID, err := authenticateForDownloads(ctx, cfg) if err != nil { - handleErr("Failed to make output folder.", err, true) + return err } + + // Handle "catalog gaps [...] fill" (requires auth) + if len(cfg.Urls) >= 4 && cfg.Urls[0] == "catalog" && cfg.Urls[1] == "gaps" && cfg.Urls[len(cfg.Urls)-1] == "fill" { + return handleCatalogGapsFill(ctx, cfg, streamParams, jsonLevel) + } + + // Handle "watch check" (requires auth) + if handled, err := handleWatchCheckCommand(ctx, cfg, streamParams, jsonLevel); handled { + return err + } + + runCancelled, runErr = dispatch(ctx, cfg, streamParams, legacyToken, uguID) + return runErr +} + +func authenticateForDownloads(ctx context.Context, cfg *Config) (*StreamParams, string, string, error) { + if err := makeDirs(cfg.OutPath); err != nil { + return nil, "", "", fmt.Errorf("failed to make output folder: %w", err) + } + var token string + var err error if cfg.Token == "" { token, err = auth(ctx, cfg.Email, cfg.Password) if err != nil { - handleErr("Failed to auth.", err, true) + return nil, "", "", fmt.Errorf("failed to authenticate: %w", err) } } else { token = cfg.Token } userId, err := getUserInfo(ctx, token) if err != nil { - handleErr("Failed to get user info.", err, true) + return nil, "", "", fmt.Errorf("failed to get user info: %w", err) } subInfo, err := getSubInfo(ctx, token) if err != nil { - handleErr("Failed to get subscription info.", err, true) + return nil, "", "", fmt.Errorf("failed to get subscription info: %w", err) } legacyToken, uguID, err := extractLegToken(token) if err != nil { - handleErr("Failed to extract legacy token.", err, true) + return nil, "", "", fmt.Errorf("failed to extract legacy token: %w", err) } planDesc, isPromo := getPlan(subInfo) if !subInfo.IsContentAccessible { @@ -246,34 +280,22 @@ func run(cfg *Config, jsonLevel string) { printSuccess(fmt.Sprintf("Signed in - %s%s%s", colorCyan, planDesc, colorReset)) streamParams, err := parseStreamParams(userId, subInfo, isPromo) if err != nil { - handleErr("Failed to parse subscription timestamps.", err, true) + return nil, "", "", fmt.Errorf("failed to parse subscription timestamps: %w", err) } - - // Handle "catalog gaps [...] fill" (requires auth) - if len(cfg.Urls) >= 4 && cfg.Urls[0] == "catalog" && cfg.Urls[1] == "gaps" && cfg.Urls[len(cfg.Urls)-1] == "fill" { - handleCatalogGapsFill(ctx, cfg, streamParams, jsonLevel) - return - } - - // Handle "watch check" (requires auth) - if handleWatchCheckCommand(ctx, cfg, streamParams, jsonLevel) { - return - } - - runCancelled = dispatch(ctx, cfg, streamParams, legacyToken, uguID) + return streamParams, legacyToken, uguID, nil } // handleListCommand routes "list" subcommands. Returns true if handled. -func handleListCommand(ctx context.Context, cfg *Config, jsonLevel string) bool { +func handleListCommand(ctx context.Context, cfg *Config, jsonLevel string) (bool, error) { if len(cfg.Urls) == 0 || cfg.Urls[0] != "list" { - return false + return false, nil } if len(cfg.Urls) < 2 { printInfo("Usage: nugs list artists | list [shows \"venue\" | latest ]") fmt.Println(" list ") fmt.Println(" list [audio|video|both]") fmt.Println(" list [\"venue\" | latest ]") - return true + return true, nil } subCmd := cfg.Urls[1] @@ -291,15 +313,12 @@ func handleListCommand(ctx context.Context, cfg *Config, jsonLevel string) bool fmt.Println(" list <=50") fmt.Println(" list =25") fmt.Println("Operators: >, <, >=, <=, =") - return true + return true, nil } showFilter = remainingArgs[1] } err := listArtists(ctx, jsonLevel, showFilter) - if err != nil { - handleErr("List artists failed.", err, true) - } - return true + return true, wrapCommandError("list artists", err) } artistId := subCmd @@ -314,14 +333,11 @@ func handleListCommand(ctx context.Context, cfg *Config, jsonLevel string) bool printInfo("Usage: nugs list shows \"\"") fmt.Println("Or: list shows \"\"") fmt.Println("Example: list 461 \"Red Rocks\"") - return true + return true, nil } venueFilter := strings.Join(remainingArgs[1:], " ") err := listArtistShowsByVenue(ctx, artistId, venueFilter, jsonLevel) - if err != nil { - handleErr("List shows by venue failed.", err, true) - } - return true + return true, wrapCommandError("list shows by venue", err) } // Check for latest N: list latest @@ -331,24 +347,25 @@ func handleListCommand(ctx context.Context, cfg *Config, jsonLevel string) bool if parsedLimit, parseErr := strconv.Atoi(remainingArgs[1]); parseErr == nil { if parsedLimit < 1 { fmt.Println("Error: limit must be a positive number (got", parsedLimit, ")") - return true + return true, nil } limit = parsedLimit } } err := listArtistLatestShows(ctx, artistId, limit, jsonLevel) - if err != nil { - handleErr("List latest shows failed.", err, true) - } - return true + return true, wrapCommandError("list latest shows", err) } // Default: list all shows for artist (with optional media filter) err := listArtistShows(ctx, artistId, jsonLevel, mediaFilter) - if err != nil { - handleErr("List shows failed.", err, true) + return true, wrapCommandError("list shows", err) +} + +func wrapCommandError(command string, err error) error { + if err == nil { + return nil } - return true + return fmt.Errorf("%s failed: %w", command, err) } // handleCatalogCommand routes pre-auth "catalog" subcommands. Returns true if handled. @@ -368,14 +385,14 @@ func parseMediaModifier(args []string) (MediaType, []string) { return MediaTypeUnknown, args } -func handleCatalogCommand(ctx context.Context, cfg *Config, jsonLevel string) bool { +func handleCatalogCommand(ctx context.Context, cfg *Config, jsonLevel string) (bool, error) { if len(cfg.Urls) == 0 || cfg.Urls[0] != "catalog" { - return false + return false, nil } // "catalog gaps ... fill" requires auth — skip here, handled post-auth in run() isCatalogGapsFill := len(cfg.Urls) >= 4 && cfg.Urls[1] == "gaps" && cfg.Urls[len(cfg.Urls)-1] == "fill" if isCatalogGapsFill { - return false + return false, nil } if len(cfg.Urls) < 2 { @@ -390,26 +407,17 @@ func handleCatalogCommand(ctx context.Context, cfg *Config, jsonLevel string) bo fmt.Println(" catalog config enable|disable|set") fmt.Println() fmt.Println(" Note: catalog = cached/offline, list = live API") - return true + return true, nil } subCmd := cfg.Urls[1] switch subCmd { case "update": - err := catalogUpdate(ctx, jsonLevel) - if err != nil { - handleErr("Catalog update failed.", err, true) - } + return true, wrapCommandError("catalog update", catalogUpdate(ctx, jsonLevel)) case "cache": - err := catalogCacheStatus(jsonLevel) - if err != nil { - handleErr("Catalog cache status failed.", err, true) - } + return true, wrapCommandError("catalog cache status", catalogCacheStatus(jsonLevel)) case "stats": - err := catalogStats(ctx, cfg, jsonLevel) - if err != nil { - handleErr("Catalog stats failed.", err, true) - } + return true, wrapCommandError("catalog stats", catalogStats(ctx, cfg, jsonLevel)) case "latest": limit := 15 argsAfterLatest := []string{} @@ -428,20 +436,17 @@ func handleCatalogCommand(ctx context.Context, cfg *Config, jsonLevel string) bo if parsedLimit, err := strconv.Atoi(remainingArgs[0]); err == nil { if parsedLimit < 1 { fmt.Println("Error: limit must be a positive number (got", parsedLimit, ")") - return true + return true, nil } limit = parsedLimit } } - err := catalogLatest(ctx, limit, jsonLevel) - if err != nil { - handleErr("Catalog latest failed.", err, true) - } + return true, wrapCommandError("catalog latest", catalogLatest(ctx, limit, jsonLevel)) case "gaps": if len(cfg.Urls) < 3 { printInfo("Usage: nugs catalog gaps [...] [audio|video|both] [fill]") fmt.Println(" catalog gaps [...] [audio|video|both] --ids-only") - return true + return true, nil } // Extract media modifier from args after "catalog gaps" @@ -461,26 +466,20 @@ func handleCatalogCommand(ctx context.Context, cfg *Config, jsonLevel string) bo if len(artistIds) == 0 { fmt.Println("Error: No artist IDs provided") - return true - } - err := catalogGaps(ctx, artistIds, cfg, jsonLevel, idsOnly, mediaFilter) - if err != nil { - handleErr("Catalog gaps failed.", err, true) + return true, nil } + return true, wrapCommandError("catalog gaps", catalogGaps(ctx, artistIds, cfg, jsonLevel, idsOnly, mediaFilter)) case "list": if len(cfg.Urls) < 3 { printInfo("Usage: nugs catalog list [...] [audio|video|both]") - return true + return true, nil } // Extract media modifier from args after "catalog list" argsAfterList := cfg.Urls[2:] mediaFilter, artistIds := parseMediaModifier(argsAfterList) - err := catalogList(ctx, artistIds, cfg, jsonLevel, mediaFilter) - if err != nil { - handleErr("Catalog list failed.", err, true) - } + return true, wrapCommandError("catalog list", catalogList(ctx, artistIds, cfg, jsonLevel, mediaFilter)) case "coverage": argsAfterCoverage := []string{} if len(cfg.Urls) > 2 { @@ -490,48 +489,36 @@ func handleCatalogCommand(ctx context.Context, cfg *Config, jsonLevel string) bo // Extract media modifier mediaFilter, artistIds := parseMediaModifier(argsAfterCoverage) - err := catalogCoverage(ctx, artistIds, cfg, jsonLevel, mediaFilter) - if err != nil { - handleErr("Catalog coverage failed.", err, true) - } + return true, wrapCommandError("catalog coverage", catalogCoverage(ctx, artistIds, cfg, jsonLevel, mediaFilter)) case "config": if len(cfg.Urls) < 3 { printInfo("Usage: nugs catalog config enable|disable|set") - return true + return true, nil } action := cfg.Urls[2] switch action { case "enable": - err := enableAutoRefresh(cfg) - if err != nil { - handleErr("Enable auto-refresh failed.", err, true) - } + return true, wrapCommandError("enable auto-refresh", enableAutoRefresh(cfg)) case "disable": - err := disableAutoRefresh(cfg) - if err != nil { - handleErr("Disable auto-refresh failed.", err, true) - } + return true, wrapCommandError("disable auto-refresh", disableAutoRefresh(cfg)) case "set": - err := configureAutoRefresh(cfg) - if err != nil { - handleErr("Configure auto-refresh failed.", err, true) - } + return true, wrapCommandError("configure auto-refresh", configureAutoRefresh(cfg)) default: fmt.Printf("Unknown config action: %s\n", action) } default: fmt.Printf("Unknown catalog command: %s\n", subCmd) } - return true + return true, nil } // handleArtistShorthand handles " latest/full [media]" shortcuts. // Returns true if the input was handled (including error cases). // Accepts optional media type modifier: "audio", "video", or "both" -func handleArtistShorthand(cfg *Config) bool { +func handleArtistShorthand(cfg *Config) (bool, error) { artistID, err := strconv.Atoi(cfg.Urls[0]) if err != nil { - return false + return false, nil } // Check for media type modifier (3rd arg: "audio", "video", "both") @@ -570,20 +557,20 @@ func handleArtistShorthand(cfg *Config) bool { fmt.Printf(" • %snugs %d full%s [audio|video|both] - Download entire catalog\n\n", colorBold, artistID, colorReset) fmt.Printf("For catalog commands, use:\n") fmt.Printf(" • %snugs catalog %s %d%s\n", colorBold, cfg.Urls[1], artistID, colorReset) - os.Exit(1) + return true, fmt.Errorf("invalid artist shortcut %q", cfg.Urls[1]) default: fmt.Printf("%s✗ Unknown command: %s%s\n\n", colorRed, cfg.Urls[1], colorReset) fmt.Printf("Valid artist shortcuts:\n") fmt.Printf(" • %snugs %d latest%s [audio|video|both] - Download latest shows\n", colorBold, artistID, colorReset) fmt.Printf(" • %snugs %d full%s [audio|video|both] - Download entire catalog\n\n", colorBold, artistID, colorReset) - os.Exit(1) + return true, fmt.Errorf("unknown artist shortcut %q", cfg.Urls[1]) } - return false // continue to auth+dispatch with rewritten URL + return false, nil // continue to auth+dispatch with rewritten URL } // handleCatalogGapsFill handles the "catalog gaps [...] fill" command // which requires authentication. -func handleCatalogGapsFill(ctx context.Context, cfg *Config, streamParams *StreamParams, jsonLevel string) { +func handleCatalogGapsFill(ctx context.Context, cfg *Config, streamParams *StreamParams, jsonLevel string) error { // Extract media modifier from args (between "gaps" and "fill") argsAfterGaps := cfg.Urls[2 : len(cfg.Urls)-1] // Everything between "gaps" and "fill" mediaFilter, artistIds := parseMediaModifier(argsAfterGaps) @@ -591,8 +578,9 @@ func handleCatalogGapsFill(ctx context.Context, cfg *Config, streamParams *Strea if len(artistIds) == 0 { fmt.Println("Error: No artist IDs provided") fmt.Println("Usage: catalog gaps [...] [audio|video|both] fill") - return + return errors.New("catalog gaps fill requires at least one artist ID") } + var failures []error for idx, artistId := range artistIds { if idx > 0 && jsonLevel == "" { fmt.Println() @@ -601,27 +589,26 @@ func handleCatalogGapsFill(ctx context.Context, cfg *Config, streamParams *Strea } err := catalogGapsFill(ctx, artistId, cfg, streamParams, jsonLevel, mediaFilter) if err != nil { - if len(artistIds) > 1 { - printWarning(fmt.Sprintf("Failed to fill gaps for artist %s: %v", artistId, err)) - continue - } - handleErr("Catalog gaps fill failed.", err, true) + printWarning(fmt.Sprintf("Failed to fill gaps for artist %s: %v", artistId, err)) + failures = append(failures, fmt.Errorf("artist %s: %w", artistId, err)) } } + return errors.Join(failures...) } // dispatch iterates over URLs, resolves their type, and routes each to the // appropriate handler (album, playlist, video, artist, etc.). // Returns true if the run was cancelled. -func dispatch(ctx context.Context, cfg *Config, streamParams *StreamParams, legacyToken, uguID string) bool { +func dispatch(ctx context.Context, cfg *Config, streamParams *StreamParams, legacyToken, uguID string) (bool, error) { albumTotal := len(cfg.Urls) var itemErr error completedItems := 0 + var failures []error for albumNum, _url := range cfg.Urls { if err := waitIfPausedOrCancelled(); err != nil { if isCrawlCancelledErr(err) { printWarning("Crawl cancelled") - return true + return true, err } } errorsBefore := ui.RunErrorCount.Load() @@ -630,6 +617,7 @@ func dispatch(ctx context.Context, cfg *Config, streamParams *StreamParams, lega itemId, mediaType := checkURL(_url) if itemId == "" { fmt.Println("Invalid URL:", _url) + failures = append(failures, fmt.Errorf("item %d: invalid URL %q", albumNum+1, _url)) continue } switch mediaType { @@ -653,11 +641,13 @@ func dispatch(ctx context.Context, cfg *Config, streamParams *StreamParams, lega if itemErr != nil { if isCrawlCancelledErr(itemErr) { printWarning("Crawl cancelled") - return true + return true, itemErr } - handleErr("Item failed.", itemErr, false) + fmt.Fprintf(os.Stderr, "item %d failed: %v\n", albumNum+1, itemErr) + failures = append(failures, fmt.Errorf("item %d: %w", albumNum+1, itemErr)) + } else { + completedItems++ } - completedItems++ itemErrors := ui.RunErrorCount.Load() - errorsBefore itemWarnings := ui.RunWarningCount.Load() - warningsBefore itemStatus := fmt.Sprintf("Item %d/%d complete", albumNum+1, albumTotal) @@ -671,5 +661,5 @@ func dispatch(ctx context.Context, cfg *Config, streamParams *StreamParams, lega printSection("Run Summary") printInfo(fmt.Sprintf("Completed %d/%d items", completedItems, albumTotal)) printInfo(fmt.Sprintf("Total health: errors=%d warnings=%d", ui.RunErrorCount.Load(), ui.RunWarningCount.Load())) - return false + return false, errors.Join(failures...) } diff --git a/cmd/nugs/main_flow_test.go b/cmd/nugs/main_flow_test.go new file mode 100644 index 0000000..b0bc2c7 --- /dev/null +++ b/cmd/nugs/main_flow_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "context" + "os" + "strings" + "testing" + + "github.com/jmagar/nugs-cli/internal/testutil" +) + +func TestRealMainVersionDoesNotRequireConfig(t *testing.T) { + testutil.WithTempHome(t) + testutil.ChdirTemp(t) + oldArgs := os.Args + os.Args = []string{"nugs", "version"} + t.Cleanup(func() { os.Args = oldArgs }) + + output := testutil.CaptureStdout(t, func() { + if code := realMain(); code != 0 { + t.Fatalf("realMain() exit code = %d, want 0", code) + } + }) + if !strings.Contains(output, "nugs ") { + t.Fatalf("version output = %q", output) + } +} + +func TestRunFinalizesFailedRuntimeStatus(t *testing.T) { + testutil.WithTempHome(t) + t.Setenv("PATH", "") + t.Setenv("NUGS_DETACHED", "1") + cfg := &Config{ + Urls: []string{"23329"}, + RcloneEnabled: true, + } + + err := run(cfg, "") + if err == nil { + t.Fatal("run() error = nil, want missing rclone error") + } + status, statusErr := readRuntimeStatus() + if statusErr != nil { + t.Fatalf("readRuntimeStatus() error = %v", statusErr) + } + if status.State != "failed" { + t.Fatalf("runtime state = %q, want failed", status.State) + } +} + +func TestDispatchReturnsAggregateErrorForInvalidItem(t *testing.T) { + cfg := &Config{Urls: []string{"not-a-nugs-url"}} + cancelled, err := dispatch(context.Background(), cfg, nil, "", "") + if cancelled { + t.Fatal("dispatch() cancelled = true, want false") + } + if err == nil || !strings.Contains(err.Error(), "invalid URL") { + t.Fatalf("dispatch() error = %v, want invalid URL", err) + } +} + +func TestArtistShorthandReturnsErrorInsteadOfExiting(t *testing.T) { + cfg := &Config{Urls: []string{"1125", "gaps"}} + handled, err := handleArtistShorthand(cfg) + if !handled || err == nil { + t.Fatalf("handleArtistShorthand() = (%v, %v), want handled error", handled, err) + } +} diff --git a/cmd/nugs/model_aliases.go b/cmd/nugs/model_aliases.go index a67730a..1c1620d 100644 --- a/cmd/nugs/model_aliases.go +++ b/cmd/nugs/model_aliases.go @@ -1,19 +1,12 @@ package main -// Type aliases bridging root package to internal/model during migration. -// These will be removed in Phase 12 when all code moves to internal packages. +// Command-layer aliases for the internal model contract. -import ( - "fmt" - "strings" - - "github.com/jmagar/nugs-cli/internal/model" -) +import "github.com/jmagar/nugs-cli/internal/model" // Type aliases for model types. type ( Config = model.Config - Args = model.Args Transport = model.Transport BatchProgressState = model.BatchProgressState RuntimeStatus = model.RuntimeStatus @@ -82,100 +75,3 @@ const ( MessagePriorityWarning = model.MessagePriorityWarning MessagePriorityError = model.MessagePriorityError ) - -func init() { - // Wire the colored help text into model.Args.Description() - model.ArgsDescriptionFunc = argsDescription -} - -func argsDescription() string { - var b strings.Builder - - heading := func(title string) { - fmt.Fprintf(&b, "\n%s◆ %s%s\n", colorBold, title, colorReset) - fmt.Fprintf(&b, "%s─────────────────────────────────────────────────────────────────────────────%s\n", colorCyan, colorReset) - } - headingWithNote := func(title, note string) { - fmt.Fprintf(&b, "\n%s◆ %s%s %s%s%s\n", colorBold, title, colorReset, colorCyan, note, colorReset) - fmt.Fprintf(&b, "%s─────────────────────────────────────────────────────────────────────────────%s\n", colorCyan, colorReset) - } - cmd := func(syntax, description string) { - fmt.Fprintf(&b, " %s•%s %s%s%s %s\n", colorGreen, colorReset, colorCyan, syntax, colorReset, description) - } - example := func(syntax string) { - fmt.Fprintf(&b, " %s▸%s %s%s%s\n", colorYellow, colorReset, colorCyan, syntax, colorReset) - } - exampleWithDesc := func(syntax, desc string) { - fmt.Fprintf(&b, " %s▸%s %s%s%s %s\n", colorYellow, colorReset, colorCyan, syntax, colorReset, desc) - } - - fmt.Fprintf(&b, "%s♪ Download music and videos from Nugs.net%s\n", colorBold, colorReset) - - heading("DOWNLOAD COMMANDS") - cmd("grab ", " Download a show by numeric ID") - cmd("grab ", " Download a show by full Nugs.net URL") - cmd("grab latest", " Download the latest show from an artist") - cmd("grab full", " Download the entire artist catalog") - cmd("grab audio|video|both", " Override media type for this download") - - heading("LIST COMMANDS") - cmd("list", " List all available artists") - cmd("list >100", " Filter artists by show count (>, <, >=, <=, =)") - cmd("list ", " List all shows for a specific artist") - cmd("list audio|video", " Filter shows by media type") - cmd(`list "venue"`, " Filter shows by venue name") - cmd("list latest ", " Show latest N shows for an artist") - - heading("CATALOG COMMANDS") - cmd("catalog update", " Fetch and cache latest catalog from API") - cmd("catalog cache", " Show cache status and metadata") - cmd("catalog stats", " Display catalog statistics") - cmd("catalog latest [limit]", " Show latest additions (default 15)") - cmd("catalog list [...]", " Browse catalog offline (cached, no auth)") - cmd("catalog gaps [...]", " List missing shows for one or more artists") - cmd("catalog gaps video", " Filter gaps by media type (audio/video/both)") - cmd("catalog gaps --ids-only", " Output just IDs for piping") - cmd("catalog gaps fill", " Auto-download all missing shows") - cmd("catalog coverage [ids...]", " Show download coverage statistics") - cmd("catalog config enable|disable|set", "Configure auto-refresh schedule") - - heading("WATCH COMMANDS") - cmd("watch add ", " Add an artist to the watch list") - cmd("watch remove ", " Remove an artist from the watch list") - cmd("watch list", " Show all watched artists") - cmd("watch check [audio|video]", " Check for new shows and download them") - cmd("watch enable", " Install and enable systemd auto-check timer") - cmd("watch disable", " Remove systemd timer") - - heading("SESSION COMMANDS") - cmd("status", " Show active download session status") - cmd("cancel", " Cancel the running download session") - - headingWithNote("JSON OUTPUT LEVELS", "(--json )") - cmd("minimal", " Essential fields only") - cmd("standard", " Adds location details (for shows)") - cmd("extended", " All available metadata") - cmd("raw", " Unmodified API response") - - heading("EXAMPLES") - exampleWithDesc("nugs grab 12345", " Download show by ID") - exampleWithDesc("nugs grab https://play.nugs.net/release/12345", " Download by URL") - example("nugs grab 1125 latest") - example("nugs grab 1125 full video") - example("nugs list 461") - example(`nugs list 461 "Red Rocks"`) - example("nugs list 1125 latest 5") - example(`nugs list ">100"`) - example("nugs catalog update") - example("nugs catalog gaps 1125") - example("nugs catalog gaps 1125 fill") - example("nugs catalog coverage 1125 461") - example("nugs watch add 1125") - example("nugs watch check") - example("nugs status") - - fmt.Fprintf(&b, "\n %s→%s Full URLs also work: %snugs https://play.nugs.net/release/12345%s\n", - colorCyan, colorReset, colorYellow, colorReset) - - return b.String() -} diff --git a/cmd/nugs/output.go b/cmd/nugs/output.go index 130ec29..0cb03fb 100644 --- a/cmd/nugs/output.go +++ b/cmd/nugs/output.go @@ -1,7 +1,6 @@ package main -// Output wrappers delegating to internal/ui during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for terminal output. import ( "fmt" diff --git a/cmd/nugs/rclone.go b/cmd/nugs/rclone.go index 6d5722a..f8caa62 100644 --- a/cmd/nugs/rclone.go +++ b/cmd/nugs/rclone.go @@ -1,7 +1,6 @@ package main -// Rclone wrappers delegating to internal/rclone during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for remote storage operations. import ( "context" diff --git a/cmd/nugs/runtime_status.go b/cmd/nugs/runtime_status.go index 8fd6031..3b9681e 100644 --- a/cmd/nugs/runtime_status.go +++ b/cmd/nugs/runtime_status.go @@ -1,19 +1,18 @@ package main -// Runtime status wrappers delegating to internal/runtime during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for runtime status publication. import ( "github.com/jmagar/nugs-cli/internal/runtime" "github.com/jmagar/nugs-cli/internal/ui" ) -func initRuntimeStatus() { runtime.InitRuntimeStatus() } -func printRuntimeStatus() { runtime.PrintRuntimeStatus() } -func readRuntimeStatus() (RuntimeStatus, error) { return runtime.ReadRuntimeStatus() } +func initRuntimeStatus() { runtime.InitRuntimeStatus() } +func printRuntimeStatus() { runtime.PrintRuntimeStatus() } +func readRuntimeStatus() (RuntimeStatus, error) { return runtime.ReadRuntimeStatus() } func readRuntimeControl() (RuntimeControl, error) { return runtime.ReadRuntimeControl() } -func requestRuntimeCancel() error { return runtime.RequestRuntimeCancel() } -func requestRuntimePause(paused bool) error { return runtime.RequestRuntimePause(paused) } +func requestRuntimeCancel() error { return runtime.RequestRuntimeCancel() } +func requestRuntimePause(paused bool) error { return runtime.RequestRuntimePause(paused) } func printActiveRuntimeHint(currentPID int, currentCommand []string) { runtime.PrintActiveRuntimeHint(currentPID, currentCommand) } diff --git a/cmd/nugs/signal_persistence.go b/cmd/nugs/signal_persistence.go index 2dcc019..7d1a241 100644 --- a/cmd/nugs/signal_persistence.go +++ b/cmd/nugs/signal_persistence.go @@ -1,7 +1,6 @@ package main -// Signal persistence wrapper delegating to internal/runtime during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapter for signal persistence. import "github.com/jmagar/nugs-cli/internal/runtime" diff --git a/cmd/nugs/structs.go b/cmd/nugs/structs.go index 4f4383b..2fed562 100644 --- a/cmd/nugs/structs.go +++ b/cmd/nugs/structs.go @@ -1,15 +1,10 @@ package main -// Theme wrappers delegating to internal/ui during migration. -// These will be removed in Phase 12 when all callers move to internal packages. -// The ui package init() runs first (Go import ordering), setting colors before -// these vars are initialized. +// Command rendering aliases are synchronized from the selected UI theme. import "github.com/jmagar/nugs-cli/internal/ui" -// Color variable wrappers - package-level copies from ui, snapshotted in init(). -// This avoids qualifying every reference as ui.ColorRed etc. in the root package -// and will be removed in Phase 12 when rendering moves fully into internal/ui. +// Color values are snapshotted from ui after theme selection. var ( colorReset string colorRed string @@ -19,25 +14,16 @@ var ( colorPurple string colorCyan string colorBold string - activeTheme string ) // Symbol variable wrappers var ( symbolCheck string symbolCross string - symbolArrow string - symbolMusic string - symbolUpload string symbolDownload string symbolInfo string symbolWarning string - symbolGear string symbolPackage string - symbolRocket string - symbolAudio string - symbolVideo string - symbolBoth string ) func init() { @@ -53,20 +39,11 @@ func syncFromUI() { colorPurple = ui.ColorPurple colorCyan = ui.ColorCyan colorBold = ui.ColorBold - activeTheme = ui.ActiveTheme symbolCheck = ui.SymbolCheck symbolCross = ui.SymbolCross - symbolArrow = ui.SymbolArrow - symbolMusic = ui.SymbolMusic - symbolUpload = ui.SymbolUpload symbolDownload = ui.SymbolDownload symbolInfo = ui.SymbolInfo symbolWarning = ui.SymbolWarning - symbolGear = ui.SymbolGear symbolPackage = ui.SymbolPackage - symbolRocket = ui.SymbolRocket - symbolAudio = ui.SymbolAudio - symbolVideo = ui.SymbolVideo - symbolBoth = ui.SymbolBoth } diff --git a/cmd/nugs/url_parser.go b/cmd/nugs/url_parser.go index c787434..55ff30f 100644 --- a/cmd/nugs/url_parser.go +++ b/cmd/nugs/url_parser.go @@ -1,7 +1,6 @@ package main -// URL parser wrappers delegating to internal/api during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for URL parsing. import "github.com/jmagar/nugs-cli/internal/api" diff --git a/cmd/nugs/version.go b/cmd/nugs/version.go new file mode 100644 index 0000000..f24018a --- /dev/null +++ b/cmd/nugs/version.go @@ -0,0 +1,55 @@ +package main + +import ( + "fmt" + "io" + "runtime" + "runtime/debug" + "strings" +) + +// These values are set by release builds with -ldflags. Development builds +// fall back to the module build information embedded by the Go toolchain. +var ( + version = "dev" + commit = "unknown" + buildDate = "unknown" +) + +func isVersionRequest(args []string) bool { + return len(args) == 1 && (args[0] == "version" || args[0] == "--version" || args[0] == "-v") +} + +func resolvedBuildIdentity() (resolvedVersion, resolvedCommit, resolvedDate string, dirty bool) { + resolvedVersion, resolvedCommit, resolvedDate = version, commit, buildDate + if info, ok := debug.ReadBuildInfo(); ok { + if resolvedVersion == "dev" && info.Main.Version != "" && info.Main.Version != "(devel)" { + resolvedVersion = info.Main.Version + } + for _, setting := range info.Settings { + switch setting.Key { + case "vcs.revision": + if resolvedCommit == "unknown" && setting.Value != "" { + resolvedCommit = setting.Value + } + case "vcs.time": + if resolvedDate == "unknown" && setting.Value != "" { + resolvedDate = setting.Value + } + case "vcs.modified": + dirty = strings.EqualFold(setting.Value, "true") + } + } + } + return resolvedVersion, resolvedCommit, resolvedDate, dirty +} + +func printVersion(w io.Writer) { + resolvedVersion, resolvedCommit, resolvedDate, dirty := resolvedBuildIdentity() + dirtySuffix := "" + if dirty { + dirtySuffix = "+dirty" + } + fmt.Fprintf(w, "nugs %s\ncommit: %s%s\nbuilt: %s\ngo: %s\nplatform: %s/%s\n", + resolvedVersion, resolvedCommit, dirtySuffix, resolvedDate, runtime.Version(), runtime.GOOS, runtime.GOARCH) +} diff --git a/cmd/nugs/version_test.go b/cmd/nugs/version_test.go new file mode 100644 index 0000000..61dc8bb --- /dev/null +++ b/cmd/nugs/version_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "bytes" + "strings" + "testing" +) + +func TestIsVersionRequest(t *testing.T) { + t.Parallel() + for _, arg := range []string{"version", "--version", "-v"} { + if !isVersionRequest([]string{arg}) { + t.Fatalf("expected %q to be a version request", arg) + } + } + for _, args := range [][]string{nil, {"version", "extra"}, {"--help"}} { + if isVersionRequest(args) { + t.Fatalf("did not expect %q to be a version request", args) + } + } +} + +func TestPrintVersion(t *testing.T) { + oldVersion, oldCommit, oldBuildDate := version, commit, buildDate + version, commit, buildDate = "v1.2.3", "abc123", "2026-07-18T00:00:00Z" + t.Cleanup(func() { version, commit, buildDate = oldVersion, oldCommit, oldBuildDate }) + + var output bytes.Buffer + printVersion(&output) + for _, expected := range []string{"nugs v1.2.3", "commit: abc123", "built: 2026-07-18T00:00:00Z", "go:", "platform:"} { + if !strings.Contains(output.String(), expected) { + t.Fatalf("version output %q missing %q", output.String(), expected) + } + } +} diff --git a/cmd/nugs/video.go b/cmd/nugs/video.go index 9172034..971b806 100644 --- a/cmd/nugs/video.go +++ b/cmd/nugs/video.go @@ -1,7 +1,6 @@ package main -// Video wrappers delegating to internal/download during migration. -// These will be removed in Phase 12 when all callers move to internal packages. +// Command adapters for internal video download operations. import ( "context" diff --git a/cmd/nugs/watch.go b/cmd/nugs/watch.go index a3b1aab..9d744e2 100644 --- a/cmd/nugs/watch.go +++ b/cmd/nugs/watch.go @@ -42,9 +42,9 @@ func watchDisable() error { // handleWatchCommand routes pre-auth "watch" subcommands (add/remove/list/enable/disable). // Returns true if the command was handled. Returns false for "watch check" (post-auth). -func handleWatchCommand(ctx context.Context, cfg *Config, jsonLevel string) bool { +func handleWatchCommand(ctx context.Context, cfg *Config, jsonLevel string) (bool, error) { if len(cfg.Urls) == 0 || cfg.Urls[0] != "watch" { - return false + return false, nil } if len(cfg.Urls) < 2 { @@ -54,60 +54,60 @@ func handleWatchCommand(ctx context.Context, cfg *Config, jsonLevel string) bool fmt.Println(" nugs watch check") fmt.Println(" nugs watch enable") fmt.Println(" nugs watch disable") - return true + return true, nil } subCmd := cfg.Urls[1] // "watch check" requires auth — defer to handleWatchCheckCommand. if subCmd == "check" { - return false + return false, nil } switch subCmd { case "add": if len(cfg.Urls) < 3 { printInfo("Usage: nugs watch add ") - return true + return true, nil } if err := watchAdd(cfg, cfg.Urls[2]); err != nil { - handleErr("Watch add failed.", err, true) + return true, fmt.Errorf("watch add failed: %w", err) } case "remove": if len(cfg.Urls) < 3 { printInfo("Usage: nugs watch remove ") - return true + return true, nil } if err := watchRemove(cfg, cfg.Urls[2]); err != nil { - handleErr("Watch remove failed.", err, true) + return true, fmt.Errorf("watch remove failed: %w", err) } case "list": if err := watchList(cfg, jsonLevel); err != nil { - handleErr("Watch list failed.", err, true) + return true, fmt.Errorf("watch list failed: %w", err) } case "enable": if err := watchEnable(cfg); err != nil { - handleErr("Watch enable failed.", err, true) + return true, fmt.Errorf("watch enable failed: %w", err) } case "disable": if err := watchDisable(); err != nil { - handleErr("Watch disable failed.", err, true) + return true, fmt.Errorf("watch disable failed: %w", err) } default: // Bare numeric ID shorthand: "nugs watch 1125" → "nugs watch add 1125" if err := watchAdd(cfg, subCmd); err != nil { - handleErr("Watch add failed.", err, true) + return true, fmt.Errorf("watch add failed: %w", err) } } _ = ctx // ctx unused for pre-auth commands but kept for signature consistency - return true + return true, nil } // handleWatchCheckCommand routes post-auth "watch check". Returns true if handled. -func handleWatchCheckCommand(ctx context.Context, cfg *Config, streamParams *StreamParams, jsonLevel string) bool { +func handleWatchCheckCommand(ctx context.Context, cfg *Config, streamParams *StreamParams, jsonLevel string) (bool, error) { if len(cfg.Urls) < 2 || cfg.Urls[0] != "watch" || cfg.Urls[1] != "check" { - return false + return false, nil } // Extract optional media modifier from remaining args. @@ -117,7 +117,7 @@ func handleWatchCheckCommand(ctx context.Context, cfg *Config, streamParams *Str } if err := watchCheck(ctx, cfg, streamParams, jsonLevel, mediaFilter); err != nil { - handleErr("Watch check failed.", err, true) + return true, fmt.Errorf("watch check failed: %w", err) } - return true + return true, nil } diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 909c98c..224c203 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -48,9 +48,10 @@ Root: Command Orchestration (cmd/nugs/main.go) nugs/ ├── cmd/ │ └── nugs/ -│ └── main.go # Entry point (648 lines) -├── internal/ # Private packages (13 total) +│ └── main.go # Entry point and command orchestration +├── internal/ # Private packages (14 total) │ ├── model/ # Core data types (no dependencies) +│ ├── notify/ # Gotify notification adapter │ ├── testutil/ # Test utilities (no dependencies) │ ├── helpers/ # Path manipulation utilities │ ├── ui/ # Display and formatting @@ -87,7 +88,10 @@ nugs/ - `ChdirTemp(t)` - Change to temp directory - `WriteExecutable(t, path)` - Create executable script -**Key Pattern:** Dependency inversion - defines types used by higher layers without importing them. +**Key Pattern:** Dependency inversion - defines types used by higher layers +without importing them. CLI argument parsing and help are owned by +`internal/config`, so the foundation layer has no package-main initializer or +root-owned behavior. --- @@ -815,17 +819,18 @@ func AcquireLock(path string, retries int) (*FileLock, error) { ### Runtime Package Platform Variants -**9 platform-specific files:** -- `detach_unix.go`, `detach_windows.go` - Background process creation +**Platform-specific files include:** +- detach/process helpers where operating systems require different behavior - `cancel_unix.go`, `cancel_windows.go` - Crawl cancellation - `hotkey_input_unix.go`, `hotkey_input_windows.go`, `hotkey_input_other.go` - Interactive pause/cancel - `process_alive_unix.go`, `process_alive_windows.go` - PID checking - `signal_persistence_unix.go`, `signal_persistence_windows.go` - Signal handling -**Why so many variants?** -- Unix uses fork/exec, Windows uses CreateProcess -- Signal handling differs (SIGTERM vs WM_CLOSE) -- Terminal control differs (termios vs Windows Console API) +**Why variants?** +- Cancellation uses SIGTERM on Unix and `os.Process.Kill` on Windows. +- Terminal control differs between Unix termios and Windows console handling. +- Auto-detach is selected by shared runtime policy: interactive terminals remain + attached; non-interactive mutating commands may detach. --- diff --git a/docs/COMMANDS.md b/docs/COMMANDS.md index 02ed442..84fada3 100644 --- a/docs/COMMANDS.md +++ b/docs/COMMANDS.md @@ -29,7 +29,8 @@ Downloads require authentication (email/password or token in config). nugs [url2 url3 ...] ``` -Accepts any nugs.net URL: +Accepts the following URL forms (other Nugs.net pages are not guaranteed to +parse): | URL Pattern | Type | |-------------|------| @@ -43,7 +44,7 @@ Accepts any nugs.net URL: | `https://play.nugs.net/artist//albums` | Artist albums | | `https://play.nugs.net/livestream//exclusive` | Livestream | | `https://play.nugs.net/watch/livestreams/exclusive/` | Livestream | -| `https://play.nugs.net/#/my-webcasts/` | Webcast | +| `https://play.nugs.net/#/my-webcasts/--` | Webcast | | `https://play.nugs.net/library/webcast/` | Webcast | | `https://www.nugs.net/.../Stash-QueueVideo?...` | Paid livestream | | `` | Album by ID | @@ -190,7 +191,8 @@ nugs catalog latest [limit] nugs latest [limit] ``` -Shows the most recently added shows (default: 15). +Shows the most recently added shows (default: 15). Media modifiers are not +supported because this catalog response does not contain product details. ```bash nugs latest # Latest 15 additions @@ -256,12 +258,14 @@ nugs catalog coverage [artist_ids...] [audio|video|both] nugs coverage [artist_ids...] [audio|video|both] ``` -Shows download coverage statistics (total vs downloaded vs missing). +Shows download coverage statistics (total vs downloaded vs missing). With no +IDs, it discovers artists from configured local and remote download folders; it +does not query account subscriptions. ```bash nugs coverage 1125 # Coverage for Billy Strings nugs coverage 1125 video # Video coverage -nugs coverage # Coverage for all subscribed artists +nugs coverage # Coverage for discovered downloaded artists ``` --- @@ -367,8 +371,8 @@ These top-level shortcuts expand to their full catalog equivalents: | Key | Action | |-----|--------| -| `p` | Pause/resume download | -| `c` | Cancel download | +| `Shift+P` | Pause/resume download | +| `Shift+C` | Cancel download | --- diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 0195f7f..2f8f5c4 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -1,1007 +1,192 @@ -# Nugs CLI Configuration Reference - -Complete reference for all configuration options in Nugs CLI. - -## Table of Contents - -- [Config Structure](#config-structure) -- [File Locations](#file-locations) -- [Field Reference](#field-reference) - - [Authentication](#authentication) - - [Download Quality](#download-quality) - - [Output Paths](#output-paths) - - [FFmpeg Integration](#ffmpeg-integration) - - [Rclone Cloud Uploads](#rclone-cloud-uploads) - - [Catalog Auto-Refresh](#catalog-auto-refresh) - - [Performance](#performance) -- [Validation Rules](#validation-rules) -- [Security](#security) -- [Examples](#examples) -- [Migrations](#migrations) -- [Environment Variables](#environment-variables) - ---- - -## Config Structure - -The `Config` struct in `internal/model/types.go` contains **25 fields** organized by category: - -| Category | Fields | Purpose | -|----------|--------|---------| -| Authentication | 3 | Nugs.net account credentials | -| Download Quality | 4 | Audio/video format selection (includes computed `wantRes`) | -| Output Paths | 3 | Local download directories (includes internal `urls`) | -| FFmpeg Integration | 4 | Video processing (includes deprecated `forceVideo`, `skipVideos`) | -| Rclone Cloud Uploads | 6 | Cloud storage automation | -| Catalog Auto-Refresh | 4 | Automatic catalog updates | -| Performance | 1 | Optimization flags | - -**Total:** 25 fields (23 user-configurable + 2 deprecated/internal) - ---- - -## File Locations - -### Search Order - -Config files are searched in this order (first found wins): - -1. `./config.json` (current directory) -2. `~/.nugs/config.json` (recommended) -3. `~/.config/nugs/config.json` (XDG standard) - -**No merging:** Only the first found config is used. - -### File Creation - -**First-time setup writes to:** `~/.nugs/config.json` - -**Subsequent writes use:** Same file that was read (`config.LoadedConfigPath`) - -**Permissions:** -- Directory: `0700` (owner-only access) -- File: `0600` (owner read/write only) - -### Security Auto-Fix - -On every config read: -- **Unix:** Auto-fixes insecure permissions (warns and sets to `0600`) -- **Windows:** Permission check skipped (relies on NTFS ACLs) - ---- - -## Field Reference - -### Authentication - -#### `email` (string) - -**Purpose:** Nugs.net account email for OAuth authentication - -**Required:** Yes (unless using `token`) - -**Example:** `"user@example.com"` - -**Validation:** None - -**Security:** ⚠️ Stored in plain text - -**Interactive prompt:** "Enter your Nugs email:" - ---- - -#### `password` (string) - -**Purpose:** Nugs.net account password for OAuth authentication - -**Required:** Yes (unless using `token`) - -**Example:** `"your_password_here"` - -**Validation:** None - -**Security:** ⚠️ Stored in plain text, **visible during interactive setup** - -**Interactive prompt:** "Enter your Nugs password:" - -**Workaround:** Use `token` authentication instead (see `docs/token.md`) - ---- - -#### `token` (string) - -**Purpose:** Pre-existing auth token for Apple/Google accounts or to avoid password storage - -**Required:** Alternative to `email`/`password` - -**Example:** `"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9..."` - -**Validation:** "Bearer " prefix automatically stripped - -**Security:** ⚠️ Stored in plain text (but not visible during input) - -**How to obtain:** See `docs/token.md` for extraction from Nugs.net web client - -**Note:** If both `token` and `email`/`password` are set, token takes precedence - ---- - -### Download Quality - -#### `format` (int 1-5) - -**Purpose:** Audio quality selection - -**Default:** `4` (360 Reality Audio / best available) - -**Options:** -- `1` - 16-bit 44.1kHz ALAC (Apple Lossless) -- `2` - 16-bit 44.1kHz FLAC (most common lossless) -- `3` - 24-bit 48kHz MQA (Master Quality Authenticated) -- `4` - 360 Reality Audio / best available (default) -- `5` - 150 Kbps AAC (HLS-only fallback) - -**Validation:** Must be between 1 and 5 - -**Error message:** `"track Format must be between 1 and 5"` - -**Fallback chain:** -- ALAC (1) → FLAC (2) → AAC (5) -- MQA (3) → FLAC (2) -- 360RA (4) → MQA (3) - -**Interactive prompt:** Dropdown with format descriptions - ---- - -#### `videoFormat` (int 1-5) - -**Purpose:** Video resolution selection - -**Default:** `5` (4K / best available) - -**Options:** -- `1` - 480p -- `2` - 720p -- `3` - 1080p -- `4` - 1440p -- `5` - 4K / best available (default) - -**Validation:** Must be between 1 and 5 - -**Error message:** `"video format must be between 1 and 5"` - -**Fallback chain:** -- 1440p (4) → 1080p (3) -- 1080p (3) → 720p (2) -- 720p (2) → 480p (1) - -**Interactive prompt:** Dropdown with resolution options - ---- - -#### `defaultOutputs` (string) - -**Purpose:** Media type preference for downloads - -**Default:** `"audio"` (download audio only) - -**Options:** -- `"audio"` - Download audio only -- `"video"` - Download video only -- `"both"` - Download both audio and video - -**Validation:** Must be "audio", "video", or "both" - -**Error message:** `"invalid defaultOutputs: "" (must be audio, video, or both)"` - -**Overrides:** Can be overridden per-command with `audio`, `video`, or `both` modifiers - -**Examples:** - -```bash -# Use config default -nugs 23329 - -# Override to video -nugs 23329 video - -# Override to both -nugs 23329 both -``` - ---- - -#### `wantRes` (string) - -**Purpose:** Computed resolution string from `videoFormat` - -**Default:** Computed automatically - -**Values:** -- `videoFormat=1` → `wantRes="854x480"` -- `videoFormat=2` → `wantRes="1280x720"` -- `videoFormat=3` → `wantRes="1920x1080"` -- `videoFormat=4` → `wantRes="2560x1440"` -- `videoFormat=5` → `wantRes="2160"` (4K, matches any 4K variant) - -**Note:** Read-only, do not set manually - ---- - -### Output Paths - -#### `outPath` (string) - -**Purpose:** Local download directory for audio files - -**Default:** `"Nugs downloads"` (relative to current directory) - -**Example:** `"/home/user/Music/Nugs"` - -**Validation:** None (directory created if doesn't exist) - -**Interactive prompt:** "Enter output path for audio downloads:" - -**Note:** This is the **local** download path, NOT affected by `rclonePath` - ---- - -#### `videoOutPath` (string) - -**Purpose:** Local download directory for video files - -**Default:** Same as `outPath` (audio and video in same location) - -**Example:** `"/home/user/Videos/Nugs"` - -**Validation:** None (directory created if doesn't exist) - -**Interactive prompt:** Not prompted (defaults to `outPath`) - -**Note:** If empty, automatically set to `outPath` value - ---- - -#### `urls` ([]string) - -**Purpose:** Parsed URLs from CLI arguments - -**Default:** `nil` (empty array) - -**Example:** `["https://play.nugs.net/#/catalog/recording/23329"]` - -**Note:** Internal field, populated by CLI parser, not user-configurable - ---- - -### FFmpeg Integration - -#### `useFfmpegEnvVar` (bool) - -**Purpose:** Whether to use FFmpeg from system PATH vs local binary - -**Default:** `false` (interactive setup), varies at runtime - -**Options:** -- `true` - Use FFmpeg from system PATH -- `false` - Search for local FFmpeg binary (`.ffmpeg`, `./ffmpeg`, binary directory) - -**Interactive prompt:** "Use FFmpeg from PATH? (y/n)" - -**Resolution order when `false`:** -1. Custom binary from `ffmpegNameStr` (if not default) -2. `./ffmpeg` in current directory -3. FFmpeg next to executable -4. Fallback to system PATH - ---- - -#### `ffmpegNameStr` (string) - -**Purpose:** Custom FFmpeg binary name or absolute path - -**Default:** `"ffmpeg"` (use default binary) - -**Example:** `"/usr/local/bin/ffmpeg"` or `"ffmpeg-custom"` - -**Validation:** Checked for existence at runtime - -**Error message:** `"ffmpeg not found in PATH (install ffmpeg or set ffmpegNameStr to an absolute/local binary path)"` - -**Interactive prompt:** Not prompted (advanced users only) - ---- - -#### `skipChapters` (bool) - -**Purpose:** Skip embedding chapter markers in video files - -**Default:** `false` (embed chapters if available) - -**Options:** -- `true` - Skip chapter embedding (faster conversion) -- `false` - Embed chapters from track metadata (default) - -**Note:** Chapters are embedded during TS→MP4 conversion via FFmpeg - ---- - -#### `forceVideo` (bool) **[DEPRECATED]** - -**Status:** ⚠️ **Deprecated** - Use `defaultOutputs="video"` instead - -**Purpose:** Force video downloads (legacy) - -**Migration:** Set `defaultOutputs: "video"` - ---- - -#### `skipVideos` (bool) **[DEPRECATED]** - -**Status:** ⚠️ **Deprecated** - Use `defaultOutputs="audio"` instead - -**Purpose:** Skip video downloads (legacy) - -**Migration:** Set `defaultOutputs: "audio"` (this is the default) - ---- - -### Rclone Cloud Uploads - -#### `rcloneEnabled` (bool) - -**Purpose:** Enable automatic cloud uploads via rclone after downloads - -**Default:** `false` (no cloud uploads) - -**Options:** -- `true` - Upload to rclone remote after download -- `false` - Local downloads only - -**Requires:** `rclone` installed and `rcloneRemote`/`rclonePath` configured - -**Interactive prompt:** "Enable rclone uploads? (y/n)" - -**Note:** Uploads happen automatically after each album/video download completes - ---- - -#### `rcloneRemote` (string) - -**Purpose:** Rclone remote name for audio uploads - -**Default:** `""` (not configured) - -**Example:** `"gdrive"`, `"dropbox"`, `"onedrive"` - -**Validation:** Remote must exist in rclone config (`rclone listremotes`) - -**Error message:** `"rclone remote not found: "` - -**Interactive prompt:** "Enter rclone remote name:" - -**Configure rclone:** Run `rclone config` to create remotes - ---- - -#### `rclonePath` (string) - -**Purpose:** Remote base path for audio uploads (on the rclone remote) - -**Default:** `""` (not configured) - -**Example:** `"/Music"`, `"/Music/Nugs"`, `"/"` - -**Validation:** None (path created if doesn't exist on remote) - -**Interactive prompt:** "Enter remote path for audio uploads:" - -**⚠️ CRITICAL:** This is the **remote** path, NOT local. Local downloads use `outPath`. - -**Breaking Change:** Before 2026-02-05, `rclonePath` incorrectly affected local downloads. See [Migrations](#migrations). - ---- - -#### `rcloneVideoPath` (string) - -**Purpose:** Remote base path for video uploads (on the rclone remote) - -**Default:** Same as `rclonePath` (audio and video in same remote location) - -**Example:** `"/Videos"`, `"/Videos/Nugs"` - -**Validation:** None (path created if doesn't exist on remote) - -**Interactive prompt:** Not prompted (defaults to `rclonePath`) - -**Note:** If empty, automatically set to `rclonePath` value - ---- - -#### `deleteAfterUpload` (bool) - -**Purpose:** Delete local files after successful upload to rclone - -**Default:** `true` (if rclone enabled during interactive setup) - -**Options:** -- `true` - Delete local files after verified upload -- `false` - Keep local files after upload - -**Safety:** Upload is verified with `rclone check --one-way` before deletion - -**Error handling:** If verification fails, local files are **NOT** deleted - -**Interactive prompt:** "Delete local files after upload? (y/n)" - ---- - -#### `rcloneTransfers` (int) - -**Purpose:** Number of parallel rclone transfer threads - -**Default:** `4` - -**Range:** 1-128 (practical limit) - -**Validation:** Must be >= 1 - -**Error message:** `"transfers must be a positive integer"` - -**Interactive prompt:** "Number of rclone transfers:" (default: 4) - -**Performance:** Higher values = faster uploads but more bandwidth/memory - -**Recommended:** -- 4-8 for home internet -- 8-16 for fast connections -- 16-32 for server environments - ---- - -### Catalog Auto-Refresh - -#### `catalogAutoRefresh` (bool) - -**Purpose:** Enable automatic catalog updates on schedule - -**Default:** `true` (enabled) - -**Options:** -- `true` - Auto-update catalog at scheduled time -- `false` - Manual updates only (`nugs catalog update`) - -**Trigger:** Checked at startup, updates if schedule elapsed - -**Configure:** Use `nugs catalog config` to change schedule - ---- - -#### `catalogRefreshTime` (string) - -**Purpose:** Time of day to refresh catalog (HH:MM format) - -**Default:** `"05:00"` (5:00 AM) - -**Format:** 24-hour time, HH:MM - -**Example:** `"03:00"`, `"14:30"`, `"23:00"` - -**Validation:** Must match regex `^\d{2}:\d{2}$` - -**Error message:** `"invalid refresh time format: (expected HH:MM)"` - -**Interactive prompt:** "Catalog refresh time (HH:MM):" (default: 05:00) - ---- - -#### `catalogRefreshTimezone` (string) - -**Purpose:** IANA timezone for refresh time interpretation - -**Default:** `"America/New_York"` (EST/EDT) - -**Format:** IANA timezone database name - -**Examples:** -- `"America/Los_Angeles"` (PST/PDT) -- `"America/Chicago"` (CST/CDT) -- `"Europe/London"` (GMT/BST) -- `"Asia/Tokyo"` (JST) - -**Validation:** Must be valid IANA timezone - -**Error message:** `"invalid timezone : unknown time zone "` - -**Common mistake:** ❌ `"EST"` → ✅ `"America/New_York"` - -**Find your timezone:** https://en.wikipedia.org/wiki/List_of_tz_database_time_zones - -**Interactive prompt:** "Timezone for refresh:" (default: America/New_York) - ---- - -#### `catalogRefreshInterval` (string) - -**Purpose:** How often to refresh catalog - -**Default:** `"daily"` - -**Options:** -- `"daily"` - Refresh every day at `catalogRefreshTime` -- `"weekly"` - Refresh once per week at `catalogRefreshTime` - -**Validation:** Must be "daily" or "weekly" - -**Error message:** `"invalid interval: (must be 'daily' or 'weekly')"` - -**Interactive prompt:** "Refresh interval (daily/weekly):" (default: daily) - -**Note:** Weekly refresh happens on the same day of week as first refresh - ---- - -### Performance - -#### `skipSizePreCalculation` (bool) - -**Purpose:** Skip pre-calculation of album size before download - -**Default:** `false` (calculate size) - -**Options:** -- `true` - Skip size calculation (faster startup) -- `false` - Calculate size (accurate progress/ETA) - -**Trade-off:** -- ✅ **Skip:** Faster start, but no total size or accurate ETA -- ✅ **Calculate:** Slower start (5-10s), but accurate progress bar - -**Method:** Uses 8 concurrent HEAD requests with 60s timeout - -**Interactive prompt:** Not prompted (advanced users only) - ---- - -## Validation Rules - -### Format Validation - -**At config parse time:** - -| Field | Rule | Error Message | -|-------|------|---------------| -| `format` | 1-5 | `"track Format must be between 1 and 5"` | -| `videoFormat` | 1-5 | `"video format must be between 1 and 5"` | -| `defaultOutputs` | "audio", "video", or "both" | `"invalid defaultOutputs: "" (must be audio, video, or both)"` | -| `catalogRefreshTimezone` | Valid IANA timezone | `"invalid timezone : unknown time zone "` | -| `catalogRefreshTime` | HH:MM format | `"invalid refresh time format: (expected HH:MM)"` | -| `catalogRefreshInterval` | "daily" or "weekly" | `"invalid interval: (must be 'daily' or 'weekly')"` | -| `rcloneTransfers` | >= 1 | `"transfers must be a positive integer"` | - -### Runtime Validation - -**At execution time:** - -| Field | Rule | Error Message | -|-------|------|---------------| -| `ffmpegNameStr` | Binary exists | `"ffmpeg not found in PATH (install ffmpeg or set ffmpegNameStr to an absolute/local binary path)"` | -| `rcloneRemote` | Remote exists | `"rclone remote not found: "` | -| `email`/`password` | Valid credentials | `"authentication failed: invalid credentials"` | -| `token` | Valid token | `"authentication failed: invalid token"` | - ---- - -## Security - -### Credentials Storage - -**⚠️ CRITICAL:** All credentials are stored in **plain text** with no encryption. - -**Fields affected:** -- `email` - Plain text -- `password` - Plain text, **visible during interactive setup** -- `token` - Plain text - -**Protection:** -- ✅ File permissions (`0600` - owner read/write only) -- ✅ Directory permissions (`0700` - owner access only) -- ✅ Auto-fix for insecure permissions (Unix) - -**Mitigations:** -1. Use `token` instead of `password` (not visible during input) -2. Ensure `chmod 600 ~/.nugs/config.json` -3. Never commit config.json to version control -4. Add `config.json` to `.gitignore` - -### File Permissions - -**Automatic security checks:** - -```bash -# On every config read: -if permissions != 0600: - warn("Config has insecure permissions") - if unix: - chmod(config, 0600) # Auto-fix - warn("Auto-fix applied: chmod 600 ") - else: # Windows - warn("Fix manually: icacls /inheritance:r /grant:r %USERNAME%:RW") -``` - -**Expected permissions:** - -```bash -# Unix/Linux/macOS --rw------- 1 user user 436 Feb 07 23:13 config.json - -# Windows (via icacls) -owner:(R,W) -``` - -### Best Practices - -**DO:** -1. ✅ Store config in `~/.nugs/config.json` -2. ✅ Use `chmod 600 ~/.nugs/config.json` -3. ✅ Use token authentication when possible -4. ✅ Review auto-fix warnings -5. ✅ Add `config.json` to `.gitignore` - -**DON'T:** -1. ❌ Use world-readable permissions (`chmod 644`) -2. ❌ Commit config.json to version control -3. ❌ Share config files (contains credentials) -4. ❌ Store config in shared/network directories -5. ❌ Ignore permission warnings - ---- - -## Examples - -### Minimal Valid Config +# Nugs CLI Configuration + +This document covers every JSON field in `internal/model.Config`. The automated +documentation check fails when a JSON-tagged field is added without appearing +here. + +## File locations + +The first existing file wins; settings are not merged: + +1. `./config.json` +2. `~/.nugs/config.json` +3. `~/.config/nugs/config.json` + +Interactive first-run setup writes `~/.nugs/config.json`. On Unix, configuration +is created with mode `0600` and its parent directory with mode `0700`. + +## First-run setup + +Run `nugs` with no existing config. Setup prompts for: + +1. Email and a hidden password +2. Audio and video quality +3. Audio and video output directories +4. FFmpeg selection +5. Optional rclone remote, audio/video paths, transfers, and deletion behavior + +Catalog auto-refresh is initialized to daily at `05:00` in +`America/New_York`; use `nugs refresh set` to change it. + +## Complete field reference + +| JSON field | Type | Purpose and behavior | +|---|---|---| +| `email` | string | Nugs.net account email. Used with `password` when `token` is empty. | +| `password` | string | Nugs.net password. Stored in the config but hidden during interactive entry. | +| `format` | integer | Audio quality, 1–5. Required and validated at startup. | +| `outPath` | string | Local audio download directory. Defaults to `Nugs downloads` in setup. | +| `videoOutPath` | string | Local video directory. Setup defaults it to `outPath`. | +| `videoFormat` | integer | Video quality, 1–5. Required and validated at startup. | +| `defaultOutputs` | string | Default media selection: `audio`, `video`, or `both`; empty resolves to `audio`. | +| `wantRes` | string | Computed from `videoFormat`; do not set manually. Values are `480`, `720`, `1080`, `1440`, or `2160`. | +| `token` | string | Session token for Apple/Google authentication. A leading `Bearer ` is stripped. | +| `useFfmpegEnvVar` | boolean | Use `ffmpeg` from `PATH` when true. | +| `ffmpegNameStr` | string | Explicit FFmpeg executable when not using `PATH`. | +| `forceVideo` | boolean | Deprecated compatibility field; prefer a media modifier or `defaultOutputs`. | +| `skipVideos` | boolean | Deprecated compatibility field; prefer a media modifier or `defaultOutputs`. | +| `skipChapters` | boolean | Skip embedding chapter metadata into video output. | +| `rcloneEnabled` | boolean | Enable remote upload after downloads. | +| `rcloneRemote` | string | Configured rclone remote name. Runtime startup verifies rclone availability, not the remote name itself. | +| `rclonePath` | string | Remote audio destination. Never used as a local download path. | +| `rcloneVideoPath` | string | Remote video destination; setup defaults it to `rclonePath`. | +| `deleteAfterUpload` | boolean | Delete local media only after upload verification succeeds. | +| `rcloneTransfers` | integer | Parallel rclone transfers. Setup accepts positive integers and defaults to 4. | +| `catalogAutoRefresh` | boolean | Check at startup whether the catalog is due for refresh. | +| `catalogRefreshTime` | string | Local scheduled time in `HH:MM` form. | +| `catalogRefreshTimezone` | string | IANA timezone such as `America/New_York`. | +| `catalogRefreshInterval` | string | `hourly`, `daily`, or `weekly`. | +| `watchedArtists` | array of strings | Artist IDs managed by `nugs watch add/remove/list`. | +| `watchInterval` | string | Go duration for the generated watch timer, such as `1h`, `30m`, or `6h`. | +| `gotifyUrl` | string | Gotify server base URL used by watch notifications. | +| `gotifyToken` | string | Gotify application token. Notification priority is selected by the application. | +| `skipSizePreCalculation` | boolean | Skip size probing before downloads. When false, probes use 8 workers, 5-second track/request timeouts, and a 60-second overall maximum. | + +`urls` is runtime CLI state tagged `json:"-"`; it is intentionally not a config +field. + +## Format values + +### Audio `format` + +| Value | Format | +|---:|---| +| 1 | 16-bit 44.1 kHz ALAC | +| 2 | 16-bit 44.1 kHz FLAC | +| 3 | 24-bit 48 kHz MQA | +| 4 | 360 Reality Audio | +| 5 | 150 Kbps AAC | + +### Video `videoFormat` + +| Value | Resolution | +|---:|---| +| 1 | 480p | +| 2 | 720p | +| 3 | 1080p | +| 4 | 1440p | +| 5 | 4K / best available | + +## Example + +Use interactive setup for the initial file. A representative complete file is: ```json { "email": "user@example.com", - "password": "your_password", + "password": "replace-me", "format": 2, - "videoFormat": 5, - "outPath": "~/Music/Nugs" -} -``` - -### Token Authentication (Recommended) - -```json -{ - "token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9...", - "format": 2, - "videoFormat": 5, - "outPath": "~/Music/Nugs" -} -``` - -### Audio Only (No Videos) - -```json -{ - "email": "user@example.com", - "password": "your_password", - "format": 2, - "videoFormat": 5, + "videoFormat": 3, "defaultOutputs": "audio", - "outPath": "~/Music/Nugs" -} -``` - -### With Rclone Upload - -```json -{ - "email": "user@example.com", - "password": "your_password", - "format": 2, - "videoFormat": 5, - "outPath": "~/Music/Nugs", - "rcloneEnabled": true, - "rcloneRemote": "gdrive", - "rclonePath": "/Music/Nugs", - "deleteAfterUpload": true, - "rcloneTransfers": 8 -} -``` - -### Full Featured Config - -```json -{ - "token": "your_auth_token_here", - "format": 2, - "videoFormat": 5, - "defaultOutputs": "both", - "outPath": "/home/user/Music/Nugs", - "videoOutPath": "/home/user/Videos/Nugs", + "outPath": "/srv/media/music", + "videoOutPath": "/srv/media/video", "useFfmpegEnvVar": true, - "ffmpegNameStr": "", "skipChapters": false, - "rcloneEnabled": true, - "rcloneRemote": "gdrive", - "rclonePath": "/Music/Nugs", - "rcloneVideoPath": "/Videos/Nugs", - "deleteAfterUpload": true, - "rcloneTransfers": 8, + "rcloneEnabled": false, + "rcloneTransfers": 4, "catalogAutoRefresh": true, - "catalogRefreshTime": "03:00", - "catalogRefreshTimezone": "America/Los_Angeles", - "catalogRefreshInterval": "weekly", + "catalogRefreshTime": "05:00", + "catalogRefreshTimezone": "America/New_York", + "catalogRefreshInterval": "daily", + "watchedArtists": ["1125"], + "watchInterval": "1h", "skipSizePreCalculation": false } ``` -### Common Invalid Configs - -#### Invalid Format Values +For Apple/Google accounts, leave `email` and `password` empty and provide a +`token`; see [token.md](token.md). -```json -{ - "format": 6, // ❌ INVALID - must be 1-5 - "videoFormat": 0 // ❌ INVALID - must be 1-5 -} -``` +## Validation boundaries -**Error:** `track Format must be between 1 and 5` +Startup validates audio/video formats and `defaultOutputs`. Refresh configuration +validates time, timezone, and interval when set interactively. Rclone setup +validates that transfer count is positive, while normal startup only checks that +the rclone executable is available. Remote reachability is discovered during +remote operations. -#### Invalid defaultOutputs - -```json -{ - "defaultOutputs": "videos" // ❌ INVALID - typo ("videos" vs "video") -} -``` - -**Error:** `invalid defaultOutputs: "videos" (must be audio, video, or both)` - -#### Invalid Timezone - -```json -{ - "catalogRefreshTimezone": "EST" // ❌ INVALID - use IANA timezone -} -``` - -**Error:** `invalid timezone EST: unknown time zone EST` - -**Fix:** Use `"America/New_York"` instead - -#### Invalid Refresh Time - -```json -{ - "catalogRefreshTime": "5am" // ❌ INVALID - wrong format -} -``` +## Security -**Error:** `invalid refresh time format: 5am (expected HH:MM)` +Credentials are stored as plaintext JSON protected by filesystem permissions. -**Fix:** Use `"05:00"` +- Keep the file at mode `0600` on Unix. +- Never commit it or include it in logs/issues. +- Prefer a token for Apple/Google accounts. +- Use HTTPS for Gotify endpoints. +- Do not pass credentials in CLI arguments. -#### Invalid Refresh Interval +## Auto-refresh -```json -{ - "catalogRefreshInterval": "hourly" // ❌ INVALID - not supported -} +```bash +nugs refresh enable +nugs refresh disable +nugs refresh set ``` -**Error:** `invalid interval: hourly (must be 'daily' or 'weekly')` +Supported intervals are hourly, daily, and weekly. Hourly refresh uses elapsed +time; daily/weekly schedules use `catalogRefreshTime` and +`catalogRefreshTimezone`. -#### Missing Rclone Configuration +## Watch and Gotify -```json -{ - "rcloneEnabled": true, - "rcloneRemote": "", // ❌ Missing - "rclonePath": "" // ❌ Missing -} +```bash +nugs watch add 1125 +nugs watch list +nugs watch check +nugs watch enable ``` -**Result:** Runtime errors when attempting uploads +`watchedArtists` is normally modified through the CLI. `watchInterval` controls +the generated systemd timer. Gotify is enabled when both `gotifyUrl` and +`gotifyToken` are configured. -**Fix:** Set `rcloneRemote` and `rclonePath` or disable rclone +## Rclone paths ---- +- Local audio: `outPath` +- Local video: `videoOutPath` +- Remote audio: `:` +- Remote video: `:` ## Migrations -### Breaking Change: rclonePath Behavior (2026-02-05) - -**⚠️ BREAKING CHANGE:** The `rclonePath` field no longer affects local download paths. - -#### Previous Behavior (before 2026-02-05) - -```go -// rclonePath was used as a fallback for local base path -basePath := cfg.OutPath -if cfg.RclonePath != "" { - basePath = cfg.RclonePath // ❌ Confusing dual-purpose -} -``` - -#### New Behavior (after 2026-02-05) +### `rclonePath` local-path behavior (2026-02-05) -```go -// rclonePath ONLY affects remote storage uploads -basePath := cfg.OutPath // Always use outPath for local downloads -remotePath := cfg.RclonePath // Only for remote storage -``` - -#### Migration Guide - -If you previously set `rclonePath` expecting it to control local download locations: - -**1. Update your config:** -- Move your desired local path to `outPath` -- Keep `rclonePath` for the remote storage path only - -**2. Example migration:** +Older versions could use `rclonePath` as a local path fallback. Current versions +never do so. Set local directories explicitly: ```json -// OLD CONFIG (relied on rclonePath for local paths) { - "outPath": "/tmp/music", - "rclonePath": "/mnt/user/data/media/music", - "rcloneEnabled": false -} - -// NEW CONFIG (explicit local path in outPath) -{ - "outPath": "/mnt/user/data/media/music", - "rclonePath": "/Music", + "outPath": "/mnt/media/music", + "videoOutPath": "/mnt/media/video", "rcloneEnabled": true, - "rcloneRemote": "gdrive" + "rcloneRemote": "archive", + "rclonePath": "/Music", + "rcloneVideoPath": "/Video" } ``` -**3. Impact:** -- **Local-only users:** Update `outPath` to your preferred download location -- **Rclone users:** `outPath` for local, `rclonePath` for remote (clear separation) - -#### Rationale - -This change eliminates a confusing "leaky abstraction" where a field named "rclone**Path**" (implying remote storage) was also controlling local filesystem behavior. The new design provides clear separation of concerns: `outPath` = local, `rclonePath` = remote. - ---- - -## Environment Variables - -### Currently Supported - -| Variable | Purpose | Where Used | -|----------|---------|------------| -| `NUGS_DETACHED` | Process running in background | `internal/runtime/status.go:17` | -| `NUGS_THEME` | Color theme ("vivid" or "nordonedark") | `internal/ui/theme.go:46` | -| `HOME` | User home directory for config/cache paths | Config path resolution | -| `PATH` | FFmpeg binary resolution | `internal/config/config.go` | -| `TERM` | Terminal color capability detection | `internal/ui/theme.go` | -| `COLORTERM` | Truecolor support detection | `internal/ui/theme.go` | - -### Future Consideration - -These environment variables are **NOT** currently implemented but documented in Python client: - -- `NUGS_EMAIL` - Override email from config -- `NUGS_PASSWORD` - Override password from config -- `NUGS_TOKEN` - Override token from config -- `NUGS_FORMAT` - Override audio format -- `NUGS_VIDEO_FORMAT` - Override video format - -**Note:** The Go CLI uses config.json exclusively. Environment variable overrides may be added in future versions. - ---- - -## Interactive Setup - -First-time setup prompts: - -```bash -$ nugs 23329 -Config not found. Let's set it up! - -Enter your Nugs email: user@example.com -Enter your Nugs password: ****** - -Audio format: - 1. 16-bit 44.1kHz ALAC - 2. 16-bit 44.1kHz FLAC - 3. 24-bit 48kHz MQA - 4. 360 Reality Audio (default) - 5. 150 Kbps AAC -Select format (1-5) [4]: 2 - -Video format: - 1. 480p - 2. 720p - 3. 1080p - 4. 1440p - 5. 4K / best (default) -Select format (1-5) [5]: 5 - -Enter output path for audio downloads [Nugs downloads]: ~/Music/Nugs - -Use FFmpeg from PATH? (y/n) [y]: y - -Enable rclone uploads? (y/n) [n]: y -Enter rclone remote name: gdrive -Enter remote path for audio uploads: /Music/Nugs -Number of rclone transfers [4]: 8 -Delete local files after upload? (y/n) [y]: y - -Catalog auto-refresh enabled by default. -Refresh time (HH:MM) [05:00]: 03:00 -Timezone [America/New_York]: America/Los_Angeles -Refresh interval (daily/weekly) [daily]: weekly - -✓ Config saved to ~/.nugs/config.json -``` - ---- - -## Troubleshooting - -### Config File Not Found - -**Error:** "Config file not found" - -**Solution:** -1. Run `nugs` with any command to trigger interactive setup -2. Manually create `~/.nugs/config.json` with minimal config - -### Insecure Permissions - -**Warning:** "Config file has insecure permissions (0644)" - -**Solution:** -- Unix: `chmod 600 ~/.nugs/config.json` (auto-fixed) -- Windows: `icacls ~/.nugs/config.json /inheritance:r /grant:r %USERNAME%:RW` - -### Invalid Format Values - -**Error:** "track Format must be between 1 and 5" - -**Solution:** Set `format` to a value between 1-5 - -### Invalid Timezone - -**Error:** "invalid timezone EST: unknown time zone EST" - -**Solution:** Use IANA timezone (e.g., "America/New_York" instead of "EST") - -**Find your timezone:** https://en.wikipedia.org/wiki/List_of_tz_database_time_zones - -### Rclone Not Found - -**Error:** "rclone is not installed or not available in PATH" - -**Solution:** -1. Install rclone: https://rclone.org/downloads/ -2. Or disable rclone: `"rcloneEnabled": false` - -### FFmpeg Not Found - -**Error:** "ffmpeg not found in PATH" - -**Solution:** -1. Install FFmpeg: https://ffmpeg.org/download.html -2. Set custom path: `"ffmpegNameStr": "/path/to/ffmpeg"` -3. Place `ffmpeg` binary in project root - ---- - -## See Also +## Environment variables -- [CLAUDE.md](../CLAUDE.md) - Development guide and quick start -- [ARCHITECTURE.md](./ARCHITECTURE.md) - Package structure and patterns -- [README.md](../README.md) - User documentation -- [token.md](./token.md) - Token extraction guide +The Go CLI does not provide environment-variable overrides for configuration +fields. Environment variables used by CI or unrelated historical clients are not +part of this CLI contract. diff --git a/docs/QUICK_REFERENCE.md b/docs/QUICK_REFERENCE.md index 56b6295..42ff2b2 100644 --- a/docs/QUICK_REFERENCE.md +++ b/docs/QUICK_REFERENCE.md @@ -1,194 +1,59 @@ -# Nugs Client - Quick Reference Guide +# Nugs CLI Quick Reference -## Installation +## Setup ```bash -cd /home/jmagar/code/nugs-client -uv sync +make build +nugs +chmod 600 ~/.nugs/config.json ``` -## Basic Usage - -```python -import asyncio -from nugs_client import NugsClient - -async def main(): - async with NugsClient() as client: - # Downloads will work once you connect the pieces - await client.download_album("23329") - -asyncio.run(main()) -``` - -## Configuration (.env file) +## Download ```bash -NUGS_EMAIL=your@email.com -NUGS_PASSWORD=your_password -NUGS_AUDIO_FORMAT=2 # 1=ALAC, 2=FLAC, 3=MQA, 4=360RA, 5=AAC -NUGS_VIDEO_FORMAT=3 # 1=480p, 2=720p, 3=1080p, 4=1440p, 5=4K -NUGS_OUTPUT_PATH=downloads -``` - -## Common Tasks - -### Get Album Info -```python -album = await client.get_album("23329") -print(f"{album.artist_name} - {album.container_info}") -print(f"Tracks: {len(album.all_tracks)}") - -# Cover artwork URL (public) -print(album.cover_art_url) -``` - -### Get Artist Catalog -```python -containers = await client.get_artist("461") -print(f"Found {len(containers)} releases") -``` - -### Download Album -```python -path = await client.download_album("23329", quality=2) # FLAC -``` - -### Download Video -```python -path = await client.download_video("27323", resolution=3) # 1080p +nugs grab 23329 +nugs grab 23329 video +nugs 1125 latest audio +nugs 1125 full ``` -### Download from URL -```python -path = await client.download_url("https://play.nugs.net/release/23329") -``` - -## Testing +## Browse and catalog ```bash -# Run tests -uv run pytest - -# With coverage -uv run pytest --cov - -# Type check -uv run mypy src/nugs_client - -# Lint -uv run ruff check src/nugs_client +nugs list +nugs list 1125 video +nugs list 1125 latest 5 +nugs update +nugs stats +nugs latest 50 +nugs gaps 1125 audio +nugs coverage ``` -## Project Structure +`nugs latest` cannot filter by media. With no IDs, `nugs coverage` examines +artists found in local and configured remote download folders. -``` -src/nugs_client/ -├── client.py # Main NugsClient class -├── auth.py # Authentication -├── endpoints.py # API endpoints -├── downloader.py # Download manager -├── hls.py # HLS processing -├── video.py # Video processing -├── models.py # Data models -├── config.py # Configuration -├── constants.py # API constants -├── utils.py # Utilities -└── exceptions.py # Exceptions -``` - -## Key Classes - -- **NugsClient** - Main client, use with `async with` -- **NugsConfig** - Configuration management -- **AuthHandler** - Authentication and tokens -- **CatalogAPI** - Metadata fetching -- **StreamAPI** - Stream URL generation -- **DownloadManager** - File downloads -- **HLSProcessor** - HLS stream processing -- **VideoProcessor** - Video downloads - -## Error Handling +## Watch -```python -from nugs_client.exceptions import ( - AuthenticationError, - DownloadError, - HLSError, - ContentNotFoundError, -) - -try: - async with NugsClient() as client: - await client.download_album("23329") -except AuthenticationError: - print("Login failed") -except ContentNotFoundError: - print("Album not found") -except DownloadError: - print("Download failed") +```bash +nugs watch add 1125 +nugs watch list +nugs watch check +nugs watch enable +nugs watch disable ``` -## Audio Formats - -| Format | Quality | Format ID | -|--------|---------|-----------| -| ALAC | 16-bit / 44.1 kHz | 1 | -| FLAC | 16-bit / 44.1 kHz | 2 | -| MQA | 24-bit / 48 kHz | 3 | -| 360 RA | Spatial | 4 | -| AAC | 150 Kbps | 5 | - -## Video Formats - -| Resolution | Format ID | -|------------|-----------| -| 480p | 1 | -| 720p | 2 | -| 1080p | 3 | -| 1440p | 4 | -| 4K | 5 | +## Interactive controls -## Supported URLs +- `Shift+P`: pause or resume +- `Shift+C`: cancel +- `Ctrl+C`: interrupt -- Albums: `https://play.nugs.net/release/23329` -- Artists: `https://play.nugs.net/artist/461` -- Playlists: `https://play.nugs.net/#/playlists/playlist/1215400` -- Catalog: `https://2nu.gs/3PmqXLW` -- Videos: `https://play.nugs.net/#/videos/artist/.../27323` - -## Dependencies - -### Runtime -- httpx - HTTP client -- pydantic - Data validation -- m3u8 - HLS parsing -- pycryptodome - Encryption -- rich - Progress bars - -### External -- FFmpeg - Required for HLS and video - -## Documentation - -- `README.md` - Complete user guide -- `COMPLETION_SUMMARY.md` - Feature overview -- `FINAL_STATUS.md` - Implementation status -- `NEXT_STEPS.md` - Future enhancements - -## Help +## Verify ```bash -# Verify installation -python verify_installation.py - -# Check structure -tree -L 3 - -# View logs -# (Add logging configuration as needed) +make test +make verify ``` ---- - -**Quick Start**: Copy `.env.example` to `.env`, add credentials, run `basic_usage.py`! +See [COMMANDS.md](COMMANDS.md) and [CONFIG.md](CONFIG.md) for complete contracts. diff --git a/docs/nugs-api-endpoints.md b/docs/nugs-api-endpoints.md index 5de1089..8e34690 100644 --- a/docs/nugs-api-endpoints.md +++ b/docs/nugs-api-endpoints.md @@ -1,8 +1,7 @@ # Known nugs API endpoints (from current codebase) -This document enumerates endpoints we can **prove** exist based on references in: -- `nugs-client` (Python) -- `nugs` (Go reference) +This document enumerates endpoints we can **prove** exist based on the Go CLI in +this repository and captured upstream response contracts. It intentionally avoids undocumented speculation. @@ -116,8 +115,9 @@ This is a method-driven endpoint; behavior changes based on query parameters. - **Headers** - `User-Agent: ` -This is implemented in the Python client as: -- `CatalogAPI.get_artist_metadata(artist_id, offset=1, limit=100)` +The Go client implements this in `internal/api/client.go`. Callers must preserve +the availability contract: `availType=1` means currently downloadable content; +`availType=2` is PREORDER content. #### `method=catalog.latest` - **Purpose** @@ -224,10 +224,8 @@ This endpoint is used both for audio and subscription-gated video manifests. - If you want broader coverage, we should do method discovery from web traffic and/or systematic probing. ## Where this is implemented -- Python: - - `src/nugs_client/auth.py` - - `src/nugs_client/endpoints.py` - - `src/nugs_client/downloader.py` -- Go: - - `nugs/main.go` - - `nugs/structs.go` + +- API requests and authentication: `internal/api/` +- API response models: `internal/model/api_types.go` +- URL parsing: `internal/api/url_parser.go` +- CLI routing: `cmd/nugs/main.go` diff --git a/docs/nugs-client-README.md b/docs/nugs-client-README.md deleted file mode 100644 index 6caa0ea..0000000 --- a/docs/nugs-client-README.md +++ /dev/null @@ -1,345 +0,0 @@ -# Nugs.net Python Client - -> **✅ PRODUCTION READY** - Modern async Python client for the nugs.net music streaming and download API - -[![Python 3.11+](https://img.shields.io/badge/python-3.11+-blue.svg)](https://www.python.org/downloads/) -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) -[![Status](https://img.shields.io/badge/status-production%20ready-brightgreen.svg)](https://github.com/) - -**🎉 All download functionality is now complete and working!** - -## Features - -- 🎵 **Complete API Coverage**: Albums, videos, playlists, livestreams, artist catalogs -- ⚡ **Async-First**: Built with `httpx` and `asyncio` for high performance -- 🎯 **Type Safe**: Full type hints and Pydantic models -- 🔐 **Dual Authentication**: Email/password or pre-existing token support -- 📦 **Quality Selection**: All formats (ALAC, FLAC, MQA, 360 RA, AAC) with automatic fallback -- 📹 **Video Support**: All resolutions (480p-4K) with chapter embedding -- 🔄 **Resume Support**: Interrupted downloads can be resumed -- 🎬 **HLS Processing**: Automatic manifest parsing, decryption, and conversion -- 📊 **Progress Tracking**: Rich progress bars for downloads - -## Installation - -### Using uv (recommended) -```bash -uv add nugs-client -``` - -### Using pip -```bash -pip install nugs-client -``` - -### Development Installation -```bash -git clone https://github.com/yourusername/nugs-client.git -cd nugs-client -uv sync --dev -``` - -## Requirements - -- Python 3.11 or higher -- FFmpeg (for HLS tracks and video processing) - - **Linux**: `sudo apt install ffmpeg` - - **macOS**: `brew install ffmpeg` - - **Windows**: Download from [FFmpeg website](https://ffmpeg.org/download.html) - -## Quick Start - -### Basic Usage - -```python -import asyncio -from nugs_client import NugsClient, NugsConfig - -async def main(): - # Configure the client - config = NugsConfig( - email="your.email@example.com", - password="your_password", - audio_format=2, # FLAC - output_path="downloads" - ) - - # Download an album - async with NugsClient(config) as client: - album_path = await client.download_album("23329") - print(f"Downloaded to: {album_path}") - -if __name__ == "__main__": - asyncio.run(main()) -``` - -### Using Environment Variables - -```bash -# Create .env file -cp .env.example .env -# Edit .env with your credentials -``` - -```python -from nugs_client import NugsClient - -async def main(): - # Automatically loads from .env - async with NugsClient() as client: - await client.download_album("23329") -``` - -## Authentication - -### Option 1: Email/Password -```python -config = NugsConfig( - email="your.email@example.com", - password="your_password" -) -``` - -### Option 2: Pre-existing Token -For Apple/Google authenticated accounts: -```python -config = NugsConfig( - token="your_access_token_here" -) -``` - -See [token.md](https://github.com/Sorrow446/Nugs-Downloader/blob/main/token.md) for how to extract tokens. - -## Supported Content Types - -| Content Type | URL Example | Method | -|--------------|-------------|--------| -| Album | `https://play.nugs.net/release/23329` | `download_album(id)` | -| Artist Catalog | `https://play.nugs.net/artist/461` | `download_artist(id)` | -| User Playlist | `https://play.nugs.net/#/playlists/playlist/1215400` | `download_playlist(id)` | -| Catalog Playlist | `https://2nu.gs/3PmqXLW` | `download_catalog_playlist(url)` | -| Video | `https://play.nugs.net/#/videos/artist/.../27323` | `download_video(id)` | -| Livestream | `https://play.nugs.net/watch/livestreams/exclusive/30119` | `download_video(id, is_livestream=True)` | - -## Audio Formats - -| Format | Quality | Format ID | -|--------|---------|-----------| -| ALAC | 16-bit / 44.1 kHz | 1 | -| FLAC | 16-bit / 44.1 kHz | 2 (default) | -| MQA | 24-bit / 48 kHz | 3 | -| 360 Reality Audio | Spatial | 4 | -| AAC | 150 Kbps | 5 | - -The client automatically selects the best available format and falls back gracefully if unavailable. - -## Video Formats - -| Resolution | Format ID | -|------------|-----------| -| 480p | 1 | -| 720p | 2 | -| 1080p | 3 (default) | -| 1440p | 4 | -| 4K/UHD | 5 | - -## Advanced Usage - -### Download with Custom Quality -```python -async with NugsClient() as client: - # MQA audio - await client.download_album("23329", quality=3) - - # 4K video - await client.download_video("27323", resolution=5) -``` - -### Download Entire Artist Catalog -```python -async with NugsClient() as client: - paths = await client.download_artist("461", skip_videos=False) - print(f"Downloaded {len(paths)} albums") -``` - -### Download Playlist -```python -async with NugsClient() as client: - # User playlist - await client.download_playlist("1215400") - - # Catalog playlist (short link) - await client.download_catalog_playlist("https://2nu.gs/3PmqXLW") -``` - -### Video with Chapters -```python -config = NugsConfig( - video_format=3, # 1080p - skip_chapters=False # Embed chapters -) - -async with NugsClient(config) as client: - await client.download_video("27323") -``` - -### Concurrent Downloads -```python -from nugs_client import DownloadManager - -config = NugsConfig(max_concurrent=5) # 5 simultaneous downloads -async with NugsClient(config) as client: - await client.download_artist("461") -``` - -## Configuration Options - -All options can be set via `NugsConfig` or environment variables: - -| Option | Env Var | Default | Description | -|--------|---------|---------|-------------| -| `email` | `NUGS_EMAIL` | None | Account email | -| `password` | `NUGS_PASSWORD` | None | Account password | -| `token` | `NUGS_TOKEN` | None | Pre-existing access token | -| `audio_format` | `NUGS_AUDIO_FORMAT` | 2 (FLAC) | Audio quality (1-5) | -| `video_format` | `NUGS_VIDEO_FORMAT` | 3 (1080p) | Video resolution (1-5) | -| `output_path` | `NUGS_OUTPUT_PATH` | "downloads" | Download directory | -| `use_system_ffmpeg` | `NUGS_USE_SYSTEM_FFMPEG` | True | Use system FFmpeg | -| `skip_videos` | `NUGS_SKIP_VIDEOS` | False | Skip videos in artist downloads | -| `skip_chapters` | `NUGS_SKIP_CHAPTERS` | False | Skip video chapter embedding | -| `max_concurrent` | `NUGS_MAX_CONCURRENT` | 3 | Concurrent downloads | -| `retry_attempts` | `NUGS_RETRY_ATTEMPTS` | 3 | HTTP retry attempts | - -## CLI Usage (Optional) - -Install with CLI support: -```bash -uv add nugs-client[cli] -``` - -Download content: -```bash -nugs download "https://play.nugs.net/release/23329" -nugs download --audio-format 3 --output ~/music "https://play.nugs.net/artist/461" -``` - -Authenticate: -```bash -nugs auth your.email@example.com -``` - -## Troubleshooting - -### FFmpeg Not Found -``` -FFmpegError: FFmpeg not found in system PATH -``` -**Solution**: Install FFmpeg or set `use_system_ffmpeg=False` and place `ffmpeg` binary in project root. - -### Authentication Failed -``` -AuthenticationError: Invalid credentials -``` -**Solution**: -- Verify email/password are correct -- For Apple/Google accounts, extract and use access token -- Check if subscription is active - -### Quality Not Available -``` -QualityNotAvailableError: MQA not available for this track -``` -**Solution**: The client automatically falls back to lower quality. This is informational. - -### HLS Decryption Failed -``` -HLSError: Failed to decrypt segment -``` -**Solution**: -- Ensure FFmpeg is installed -- Check internet connection -- Try again (may be temporary API issue) - -## Development - -### Setup -```bash -git clone https://github.com/yourusername/nugs-client.git -cd nugs-client -uv sync --dev -``` - -### Run Tests -```bash -# Unit tests only -uv run pytest -m "not integration" - -# All tests (requires credentials) -export NUGS_EMAIL="your@email.com" -export NUGS_PASSWORD="your_password" -uv run pytest - -# With coverage -uv run pytest --cov -``` - -### Type Checking -```bash -uv run mypy src/nugs_client -``` - -### Linting -```bash -uv run ruff check src/ tests/ -uv run ruff format src/ tests/ -``` - -## Architecture - -``` -nugs_client/ -├── client.py # Main NugsClient class -├── auth.py # Authentication handler -├── models.py # Pydantic data models -├── endpoints.py # API endpoint methods -├── downloader.py # Download manager -├── hls.py # HLS manifest parsing & decryption -├── video.py # Video processing -├── constants.py # API constants -├── exceptions.py # Custom exceptions -└── utils.py # Helper utilities -``` - -## Contributing - -Contributions are welcome! Please: - -1. Fork the repository -2. Create a feature branch (`git checkout -b feature/amazing-feature`) -3. Make your changes -4. Add tests -5. Run linting and tests -6. Commit (`git commit -m 'Add amazing feature'`) -7. Push (`git push origin feature/amazing-feature`) -8. Open a Pull Request - -## License - -MIT License - see [LICENSE](LICENSE) file for details - -## Disclaimer - -- This project is not affiliated with nugs.net -- Use responsibly and respect content licensing -- Ensure you have proper subscription/purchase rights for downloaded content - -## Credits - -- Based on the excellent [Nugs-Downloader](https://github.com/Sorrow446/Nugs-Downloader) by Sorrow446 -- API documentation derived from reverse engineering efforts - -## Links - -- [nugs.net Official Site](https://www.nugs.net/) -- [Issue Tracker](https://github.com/yourusername/nugs-client/issues) -- [Changelog](CHANGELOG.md) diff --git a/go.mod b/go.mod index bbc5594..65a5b85 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/jmagar/nugs-cli -go 1.24.12 +go 1.25.12 require ( github.com/alexflint/go-arg v1.5.0 diff --git a/internal/api/apilog.go b/internal/api/apilog.go index 796a27d..d7b00c1 100644 --- a/internal/api/apilog.go +++ b/internal/api/apilog.go @@ -13,29 +13,33 @@ import ( // Each field uses snake_case JSON keys for easy grep/jq consumption. type APILogEntry struct { Timestamp string `json:"ts"` - Event string `json:"event"` // "request", "retry", "rate_limit_wait", "circuit_open", "circuit_closed", "circuit_rejected" - Label string `json:"label,omitempty"` // human-readable API endpoint name - StatusCode int `json:"status_code,omitempty"` // HTTP status (0 = network error) - DurationMS int64 `json:"duration_ms,omitempty"` // round-trip time - Attempt int `json:"attempt,omitempty"` // retry attempt (0 = first try) + Event string `json:"event"` // "request", "retry", "rate_limit_wait", "circuit_open", "circuit_closed", "circuit_rejected" + Label string `json:"label,omitempty"` // human-readable API endpoint name + StatusCode int `json:"status_code,omitempty"` // HTTP status (0 = network error) + DurationMS int64 `json:"duration_ms,omitempty"` // round-trip time + Attempt int `json:"attempt,omitempty"` // retry attempt (0 = first try) RateLimitedMS int64 `json:"rate_limited_ms,omitempty"` // ms spent waiting for rate limiter - CircuitState string `json:"circuit_state,omitempty"` // closed / open / half-open + CircuitState string `json:"circuit_state,omitempty"` // closed / open / half-open Error string `json:"error,omitempty"` } // apiLogger writes structured JSON-line entries to a dedicated log file. // All methods are safe for concurrent use. type apiLogger struct { - mu sync.Mutex - enc *json.Encoder - f *os.File + mu sync.Mutex + enc *json.Encoder + f *os.File + path string } // Logger is the package-level API logger. It is nil until InitAPILogger is called. // All log functions are no-ops when Logger is nil. var Logger *apiLogger +var loggerMu sync.RWMutex -var loggerOnce sync.Once +var apiLogMaxBytes int64 = 5 << 20 + +const apiLogBackups = 3 // InitAPILogger opens (or creates) the dedicated API log file at logPath. // It must be called once at startup before any API requests are made. @@ -43,20 +47,75 @@ var loggerOnce sync.Once // Returns a non-nil error only when the file cannot be opened; in that case // logging is silently disabled (all other operations continue normally). func InitAPILogger(logPath string) error { - var initErr error - loggerOnce.Do(func() { - if err := os.MkdirAll(filepath.Dir(logPath), 0700); err != nil { - initErr = fmt.Errorf("api logger: mkdir %s: %w", filepath.Dir(logPath), err) - return - } - f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) - if err != nil { - initErr = fmt.Errorf("api logger: open %s: %w", logPath, err) - return - } - Logger = &apiLogger{f: f, enc: json.NewEncoder(f)} - }) - return initErr + if err := os.MkdirAll(filepath.Dir(logPath), 0700); err != nil { + return fmt.Errorf("api logger: mkdir %s: %w", filepath.Dir(logPath), err) + } + f, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + return fmt.Errorf("api logger: open %s: %w", logPath, err) + } + if err := f.Chmod(0600); err != nil { + _ = f.Close() + return fmt.Errorf("api logger: chmod %s: %w", logPath, err) + } + loggerMu.Lock() + old := Logger + Logger = &apiLogger{f: f, enc: json.NewEncoder(f), path: logPath} + loggerMu.Unlock() + if old != nil { + return old.close() + } + return nil +} + +// CloseAPILogger flushes and closes the logger. A later InitAPILogger may retry. +func CloseAPILogger() error { + loggerMu.Lock() + old := Logger + Logger = nil + loggerMu.Unlock() + if old == nil { + return nil + } + return old.close() +} + +func currentLogger() *apiLogger { + loggerMu.RLock() + defer loggerMu.RUnlock() + return Logger +} + +func (l *apiLogger) close() error { + l.mu.Lock() + defer l.mu.Unlock() + if l.f == nil { + return nil + } + err := l.f.Close() + l.f = nil + return err +} + +func (l *apiLogger) rotateIfNeeded() error { + info, err := l.f.Stat() + if err != nil || info.Size() < apiLogMaxBytes { + return err + } + if err := l.f.Close(); err != nil { + return err + } + for i := apiLogBackups - 1; i >= 1; i-- { + _ = os.Rename(fmt.Sprintf("%s.%d", l.path, i), fmt.Sprintf("%s.%d", l.path, i+1)) + } + if err := os.Rename(l.path, l.path+".1"); err != nil && !os.IsNotExist(err) { + return err + } + l.f, err = os.OpenFile(l.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err == nil { + l.enc = json.NewEncoder(l.f) + } + return err } // write is the internal append function. Failures are silently ignored — @@ -65,12 +124,16 @@ func (l *apiLogger) write(e APILogEntry) { e.Timestamp = time.Now().UTC().Format(time.RFC3339Nano) l.mu.Lock() defer l.mu.Unlock() + if l.f == nil || l.rotateIfNeeded() != nil { + return + } _ = l.enc.Encode(e) } // LogRequest records a completed HTTP request (success or API-level error). func LogRequest(label string, statusCode int, duration time.Duration, attempt int, circState string, reqErr error) { - if Logger == nil { + logger := currentLogger() + if logger == nil { return } e := APILogEntry{ @@ -87,15 +150,16 @@ func LogRequest(label string, statusCode int, duration time.Duration, attempt in if attempt > 0 { e.Event = "retry" } - Logger.write(e) + logger.write(e) } // LogRateLimitWait records that a request was delayed by the rate limiter. func LogRateLimitWait(label string, waited time.Duration) { - if Logger == nil { + logger := currentLogger() + if logger == nil { return } - Logger.write(APILogEntry{ + logger.write(APILogEntry{ Event: "rate_limit_wait", Label: label, RateLimitedMS: waited.Milliseconds(), @@ -104,10 +168,11 @@ func LogRateLimitWait(label string, waited time.Duration) { // LogCircuitStateChange records a circuit breaker state transition. func LogCircuitStateChange(event, label, fromState, toState string) { - if Logger == nil { + logger := currentLogger() + if logger == nil { return } - Logger.write(APILogEntry{ + logger.write(APILogEntry{ Event: event, Label: label, CircuitState: toState, @@ -118,10 +183,11 @@ func LogCircuitStateChange(event, label, fromState, toState string) { // LogCircuitRejected records a request that was rejected immediately because // the circuit breaker is open. func LogCircuitRejected(label string) { - if Logger == nil { + logger := currentLogger() + if logger == nil { return } - Logger.write(APILogEntry{ + logger.write(APILogEntry{ Event: "circuit_rejected", Label: label, CircuitState: "open", diff --git a/internal/api/apilog_test.go b/internal/api/apilog_test.go new file mode 100644 index 0000000..3fbf396 --- /dev/null +++ b/internal/api/apilog_test.go @@ -0,0 +1,51 @@ +package api + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestAPILoggerInitCanRetryAfterFailure(t *testing.T) { + defer CloseAPILogger() + root := t.TempDir() + blocker := filepath.Join(root, "not-a-directory") + if err := os.WriteFile(blocker, []byte("x"), 0600); err != nil { + t.Fatal(err) + } + if err := InitAPILogger(filepath.Join(blocker, "api.log")); err == nil { + t.Fatal("initialization unexpectedly succeeded") + } + valid := filepath.Join(root, "logs", "api.log") + if err := InitAPILogger(valid); err != nil { + t.Fatalf("retry initialization: %v", err) + } + LogRequest("test", 200, time.Millisecond, 0, "closed", nil) + if err := CloseAPILogger(); err != nil { + t.Fatal(err) + } + info, err := os.Stat(valid) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0600 { + t.Fatalf("log mode = %o, want 600", info.Mode().Perm()) + } +} + +func TestAPILoggerRotates(t *testing.T) { + defer CloseAPILogger() + oldMax := apiLogMaxBytes + apiLogMaxBytes = 1 + defer func() { apiLogMaxBytes = oldMax }() + logPath := filepath.Join(t.TempDir(), "api.log") + if err := InitAPILogger(logPath); err != nil { + t.Fatal(err) + } + LogRequest("first", 200, 0, 0, "closed", nil) + LogRequest("second", 200, 0, 0, "closed", nil) + if _, err := os.Stat(logPath + ".1"); err != nil { + t.Fatalf("rotated log missing: %v", err) + } +} diff --git a/internal/api/circuit.go b/internal/api/circuit.go index 83fd64c..b18bf5f 100644 --- a/internal/api/circuit.go +++ b/internal/api/circuit.go @@ -36,12 +36,13 @@ func (s circuitState) String() string { // stays open for `resetTimeout`, then enters half-open to probe recovery. // All methods are safe for concurrent use. type circuitBreaker struct { - mu sync.Mutex - state circuitState - consecutive int // consecutive failure count - threshold int // failures required to open the circuit - resetTimeout time.Duration // how long to stay open before probing - openedAt time.Time + mu sync.Mutex + state circuitState + consecutive int // consecutive failure count + threshold int // failures required to open the circuit + resetTimeout time.Duration // how long to stay open before probing + openedAt time.Time + probeInFlight bool } func newCircuitBreaker(threshold int, resetTimeout time.Duration) *circuitBreaker { @@ -65,10 +66,15 @@ func (cb *circuitBreaker) Allow() (circuitState, bool) { case circuitOpen: if time.Since(cb.openedAt) >= cb.resetTimeout { cb.state = circuitHalfOpen + cb.probeInFlight = true return circuitHalfOpen, true } return circuitOpen, false case circuitHalfOpen: + if cb.probeInFlight { + return circuitHalfOpen, false + } + cb.probeInFlight = true return circuitHalfOpen, true } return cb.state, false @@ -83,6 +89,7 @@ func (cb *circuitBreaker) RecordSuccess() (prev circuitState) { prev = cb.state cb.consecutive = 0 cb.state = circuitClosed + cb.probeInFlight = false return prev } @@ -96,6 +103,7 @@ func (cb *circuitBreaker) RecordFailure() (newState circuitState) { if cb.state == circuitHalfOpen || cb.consecutive >= cb.threshold { cb.state = circuitOpen cb.openedAt = time.Now() + cb.probeInFlight = false } return cb.state } diff --git a/internal/api/circuit_test.go b/internal/api/circuit_test.go new file mode 100644 index 0000000..8d15ca1 --- /dev/null +++ b/internal/api/circuit_test.go @@ -0,0 +1,46 @@ +package api + +import ( + "sync" + "testing" + "time" +) + +func TestCircuitBreakerAllowsOnlyOneHalfOpenProbe(t *testing.T) { + cb := newCircuitBreaker(1, time.Nanosecond) + cb.RecordFailure() + time.Sleep(time.Millisecond) + + const callers = 32 + var wg sync.WaitGroup + var mu sync.Mutex + allowed := 0 + for range callers { + wg.Add(1) + go func() { + defer wg.Done() + _, ok := cb.Allow() + if ok { + mu.Lock() + allowed++ + mu.Unlock() + } + }() + } + wg.Wait() + if allowed != 1 { + t.Fatalf("half-open allowed %d probes, want 1", allowed) + } +} + +func TestFailureDomainsAreScoped(t *testing.T) { + if got := failureDomain("auth"); got != "identity" { + t.Fatalf("auth domain = %q", got) + } + if got := failureDomain("subPlayer.aspx"); got != "stream" { + t.Fatalf("player domain = %q", got) + } + if got := failureDomain("catalog.container"); got != "catalog" { + t.Fatalf("catalog domain = %q", got) + } +} diff --git a/internal/api/client.go b/internal/api/client.go index 3a30a16..e11ad96 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -6,11 +6,13 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "net/http/cookiejar" "net/url" "strconv" "strings" + "sync" "time" "github.com/jmagar/nugs-cli/internal/model" @@ -33,8 +35,8 @@ const ( ) var ( - Jar = mustCookieJar() - Client = &http.Client{ + Jar = mustCookieJar() + defaultHTTPClient = &http.Client{ Jar: Jar, Timeout: 30 * time.Second, } @@ -47,8 +49,65 @@ var ( // CircuitBreaker trips after 5 consecutive API-level failures (HTTP 429 or 5xx) // and stays open for 60 seconds before probing recovery. CircuitBreaker = newCircuitBreaker(5, 60*time.Second) + circuitMu sync.Mutex + circuits = map[string]*circuitBreaker{} ) +type httpClientContextKey struct{} + +// WithHTTPClient returns a context that scopes HTTP execution to client. It is +// primarily useful for tests and avoids mutating process-global transport state. +func WithHTTPClient(ctx context.Context, client *http.Client) context.Context { + if client == nil { + return ctx + } + return context.WithValue(ctx, httpClientContextKey{}, client) +} + +// Do executes req with the client scoped to its context, or the immutable +// process default when no override is present. +func Do(req *http.Request) (*http.Response, error) { + client, _ := req.Context().Value(httpClientContextKey{}).(*http.Client) + if client == nil { + client = defaultHTTPClient + } + return client.Do(req) +} + +const maxDrainBytes = 64 << 10 + +func failureDomain(label string) string { + switch { + case label == "auth" || label == "userinfo" || label == "subscriptions": + return "identity" + case strings.Contains(label, "Player") || strings.Contains(label, "stream"): + return "stream" + default: + return "catalog" + } +} + +func breakerFor(label string) *circuitBreaker { + domain := failureDomain(label) + circuitMu.Lock() + defer circuitMu.Unlock() + if cb := circuits[domain]; cb != nil { + return cb + } + cb := newCircuitBreaker(5, 60*time.Second) + circuits[domain] = cb + return cb +} + +func drainAndClose(body io.ReadCloser) error { + if body == nil { + return nil + } + _, copyErr := io.Copy(io.Discard, io.LimitReader(body, maxDrainBytes)) + closeErr := body.Close() + return errors.Join(copyErr, closeErr) +} + func mustCookieJar() *cookiejar.Jar { jar, err := cookiejar.New(nil) if err != nil { @@ -73,6 +132,7 @@ func retryDo(ctx context.Context, label string, makeReq func() (*http.Request, e backoff := 500 * time.Millisecond for attempt := 0; ; attempt++ { + breaker := breakerFor(label) // 1. Rate limiter — block until a token is available. waited, err := RateLimiter.Wait(ctx) if err != nil { @@ -84,7 +144,7 @@ func retryDo(ctx context.Context, label string, makeReq func() (*http.Request, e } // 2. Circuit breaker — fail fast when the API is known-down. - cbState, allowed := CircuitBreaker.Allow() + cbState, allowed := breaker.Allow() if !allowed { LogCircuitRejected(label) return nil, fmt.Errorf("%w (label: %s)", ErrCircuitOpen, label) @@ -96,7 +156,7 @@ func retryDo(ctx context.Context, label string, makeReq func() (*http.Request, e return nil, err } start := time.Now() - resp, err := Client.Do(req) + resp, err := Do(req) duration := time.Since(start) if err != nil { @@ -109,7 +169,7 @@ func retryDo(ctx context.Context, label string, makeReq func() (*http.Request, e isAPIError := resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 if !isAPIError { // Success (2xx, 3xx, or client-error 4xx — server is healthy). - prev := CircuitBreaker.RecordSuccess() + prev := breaker.RecordSuccess() if prev != circuitClosed { LogCircuitStateChange("circuit_closed", label, prev.String(), circuitClosed.String()) } @@ -118,8 +178,8 @@ func retryDo(ctx context.Context, label string, makeReq func() (*http.Request, e } // API-level error: trip circuit breaker, log, then retry or give up. - resp.Body.Close() - newState := CircuitBreaker.RecordFailure() + _ = drainAndClose(resp.Body) + newState := breaker.RecordFailure() if newState == circuitOpen && cbState != circuitOpen { LogCircuitStateChange("circuit_opened", label, cbState.String(), newState.String()) } @@ -145,6 +205,25 @@ func retryDo(ctx context.Context, label string, makeReq func() (*http.Request, e } } +// doJSON centralizes retry execution, status validation, bounded body draining, +// and JSON decoding for API endpoints. +func doJSON[T any](ctx context.Context, label string, expectedStatus int, makeReq func() (*http.Request, error)) (T, error) { + var zero T + resp, err := retryDo(ctx, label, makeReq) + if err != nil { + return zero, err + } + defer func() { _ = drainAndClose(resp.Body) }() + if resp.StatusCode != expectedStatus { + return zero, fmt.Errorf("API %s failed: %s", label, resp.Status) + } + var value T + if err := json.NewDecoder(resp.Body).Decode(&value); err != nil { + return zero, fmt.Errorf("API %s decode: %w", label, err) + } + return value, nil +} + // qualityPattern pairs a URL substring with its quality info. type qualityPattern struct { Pattern string @@ -185,7 +264,7 @@ func Auth(ctx context.Context, email, pwd string) (string, error) { data.Set("username", email) data.Set("password", pwd) encoded := data.Encode() - do, err := retryDo(ctx, "auth", func() (*http.Request, error) { + obj, err := doJSON[model.Auth](ctx, "auth", http.StatusOK, func() (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodPost, AuthURL, strings.NewReader(encoded)) if err != nil { return nil, err @@ -197,20 +276,12 @@ func Auth(ctx context.Context, email, pwd string) (string, error) { if err != nil { return "", err } - defer do.Body.Close() - if do.StatusCode != http.StatusOK { - return "", fmt.Errorf("API authentication failed: %s", do.Status) - } - var obj model.Auth - if err = json.NewDecoder(do.Body).Decode(&obj); err != nil { - return "", err - } return obj.AccessToken, nil } // GetUserInfo retrieves user subscription ID. func GetUserInfo(ctx context.Context, token string) (string, error) { - do, err := retryDo(ctx, "userinfo", func() (*http.Request, error) { + obj, err := doJSON[model.UserInfo](ctx, "userinfo", http.StatusOK, func() (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, UserInfoURL, nil) if err != nil { return nil, err @@ -222,20 +293,12 @@ func GetUserInfo(ctx context.Context, token string) (string, error) { if err != nil { return "", err } - defer do.Body.Close() - if do.StatusCode != http.StatusOK { - return "", fmt.Errorf("API GetUserInfo failed: %s", do.Status) - } - var obj model.UserInfo - if err = json.NewDecoder(do.Body).Decode(&obj); err != nil { - return "", err - } return obj.Sub, nil } // GetSubInfo retrieves subscription information. func GetSubInfo(ctx context.Context, token string) (*model.SubInfo, error) { - do, err := retryDo(ctx, "subscriptions", func() (*http.Request, error) { + obj, err := doJSON[model.SubInfo](ctx, "subscriptions", http.StatusOK, func() (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, SubInfoURL, nil) if err != nil { return nil, err @@ -247,14 +310,6 @@ func GetSubInfo(ctx context.Context, token string) (*model.SubInfo, error) { if err != nil { return nil, err } - defer do.Body.Close() - if do.StatusCode != http.StatusOK { - return nil, fmt.Errorf("API GetSubInfo failed: %s", do.Status) - } - var obj model.SubInfo - if err = json.NewDecoder(do.Body).Decode(&obj); err != nil { - return nil, err - } return &obj, nil } @@ -292,7 +347,7 @@ func GetAlbumMeta(ctx context.Context, albumId string) (*model.AlbumMeta, error) query.Set("containerID", albumId) query.Set("vdisp", "1") rawQuery := query.Encode() - do, err := retryDo(ctx, "catalog.container", func() (*http.Request, error) { + obj, err := doJSON[model.AlbumMeta](ctx, "catalog.container", http.StatusOK, func() (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, StreamAPIBase+"api.aspx", nil) if err != nil { return nil, err @@ -304,14 +359,6 @@ func GetAlbumMeta(ctx context.Context, albumId string) (*model.AlbumMeta, error) if err != nil { return nil, err } - defer do.Body.Close() - if do.StatusCode != http.StatusOK { - return nil, fmt.Errorf("API GetAlbumMeta failed: %s", do.Status) - } - var obj model.AlbumMeta - if err = json.NewDecoder(do.Body).Decode(&obj); err != nil { - return nil, err - } return &obj, nil } @@ -337,7 +384,7 @@ func GetPlistMeta(ctx context.Context, plistId, email, legacyToken string, cat b query.Set("token", legacyToken) } rawQuery := query.Encode() - do, err := retryDo(ctx, label, func() (*http.Request, error) { + obj, err := doJSON[model.PlistMeta](ctx, label, http.StatusOK, func() (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, StreamAPIBase+apiPath, nil) if err != nil { return nil, err @@ -349,14 +396,6 @@ func GetPlistMeta(ctx context.Context, plistId, email, legacyToken string, cat b if err != nil { return nil, err } - defer do.Body.Close() - if do.StatusCode != http.StatusOK { - return nil, fmt.Errorf("API GetPlistMeta failed: %s", do.Status) - } - var obj model.PlistMeta - if err = json.NewDecoder(do.Body).Decode(&obj); err != nil { - return nil, err - } return &obj, nil } @@ -366,7 +405,7 @@ func GetLatestCatalog(ctx context.Context) (*model.LatestCatalogResp, error) { query.Set("method", "catalog.latest") query.Set("vdisp", "1") rawQuery := query.Encode() - do, err := retryDo(ctx, "catalog.latest", func() (*http.Request, error) { + obj, err := doJSON[model.LatestCatalogResp](ctx, "catalog.latest", http.StatusOK, func() (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, StreamAPIBase+"api.aspx", nil) if err != nil { return nil, err @@ -378,14 +417,6 @@ func GetLatestCatalog(ctx context.Context) (*model.LatestCatalogResp, error) { if err != nil { return nil, err } - defer do.Body.Close() - if do.StatusCode != http.StatusOK { - return nil, fmt.Errorf("API GetLatestCatalog failed: %s", do.Status) - } - var obj model.LatestCatalogResp - if err = json.NewDecoder(do.Body).Decode(&obj); err != nil { - return nil, err - } return &obj, nil } @@ -400,7 +431,7 @@ func getArtistMetaByAvailType(ctx context.Context, artistId string, availType in baseQuery.Set("vdisp", "1") for { currentOffset := offset // capture for closure - do, err := retryDo(ctx, "catalog.containersAll", func() (*http.Request, error) { + obj, err := doJSON[model.ArtistMeta](ctx, "catalog.containersAll", http.StatusOK, func() (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, StreamAPIBase+"api.aspx", nil) if err != nil { return nil, err @@ -414,16 +445,6 @@ func getArtistMetaByAvailType(ctx context.Context, artistId string, availType in if err != nil { return nil, err } - if do.StatusCode != http.StatusOK { - do.Body.Close() - return nil, fmt.Errorf("API getArtistMetaByAvailType failed: %s", do.Status) - } - var obj model.ArtistMeta - err = json.NewDecoder(do.Body).Decode(&obj) - do.Body.Close() - if err != nil { - return nil, err - } retLen := len(obj.Response.Containers) if retLen == 0 { break @@ -450,7 +471,7 @@ func GetArtistList(ctx context.Context) (*model.ArtistListResp, error) { query.Set("method", "catalog.artists") query.Set("vdisp", "1") rawQuery := query.Encode() - do, err := retryDo(ctx, "catalog.artists", func() (*http.Request, error) { + obj, err := doJSON[model.ArtistListResp](ctx, "catalog.artists", http.StatusOK, func() (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, StreamAPIBase+"api.aspx", nil) if err != nil { return nil, err @@ -462,14 +483,6 @@ func GetArtistList(ctx context.Context) (*model.ArtistListResp, error) { if err != nil { return nil, err } - defer do.Body.Close() - if do.StatusCode != http.StatusOK { - return nil, fmt.Errorf("API GetArtistList failed: %s", do.Status) - } - var obj model.ArtistListResp - if err = json.NewDecoder(do.Body).Decode(&obj); err != nil { - return nil, err - } return &obj, nil } @@ -482,7 +495,7 @@ func GetPurchasedManURL(ctx context.Context, skuID int, showID, userID, uguID st query.Set("nn_userID", userID) query.Set("app", "1") rawQuery := query.Encode() - do, err := retryDo(ctx, "vidPlayer", func() (*http.Request, error) { + obj, err := doJSON[model.PurchasedManResp](ctx, "vidPlayer", http.StatusOK, func() (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, StreamAPIBase+"bigriver/vidPlayer.aspx", nil) if err != nil { return nil, err @@ -494,14 +507,6 @@ func GetPurchasedManURL(ctx context.Context, skuID int, showID, userID, uguID st if err != nil { return "", err } - defer do.Body.Close() - if do.StatusCode != http.StatusOK { - return "", fmt.Errorf("API GetPurchasedManURL failed: %s", do.Status) - } - var obj model.PurchasedManResp - if err = json.NewDecoder(do.Body).Decode(&obj); err != nil { - return "", err - } return obj.FileURL, nil } @@ -527,7 +532,7 @@ func GetStreamMeta(ctx context.Context, trackId, skuId, format int, streamParams return q.Encode() } rawQuery := buildQuery() - do, err := retryDo(ctx, "subPlayer", func() (*http.Request, error) { + obj, err := doJSON[model.StreamMeta](ctx, "subPlayer", http.StatusOK, func() (*http.Request, error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, StreamAPIBase+"bigriver/subPlayer.aspx", nil) if err != nil { return nil, err @@ -539,14 +544,6 @@ func GetStreamMeta(ctx context.Context, trackId, skuId, format int, streamParams if err != nil { return "", err } - defer do.Body.Close() - if do.StatusCode != http.StatusOK { - return "", fmt.Errorf("API GetStreamMeta failed: %s", do.Status) - } - var obj model.StreamMeta - if err = json.NewDecoder(do.Body).Decode(&obj); err != nil { - return "", err - } return obj.StreamLink, nil } diff --git a/internal/cache/artist_cache.go b/internal/cache/artist_cache.go index 3688157..a45a6b7 100644 --- a/internal/cache/artist_cache.go +++ b/internal/cache/artist_cache.go @@ -5,6 +5,7 @@ import ( "fmt" "os" "path/filepath" + "sort" "time" "github.com/jmagar/nugs-cli/internal/model" @@ -12,6 +13,11 @@ import ( const artistListCacheFile = "artist-list.json" +const ( + defaultArtistCacheMaxAge = 30 * 24 * time.Hour + defaultArtistCacheMaxBytes = int64(256 << 20) +) + // ReadArtistListCache reads the cached catalog.artists response. // Returns the response, the time it was cached, and any error. // A missing cache file returns os.ErrNotExist. @@ -103,6 +109,68 @@ func WriteArtistMetaCache(artistID string, pages []*model.ArtistMeta) error { return fmt.Errorf("failed to marshal artist cache: %w", err) } - return atomicWriteFile(cachePath, data) + if err := atomicWriteFile(cachePath, data); err != nil { + return err + } + cacheDir := filepath.Dir(filepath.Dir(cachePath)) + if err := updateFullCatalogArtistLocked(cacheDir, artistID, pages); err != nil { + return err + } + return pruneArtistMetaCacheLocked(filepath.Dir(cachePath), defaultArtistCacheMaxAge, defaultArtistCacheMaxBytes, cachePath) + }) +} + +// PruneArtistMetaCache removes expired entries and then evicts least-recently +// modified entries until the cache is within maxBytes. +func PruneArtistMetaCache(maxAge time.Duration, maxBytes int64) error { + return WithCacheLock(func() error { + cacheDir, err := GetCacheDir() + if err != nil { + return err + } + return pruneArtistMetaCacheLocked(filepath.Join(cacheDir, "artists"), maxAge, maxBytes, "") }) } + +func pruneArtistMetaCacheLocked(dir string, maxAge time.Duration, maxBytes int64, preserve string) error { + entries, err := os.ReadDir(dir) + if os.IsNotExist(err) { + return nil + } + if err != nil { + return err + } + type candidate struct { + path string + info os.FileInfo + } + files := make([]candidate, 0, len(entries)) + var total int64 + now := time.Now() + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + continue + } + path := filepath.Join(dir, entry.Name()) + info, statErr := entry.Info() + if statErr != nil { + continue + } + if path != preserve && maxAge > 0 && now.Sub(info.ModTime()) > maxAge { + _ = os.Remove(path) + continue + } + files = append(files, candidate{path: path, info: info}) + total += info.Size() + } + sort.Slice(files, func(i, j int) bool { return files[i].info.ModTime().Before(files[j].info.ModTime()) }) + for _, file := range files { + if maxBytes <= 0 || total <= maxBytes || file.path == preserve { + continue + } + if err := os.Remove(file.path); err == nil { + total -= file.info.Size() + } + } + return nil +} diff --git a/internal/cache/atomic.go b/internal/cache/atomic.go new file mode 100644 index 0000000..74e9b9c --- /dev/null +++ b/internal/cache/atomic.go @@ -0,0 +1,48 @@ +package cache + +import ( + "fmt" + "os" + "path/filepath" +) + +// WriteFileAtomic durably writes data to a unique temporary file in the target +// directory and then atomically replaces targetPath. +func WriteFileAtomic(targetPath string, data []byte, mode os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(targetPath), 0755); err != nil { + return fmt.Errorf("failed to create target directory: %w", err) + } + tmp, err := os.CreateTemp(filepath.Dir(targetPath), ".nugs-*.tmp") + if err != nil { + return fmt.Errorf("failed to create temp file: %w", err) + } + tmpPath := tmp.Name() + ok := false + defer func() { + _ = tmp.Close() + if !ok { + _ = os.Remove(tmpPath) + } + }() + if err := tmp.Chmod(mode); err != nil { + return fmt.Errorf("failed to set temp file mode: %w", err) + } + if _, err := tmp.Write(data); err != nil { + return fmt.Errorf("failed to write temp file: %w", err) + } + if err := tmp.Sync(); err != nil { + return fmt.Errorf("failed to sync temp file: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("failed to close temp file: %w", err) + } + if err := os.Rename(tmpPath, targetPath); err != nil { + return fmt.Errorf("failed to rename temp file: %w", err) + } + ok = true + return nil +} + +func atomicWriteFile(targetPath string, data []byte) error { + return WriteFileAtomic(targetPath, data, 0644) +} diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 635c9c3..2ffa990 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -5,12 +5,37 @@ import ( "fmt" "os" "path/filepath" + "sort" "strings" "time" "github.com/jmagar/nugs-cli/internal/model" ) +const ( + catalogManifestFile = "catalog-current.json" + catalogGenerationDir = "catalog-generations" +) + +type catalogManifest struct { + Generation string `json:"generation"` +} + +func catalogArtifactPath(cacheDir, name string) (string, error) { + data, err := os.ReadFile(filepath.Join(cacheDir, catalogManifestFile)) + if err != nil { + if os.IsNotExist(err) { + return filepath.Join(cacheDir, name), nil + } + return "", fmt.Errorf("failed to read catalog manifest: %w", err) + } + var manifest catalogManifest + if err := json.Unmarshal(data, &manifest); err != nil || filepath.Base(manifest.Generation) != manifest.Generation || manifest.Generation == "" { + return "", fmt.Errorf("invalid catalog manifest") + } + return filepath.Join(cacheDir, catalogGenerationDir, manifest.Generation, name), nil +} + // GetCacheDir returns the cache directory path, creating it if needed. func GetCacheDir() (string, error) { homeDir, err := os.UserHomeDir() @@ -18,10 +43,13 @@ func GetCacheDir() (string, error) { return "", fmt.Errorf("failed to get home directory: %w", err) } cacheDir := filepath.Join(homeDir, ".cache", "nugs") - err = os.MkdirAll(cacheDir, 0755) + err = os.MkdirAll(cacheDir, 0700) if err != nil { return "", fmt.Errorf("failed to create cache directory: %w", err) } + if err := os.Chmod(cacheDir, 0700); err != nil { + return "", fmt.Errorf("failed to secure cache directory: %w", err) + } return cacheDir, nil } @@ -31,7 +59,10 @@ func ReadCacheMeta() (*model.CacheMeta, error) { if err != nil { return nil, err } - metaPath := filepath.Join(cacheDir, "catalog-meta.json") + metaPath, err := catalogArtifactPath(cacheDir, "catalog-meta.json") + if err != nil { + return nil, err + } data, err := os.ReadFile(metaPath) if err != nil { @@ -55,7 +86,10 @@ func ReadCatalogCache() (*model.LatestCatalogResp, error) { if err != nil { return nil, err } - catalogPath := filepath.Join(cacheDir, "catalog.json") + catalogPath, err := catalogArtifactPath(cacheDir, "catalog.json") + if err != nil { + return nil, err + } data, err := os.ReadFile(catalogPath) if err != nil { @@ -150,49 +184,71 @@ func prepareCatalogCacheData(catalog *model.LatestCatalogResp, updateDuration ti } func writePreparedCatalogCache(cacheDir string, prepared *preparedCatalogCacheData) error { - if err := atomicWriteFile(filepath.Join(cacheDir, "catalog.json"), prepared.catalogData); err != nil { + return writePreparedCatalogCacheWithFailpoint(cacheDir, prepared, -1) +} + +// writePreparedCatalogCacheWithFailpoint exists so tests can prove that a +// failure before the manifest swap never exposes a mixed generation. +func writePreparedCatalogCacheWithFailpoint(cacheDir string, prepared *preparedCatalogCacheData, failAfter int) error { + root := filepath.Join(cacheDir, catalogGenerationDir) + if err := os.MkdirAll(root, 0755); err != nil { return err } - if err := atomicWriteFile(filepath.Join(cacheDir, "catalog-meta.json"), prepared.metaData); err != nil { + generation, err := os.MkdirTemp(root, "generation-") + if err != nil { return err } - if err := atomicWriteFile(filepath.Join(cacheDir, "artists_index.json"), prepared.artistIdxData); err != nil { - return err + committed := false + defer func() { + if !committed { + _ = os.RemoveAll(generation) + } + }() + artifacts := []struct { + name string + data []byte + }{ + {"catalog.json", prepared.catalogData}, + {"catalog-meta.json", prepared.metaData}, + {"artists_index.json", prepared.artistIdxData}, + {"containers_index.json", prepared.containerData}, + } + for i, artifact := range artifacts { + if failAfter == i { + return fmt.Errorf("injected catalog generation failure after %d artifacts", i) + } + if err := WriteFileAtomic(filepath.Join(generation, artifact.name), artifact.data, 0644); err != nil { + return err + } } - return atomicWriteFile(filepath.Join(cacheDir, "containers_index.json"), prepared.containerData) -} - -// writeCatalogJSON writes catalog.json atomically using a temp file. -func writeCatalogJSON(cacheDir string, catalog *model.LatestCatalogResp) error { - catalogPath := filepath.Join(cacheDir, "catalog.json") - catalogData, err := json.Marshal(catalog) + manifestData, err := json.Marshal(catalogManifest{Generation: filepath.Base(generation)}) if err != nil { - return fmt.Errorf("failed to marshal catalog: %w", err) + return err } - return atomicWriteFile(catalogPath, catalogData) -} - -// writeCatalogMeta writes catalog-meta.json atomically. -func writeCatalogMeta(cacheDir string, catalog *model.LatestCatalogResp, updateDuration time.Duration, formatDurationFn func(time.Duration) string) error { - artistSet := make(map[int]bool) - for _, item := range catalog.Response.RecentItems { - artistSet[item.ArtistID] = true + if err := WriteFileAtomic(filepath.Join(cacheDir, catalogManifestFile), manifestData, 0644); err != nil { + return err } + committed = true + pruneCatalogGenerations(root, filepath.Base(generation)) + return nil +} - meta := model.CacheMeta{ - LastUpdated: time.Now(), - CacheVersion: "v1.0.0", - TotalShows: len(catalog.Response.RecentItems), - TotalArtists: len(artistSet), - APIMethod: "catalog.latest", - UpdateDuration: formatDurationFn(updateDuration), - } - metaPath := filepath.Join(cacheDir, "catalog-meta.json") - metaData, err := json.Marshal(meta) +func pruneCatalogGenerations(root, current string) { + entries, err := os.ReadDir(root) if err != nil { - return fmt.Errorf("failed to marshal metadata: %w", err) + return + } + sort.Slice(entries, func(i, j int) bool { return entries[i].Name() > entries[j].Name() }) + kept := 0 + for _, entry := range entries { + if !entry.IsDir() || entry.Name() == current || kept < 1 { + if entry.IsDir() && entry.Name() != current { + kept++ + } + continue + } + _ = os.RemoveAll(filepath.Join(root, entry.Name())) } - return atomicWriteFile(metaPath, metaData) } // ReadContainersIndex reads the containers index file. @@ -202,7 +258,10 @@ func ReadContainersIndex() (*model.ContainersIndex, error) { if err != nil { return nil, err } - indexPath := filepath.Join(cacheDir, "containers_index.json") + indexPath, err := catalogArtifactPath(cacheDir, "containers_index.json") + if err != nil { + return nil, err + } data, err := os.ReadFile(indexPath) if err != nil { if os.IsNotExist(err) { @@ -219,68 +278,3 @@ func ReadContainersIndex() (*model.ContainersIndex, error) { } return &idx, nil } - -// atomicWriteFile writes data to a temp file then atomically renames it. -func atomicWriteFile(targetPath string, data []byte) error { - tmpFile, err := os.CreateTemp(filepath.Dir(targetPath), "nugs_cache_*.tmp") - if err != nil { - return fmt.Errorf("failed to create temp file: %w", err) - } - tmpPath := tmpFile.Name() - - if _, err := tmpFile.Write(data); err != nil { - tmpFile.Close() - os.Remove(tmpPath) - return fmt.Errorf("failed to write temp file: %w", err) - } - if err := tmpFile.Close(); err != nil { - os.Remove(tmpPath) - return fmt.Errorf("failed to close temp file: %w", err) - } - - if err := os.Rename(tmpPath, targetPath); err != nil { - os.Remove(tmpPath) - return fmt.Errorf("failed to rename temp file: %w", err) - } - return nil -} - -// buildArtistIndex creates artist name -> ID lookup index (artists_index.json). -// Must be called within WithCacheLock. -func buildArtistIndex(cacheDir string, catalog *model.LatestCatalogResp) error { - index := make(map[string]int) - for _, item := range catalog.Response.RecentItems { - normalizedName := strings.ToLower(strings.TrimSpace(item.ArtistName)) - index[normalizedName] = item.ArtistID - } - - artistIndex := model.ArtistsIndex{Index: index} - indexPath := filepath.Join(cacheDir, "artists_index.json") - indexData, err := json.Marshal(artistIndex) - if err != nil { - return fmt.Errorf("failed to marshal artist index: %w", err) - } - return atomicWriteFile(indexPath, indexData) -} - -// buildContainerIndex creates containerID -> artist mapping (containers_index.json). -// Must be called within WithCacheLock. -func buildContainerIndex(cacheDir string, catalog *model.LatestCatalogResp) error { - containers := make(map[int]model.ContainerIndexEntry) - for _, item := range catalog.Response.RecentItems { - containers[item.ContainerID] = model.ContainerIndexEntry{ - ArtistID: item.ArtistID, - ArtistName: item.ArtistName, - ContainerInfo: item.ContainerInfo, - PerformanceDate: item.PerformanceDateStr, - } - } - - containerIndex := model.ContainersIndex{Containers: containers} - indexPath := filepath.Join(cacheDir, "containers_index.json") - indexData, err := json.Marshal(containerIndex) - if err != nil { - return fmt.Errorf("failed to marshal container index: %w", err) - } - return atomicWriteFile(indexPath, indexData) -} diff --git a/internal/cache/containers_index_test.go b/internal/cache/containers_index_test.go index 836c876..41fbff5 100644 --- a/internal/cache/containers_index_test.go +++ b/internal/cache/containers_index_test.go @@ -66,7 +66,11 @@ func TestReadContainersIndex_CorruptJSON(t *testing.T) { if err != nil { t.Fatalf("GetCacheDir() error = %v", err) } - if err := os.WriteFile(filepath.Join(cacheDir, "containers_index.json"), []byte("{corrupt"), 0644); err != nil { + indexPath, err := catalogArtifactPath(cacheDir, "containers_index.json") + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(indexPath, []byte("{corrupt"), 0644); err != nil { t.Fatalf("failed to write corrupt file: %v", err) } diff --git a/internal/cache/filelock_windows.go b/internal/cache/filelock_windows.go index ba33335..6d4374f 100644 --- a/internal/cache/filelock_windows.go +++ b/internal/cache/filelock_windows.go @@ -3,56 +3,50 @@ package cache import ( + "errors" "fmt" "os" "path/filepath" "time" + + "golang.org/x/sys/windows" ) // FileLock represents a file-based lock (Windows stub). type FileLock struct { lockFile *os.File - path string } -// staleLockThreshold is the age beyond which a lock file is considered stale. -// This handles the case where a process crashes without calling Release(). -const staleLockThreshold = 10 * time.Second - // AcquireLock acquires an exclusive lock on a lock file. -// On Windows this uses a simple file-existence check as a best-effort lock. -// Includes stale lock detection: if the lock file is older than staleLockThreshold -// after all retries are exhausted, it is removed and one final attempt is made. +// On Windows this uses LockFileEx so process death releases the kernel lock; +// the lock file itself is persistent and is never guessed to be stale. func AcquireLock(lockPath string, maxRetries int) (*FileLock, error) { lockDir := filepath.Dir(lockPath) if err := os.MkdirAll(lockDir, 0755); err != nil { return nil, fmt.Errorf("failed to create lock directory: %w", err) } + f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_RDWR, 0644) + if err != nil { + return nil, fmt.Errorf("failed to open lock file: %w", err) + } var lastErr error for i := 0; i <= maxRetries; i++ { - f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) + var overlapped windows.Overlapped + err := windows.LockFileEx(windows.Handle(f.Fd()), windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY, 0, 1, 0, &overlapped) if err == nil { - return &FileLock{lockFile: f, path: lockPath}, nil + return &FileLock{lockFile: f}, nil } lastErr = err + if !errors.Is(err, windows.ERROR_LOCK_VIOLATION) { + _ = f.Close() + return nil, fmt.Errorf("failed to lock cache file: %w", err) + } if i < maxRetries { time.Sleep(100 * time.Millisecond) } } - - // Check if the lock file is stale (older than threshold) - info, statErr := os.Stat(lockPath) - if statErr == nil && time.Since(info.ModTime()) > staleLockThreshold { - _ = os.Remove(lockPath) - // One final attempt after removing stale lock - f, err := os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0644) - if err == nil { - return &FileLock{lockFile: f, path: lockPath}, nil - } - lastErr = err - } - + _ = f.Close() return nil, fmt.Errorf("failed to acquire lock after %d retries: %w", maxRetries, lastErr) } @@ -61,10 +55,14 @@ func (fl *FileLock) Release() error { if fl.lockFile == nil { return nil } - err := fl.lockFile.Close() + var overlapped windows.Overlapped + unlockErr := windows.UnlockFileEx(windows.Handle(fl.lockFile.Fd()), 0, 1, 0, &overlapped) + closeErr := fl.lockFile.Close() fl.lockFile = nil - _ = os.Remove(fl.path) - return err + if unlockErr != nil { + return unlockErr + } + return closeErr } // WithCacheLock executes a function with the catalog cache lock acquired. diff --git a/internal/cache/full_catalog.go b/internal/cache/full_catalog.go new file mode 100644 index 0000000..6e6b1e4 --- /dev/null +++ b/internal/cache/full_catalog.go @@ -0,0 +1,80 @@ +package cache + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/jmagar/nugs-cli/internal/model" +) + +const fullCatalogIndexFile = "full-catalog-index.json" + +// FullCatalogIndex is the durable, API-independent source used by catalog +// coverage and gap analysis after the first complete crawl. +type FullCatalogIndex struct { + Version int `json:"version"` + UpdatedAt time.Time `json:"updatedAt"` + Artists map[string]FullArtist `json:"artists"` +} + +type FullArtist struct { + ArtistName string `json:"artistName"` + Pages []*model.ArtistMeta `json:"pages"` +} + +func ReadFullCatalogArtist(artistID string) ([]*model.ArtistMeta, string, error) { + cacheDir, err := GetCacheDir() + if err != nil { + return nil, "", err + } + data, err := os.ReadFile(filepath.Join(cacheDir, fullCatalogIndexFile)) + if err != nil { + return nil, "", err + } + var index FullCatalogIndex + if err := json.Unmarshal(data, &index); err != nil { + return nil, "", fmt.Errorf("failed to parse full catalog index: %w", err) + } + artist, ok := index.Artists[artistID] + if !ok { + return nil, "", os.ErrNotExist + } + return artist.Pages, artist.ArtistName, nil +} + +func updateFullCatalogArtistLocked(cacheDir, artistID string, pages []*model.ArtistMeta) error { + path := filepath.Join(cacheDir, fullCatalogIndexFile) + index := FullCatalogIndex{Version: 1, Artists: make(map[string]FullArtist)} + if data, err := os.ReadFile(path); err == nil { + if err := json.Unmarshal(data, &index); err != nil { + return fmt.Errorf("failed to parse full catalog index: %w", err) + } + } else if !errors.Is(err, os.ErrNotExist) { + return err + } + if index.Artists == nil { + index.Artists = make(map[string]FullArtist) + } + name := "" + for _, page := range pages { + if page != nil { + if value, ok := page.Response.ArtistName.(string); ok && value != "" { + name = value + } + if name == "" && len(page.Response.Containers) > 0 && page.Response.Containers[0] != nil { + name = page.Response.Containers[0].ArtistName + } + } + } + index.Artists[artistID] = FullArtist{ArtistName: name, Pages: pages} + index.UpdatedAt = time.Now() + data, err := json.Marshal(index) + if err != nil { + return err + } + return WriteFileAtomic(path, data, 0644) +} diff --git a/internal/cache/regression_test.go b/internal/cache/regression_test.go index baaa429..695d8f9 100644 --- a/internal/cache/regression_test.go +++ b/internal/cache/regression_test.go @@ -96,7 +96,10 @@ func TestWriteCatalogCache_ConcurrentWriters_Consistency(t *testing.T) { } for _, name := range files { - path := filepath.Join(cacheDir, name) + path, pathErr := catalogArtifactPath(cacheDir, name) + if pathErr != nil { + t.Fatalf("resolve %s: %v", name, pathErr) + } data, readErr := os.ReadFile(path) if readErr != nil { t.Fatalf("failed reading %s: %v", name, readErr) diff --git a/internal/cache/remediation_test.go b/internal/cache/remediation_test.go new file mode 100644 index 0000000..017a6fb --- /dev/null +++ b/internal/cache/remediation_test.go @@ -0,0 +1,71 @@ +package cache + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jmagar/nugs-cli/internal/model" +) + +func TestCatalogGenerationFailureKeepsPreviousSnapshot(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + oldCatalog := buildCatalog(1, 101, "old") + if err := WriteCatalogCache(oldCatalog, 0, time.Duration.String); err != nil { + t.Fatal(err) + } + cacheDir, _ := GetCacheDir() + prepared, err := prepareCatalogCacheData(buildCatalog(2, 202, "new"), 0, time.Duration.String) + if err != nil { + t.Fatal(err) + } + if err := WithCacheLock(func() error { + return writePreparedCatalogCacheWithFailpoint(cacheDir, prepared, 2) + }); err == nil { + t.Fatal("expected injected failure") + } + got, err := ReadCatalogCache() + if err != nil { + t.Fatal(err) + } + if id := got.Response.RecentItems[0].ContainerID; id != 1 { + t.Fatalf("mixed/new generation exposed: containerID=%d", id) + } + idx, err := ReadContainersIndex() + if err != nil { + t.Fatal(err) + } + if _, ok := idx.Containers[1]; !ok { + t.Fatal("previous index was not preserved") + } +} + +func TestArtistCacheEvictionPreservesDurableFullIndex(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + page := &model.ArtistMeta{} + page.Response.ArtistName = "Artist" + if err := WriteArtistMetaCache("1", []*model.ArtistMeta{page}); err != nil { + t.Fatal(err) + } + path, _ := GetArtistMetaCachePath("1") + old := time.Now().Add(-48 * time.Hour) + if err := os.Chtimes(path, old, old); err != nil { + t.Fatal(err) + } + if err := PruneArtistMetaCache(24*time.Hour, 1); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("artist cache was not evicted: %v", err) + } + pages, name, err := ReadFullCatalogArtist("1") + if err != nil || len(pages) != 1 || name != "Artist" { + t.Fatalf("durable full index lost: pages=%d name=%q err=%v", len(pages), name, err) + } + cacheDir, _ := GetCacheDir() + if _, err := os.Stat(filepath.Join(cacheDir, fullCatalogIndexFile)); err != nil { + t.Fatal(err) + } +} diff --git a/internal/catalog/catalog_update_test.go b/internal/catalog/catalog_update_test.go index 1d8ac35..2f84f95 100644 --- a/internal/catalog/catalog_update_test.go +++ b/internal/catalog/catalog_update_test.go @@ -320,8 +320,8 @@ func TestCatalogUpdate_OldIndexReadError(t *testing.T) { t.Fatalf("WriteCatalogCache() error = %v", err) } - // Now corrupt the containers_index so ReadContainersIndex fails. - indexPath := filepath.Join(cacheDir, "containers_index.json") + // Now corrupt the generation manifest so ReadContainersIndex fails. + indexPath := filepath.Join(cacheDir, "catalog-current.json") if err := os.WriteFile(indexPath, []byte("{bad json"), 0644); err != nil { t.Fatalf("failed to corrupt index: %v", err) } @@ -341,4 +341,3 @@ func TestCatalogUpdate_OldIndexReadError(t *testing.T) { t.Errorf("expected first-update fallback message, got:\n%s", stdout) } } - diff --git a/internal/catalog/handlers.go b/internal/catalog/handlers.go index 0be0af1..a8bb931 100644 --- a/internal/catalog/handlers.go +++ b/internal/catalog/handlers.go @@ -205,12 +205,12 @@ func CatalogUpdate(ctx context.Context, jsonLevel string, deps *Deps) error { } } output := map[string]any{ - "success": true, - "firstUpdate": isFirstUpdate, - "totalShows": len(catalog.Response.RecentItems), - "newShows": len(newShows), - "updateTime": deps.FormatDuration(updateDuration), - "cacheDir": cacheDir, + "success": true, + "firstUpdate": isFirstUpdate, + "totalShows": len(catalog.Response.RecentItems), + "newShows": len(newShows), + "updateTime": deps.FormatDuration(updateDuration), + "cacheDir": cacheDir, "newShowsList": newShowsData, } if err := PrintJSON(output); err != nil { @@ -856,6 +856,7 @@ func CatalogCoverage(ctx context.Context, artistIds []string, cfg *model.Config, var allStats []coverageStats var remoteScanErr error + var failures []string if len(artistIds) == 0 { if jsonLevel == "" { @@ -894,9 +895,14 @@ func CatalogCoverage(ctx context.Context, artistIds []string, cfg *model.Config, if len(discoveredArtistDirs) == 0 { if jsonLevel != "" { output := map[string]any{ - "artists": []map[string]any{}, - "total": 0, - "message": "No downloaded artists found", + "artists": []map[string]any{}, + "total": 0, + "message": "No downloaded artists found", + "partial": remoteScanErr != nil, + "remoteScanError": nil, + } + if remoteScanErr != nil { + output["remoteScanError"] = remoteScanErr.Error() } if err := PrintJSON(output); err != nil { return err @@ -904,6 +910,9 @@ func CatalogCoverage(ctx context.Context, artistIds []string, cfg *model.Config, } else { fmt.Println("No downloaded artists found in local output or remote storage") } + if remoteScanErr != nil { + return &CoveragePartialError{RemoteScanErr: remoteScanErr} + } return nil } @@ -1009,6 +1018,7 @@ func CatalogCoverage(ctx context.Context, artistIds []string, cfg *model.Config, for result := range results { if result.err != nil { + failures = append(failures, result.err.Error()) if jsonLevel == "" { ui.PrintWarning(fmt.Sprintf("Failed to get metadata for %v", result.err)) } @@ -1054,6 +1064,8 @@ func CatalogCoverage(ctx context.Context, artistIds []string, cfg *model.Config, "coverage": totalCoveragePct, }, "remoteScanError": nil, + "partial": remoteScanErr != nil || len(failures) > 0, + "failures": failures, } if remoteScanErr != nil { output["remoteScanError"] = remoteScanErr.Error() @@ -1097,9 +1109,28 @@ func CatalogCoverage(ctx context.Context, artistIds []string, cfg *model.Config, fmt.Println() } + if remoteScanErr != nil || len(failures) > 0 { + return &CoveragePartialError{RemoteScanErr: remoteScanErr, Failures: failures, Completed: len(allStats), Requested: len(artistIds)} + } return nil } +// CoveragePartialError makes best-effort coverage explicit to callers instead +// of silently turning failed artists or a failed remote scan into success. +type CoveragePartialError struct { + RemoteScanErr error + Failures []string + Completed int + Requested int +} + +func (e *CoveragePartialError) Error() string { + return fmt.Sprintf("coverage partial: completed=%d requested=%d artist_failures=%d remote_scan_failed=%t", + e.Completed, e.Requested, len(e.Failures), e.RemoteScanErr != nil) +} + +func (e *CoveragePartialError) Unwrap() error { return e.RemoteScanErr } + // CatalogList displays all shows for an artist with status indicators. func CatalogList(ctx context.Context, artistIds []string, cfg *model.Config, jsonLevel string, mediaFilter model.MediaType, deps *Deps) error { for i, artistId := range artistIds { diff --git a/internal/catalog/handlers_coverage_test.go b/internal/catalog/handlers_coverage_test.go index a0da599..d0a287b 100644 --- a/internal/catalog/handlers_coverage_test.go +++ b/internal/catalog/handlers_coverage_test.go @@ -3,6 +3,7 @@ package catalog import ( "context" "encoding/json" + "errors" "strings" "testing" @@ -22,15 +23,18 @@ func TestCatalogCoverage_JSONMode_RemoteScanFailurePreservesJSONOutput(t *testin stdout := testutil.CaptureStdout(t, func() { err := CatalogCoverage(context.Background(), nil, cfg, "normal", model.MediaTypeAudio, &Deps{}) - if err != nil { - t.Fatalf("CatalogCoverage() error = %v", err) + var partial *CoveragePartialError + if !errors.As(err, &partial) || partial.RemoteScanErr == nil { + t.Fatalf("CatalogCoverage() error = %v, want structured partial error", err) } }) var payload struct { - Artists []map[string]any `json:"artists"` - Total int `json:"total"` - Message string `json:"message"` + Artists []map[string]any `json:"artists"` + Total int `json:"total"` + Message string `json:"message"` + Partial bool `json:"partial"` + RemoteScanError string `json:"remoteScanError"` } if err := json.Unmarshal([]byte(stdout), &payload); err != nil { @@ -39,6 +43,9 @@ func TestCatalogCoverage_JSONMode_RemoteScanFailurePreservesJSONOutput(t *testin if payload.Message != "No downloaded artists found" { t.Fatalf("message = %q, want %q", payload.Message, "No downloaded artists found") } + if !payload.Partial || payload.RemoteScanError == "" { + t.Fatalf("partial failure fields missing: %+v", payload) + } if strings.Contains(stdout, "Warning:") { t.Fatalf("expected JSON-only stdout, found warning text in output=%q", stdout) } diff --git a/internal/catalog/helpers.go b/internal/catalog/helpers.go index e861b94..f70161f 100644 --- a/internal/catalog/helpers.go +++ b/internal/catalog/helpers.go @@ -15,6 +15,7 @@ import ( "github.com/jmagar/nugs-cli/internal/helpers" "github.com/jmagar/nugs-cli/internal/model" + "github.com/jmagar/nugs-cli/internal/rclone" "github.com/jmagar/nugs-cli/internal/ui" ) @@ -61,7 +62,12 @@ func ListAllRemoteArtistFolders(ctx context.Context, cfg *model.Config) (map[str remoteDest := cfg.RcloneRemote + ":" + cfg.RclonePath cmd := exec.CommandContext(ctx, "rclone", "lsf", remoteDest, "--dirs-only") - output, err := cmd.Output() + var output []byte + err := rclone.WithProcessSlot(ctx, func() error { + var runErr error + output, runErr = cmd.Output() + return runErr + }) if err != nil { var exitErr *exec.ExitError if errors.As(err, &exitErr) && exitErr.ExitCode() == 3 { @@ -241,7 +247,12 @@ func CountRemoteShowsPerArtist(ctx context.Context, cfg *model.Config) (map[stri for _, dest := range remoteDests { cmd := exec.CommandContext(ctx, "rclone", "lsf", dest, "--dirs-only", "--recursive") - output, err := cmd.Output() + var output []byte + err := rclone.WithProcessSlot(ctx, func() error { + var runErr error + output, runErr = cmd.Output() + return runErr + }) if err != nil { var exitErr *exec.ExitError if errors.As(err, &exitErr) && exitErr.ExitCode() == 3 { diff --git a/internal/catalog/media_filter.go b/internal/catalog/media_filter.go index 39966c3..e9e42fe 100644 --- a/internal/catalog/media_filter.go +++ b/internal/catalog/media_filter.go @@ -8,6 +8,7 @@ import ( "strings" "sync" + "github.com/jmagar/nugs-cli/internal/cache" "github.com/jmagar/nugs-cli/internal/helpers" "github.com/jmagar/nugs-cli/internal/model" "github.com/jmagar/nugs-cli/internal/ui" @@ -219,9 +220,17 @@ func AnalyzeArtistCatalogMediaAware(ctx context.Context, artistID string, cfg *m return nil, fmt.Errorf("GetArtistMetaCached callback not configured") } - artistMetas, cacheUsed, cacheStaleUse, err := deps.GetArtistMetaCached(ctx, artistID, ArtistMetaCacheTTL) - if err != nil { - return nil, fmt.Errorf("failed to get artist metadata: %w", err) + // The complete local index is authoritative for warm catalog analysis. It + // deliberately has no TTL: catalog update/crawl refreshes it, while normal + // coverage and gap commands have a zero-network request budget. + artistMetas, _, localErr := cache.ReadFullCatalogArtist(artistID) + cacheUsed, cacheStaleUse := localErr == nil, false + if localErr != nil { + var err error + artistMetas, cacheUsed, cacheStaleUse, err = deps.GetArtistMetaCached(ctx, artistID, ArtistMetaCacheTTL) + if err != nil { + return nil, fmt.Errorf("failed to get artist metadata: %w", err) + } } if len(artistMetas) == 0 { diff --git a/internal/catalog/request_budget_test.go b/internal/catalog/request_budget_test.go new file mode 100644 index 0000000..01eb94a --- /dev/null +++ b/internal/catalog/request_budget_test.go @@ -0,0 +1,48 @@ +package catalog + +import ( + "context" + "testing" + "time" + + "github.com/jmagar/nugs-cli/internal/cache" + "github.com/jmagar/nugs-cli/internal/model" +) + +func TestAnalyzeArtistWarmIndexUsesZeroAPIRequests(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + page := &model.ArtistMeta{} + page.Response.ArtistName = "Artist" + page.Response.Containers = []*model.AlbArtResp{{ + ArtistName: "Artist", + ContainerID: 10, + ContainerInfo: "Show", + PerformanceDate: "2025-01-01", + AvailabilityTypeStr: model.AvailableAvailabilityType, + ProductFormatList: []*model.ProductFormatList{{}}, + }} + requests := 0 + deps := &Deps{ + GetArtistMetaCached: func(_ context.Context, _ string, _ time.Duration) ([]*model.ArtistMeta, bool, bool, error) { + requests++ + if err := cache.WriteArtistMetaCache("1", []*model.ArtistMeta{page}); err != nil { + return nil, false, false, err + } + return []*model.ArtistMeta{page}, false, false, nil + }, + GetShowMediaType: func(*model.AlbArtResp) model.MediaType { return model.MediaTypeAudio }, + } + cfg := &model.Config{OutPath: t.TempDir(), DefaultOutputs: "audio"} + if _, err := AnalyzeArtistCatalogMediaAware(context.Background(), "1", cfg, "", model.MediaTypeAudio, deps); err != nil { + t.Fatal(err) + } + if requests != 1 { + t.Fatalf("cold request count = %d, want 1", requests) + } + if _, err := AnalyzeArtistCatalogMediaAware(context.Background(), "1", cfg, "", model.MediaTypeAudio, deps); err != nil { + t.Fatal(err) + } + if requests != 1 { + t.Fatalf("warm analysis made %d requests, want zero additional", requests) + } +} diff --git a/internal/catalog/watch.go b/internal/catalog/watch.go index eadcd6b..af3b95f 100644 --- a/internal/catalog/watch.go +++ b/internal/catalog/watch.go @@ -13,6 +13,23 @@ import ( "github.com/jmagar/nugs-cli/internal/ui" ) +// WatchOutcomeError reports a completed but degraded watch run. Returning it +// makes systemd mark the one-shot unit failed while preserving all counts in +// the journal and notifications. +type WatchOutcomeError struct { + Downloaded int + Failed int + ArtistErrors []string + CatalogUpdate error +} + +func (e *WatchOutcomeError) Error() string { + return fmt.Sprintf("watch outcome: downloaded=%d failed=%d artist_errors=%d catalog_degraded=%t", + e.Downloaded, e.Failed, len(e.ArtistErrors), e.CatalogUpdate != nil) +} + +func (e *WatchOutcomeError) Unwrap() error { return e.CatalogUpdate } + // WatchAdd adds an artist ID to the watch list in config. // The ID must be a valid integer. Duplicates are silently ignored. func WatchAdd(cfg *model.Config, artistID string) error { @@ -137,7 +154,9 @@ func WatchCheck(ctx context.Context, cfg *model.Config, streamParams *model.Stre // Silently update the catalog first. Pass jsonLevel so CatalogUpdate suppresses // its human-readable table output when the caller expects JSON on stdout. + var catalogUpdateErr error if err := CatalogUpdate(ctx, jsonLevel, deps); err != nil { + catalogUpdateErr = err // Non-fatal: proceed with potentially stale catalog. if jsonLevel == "" { ui.PrintWarning(fmt.Sprintf("Catalog update failed (continuing with cached data): %v", err)) @@ -147,6 +166,7 @@ func WatchCheck(ctx context.Context, cfg *model.Config, streamParams *model.Stre totalDownloaded := 0 totalFailed := 0 var artistErrors []string + nameByID := buildArtistNameMap() for _, artistID := range cfg.WatchedArtists { select { @@ -156,7 +176,7 @@ func WatchCheck(ctx context.Context, cfg *model.Config, streamParams *model.Stre } if jsonLevel == "" { - name := resolveArtistName(artistID) + name := resolveArtistNameFromMap(artistID, nameByID) if name != "" { ui.PrintHeader(fmt.Sprintf("Checking %s (%s)", artistID, name)) } else { @@ -184,6 +204,13 @@ func WatchCheck(ctx context.Context, cfg *model.Config, streamParams *model.Stre } sendWatchSummary(ctx, deps.Notify, totalDownloaded, totalFailed, artistErrors) + if catalogUpdateErr != nil || totalFailed > 0 || len(artistErrors) > 0 { + outcome := &WatchOutcomeError{Downloaded: totalDownloaded, Failed: totalFailed, ArtistErrors: artistErrors, CatalogUpdate: catalogUpdateErr} + if catalogUpdateErr != nil { + return fmt.Errorf("%w: catalog update: %v", outcome, catalogUpdateErr) + } + return outcome + } return nil } @@ -228,7 +255,7 @@ func sendWatchSummary(ctx context.Context, notify func(ctx context.Context, titl parts = append(parts, errs...) _ = notify(ctx, "Nugs Watch Error", strings.Join(parts, "\n"), 7) - // Nothing to report: all artists up-to-date, no errors. Stay silent. + // Nothing to report: all artists up-to-date, no errors. Stay silent. } } @@ -251,11 +278,14 @@ func buildArtistNameMap() map[int]string { // resolveArtistName looks up an artist name from the containers index cache. // Returns an empty string if the cache is unavailable or the artist is not found. func resolveArtistName(artistID string) string { + return resolveArtistNameFromMap(artistID, buildArtistNameMap()) +} + +func resolveArtistNameFromMap(artistID string, names map[int]string) string { idInt, err := strconv.Atoi(artistID) if err != nil { return "" } - names := buildArtistNameMap() return names[idInt] } diff --git a/internal/catalog/watch_systemd.go b/internal/catalog/watch_systemd.go index 6f0c339..11bf1ad 100644 --- a/internal/catalog/watch_systemd.go +++ b/internal/catalog/watch_systemd.go @@ -11,6 +11,7 @@ import ( "text/template" "time" + "github.com/jmagar/nugs-cli/internal/cache" "github.com/jmagar/nugs-cli/internal/model" "github.com/jmagar/nugs-cli/internal/ui" ) @@ -23,7 +24,8 @@ Wants=network-online.target [Service] Type=oneshot Environment=NUGS_DETACHED=1 -ExecStart={{.BinaryPath}} watch check +Environment="PATH={{.Path}}" +ExecStart="{{.BinaryPath}}" watch check StandardOutput=journal StandardError=journal @@ -54,26 +56,54 @@ func WatchEnable(cfg *model.Config) error { if err != nil { return fmt.Errorf("failed to resolve binary path: %w", err) } + binPath, err = filepath.Abs(binPath) + if err != nil { + return fmt.Errorf("failed to make binary path absolute: %w", err) + } + if info, statErr := os.Stat(binPath); statErr != nil || info.IsDir() || info.Mode()&0111 == 0 { + return fmt.Errorf("nugs binary is not an executable regular file: %s", binPath) + } + ffmpegName := cfg.FfmpegNameStr + if ffmpegName == "" { + ffmpegName = "ffmpeg" + } + if _, err := exec.LookPath(ffmpegName); err != nil { + return fmt.Errorf("watch requires ffmpeg in the service PATH: %w", err) + } + if cfg.RcloneEnabled { + if _, err := exec.LookPath("rclone"); err != nil { + return fmt.Errorf("watch has rclone enabled but rclone is not in the service PATH: %w", err) + } + } unitDir, err := systemdUserDir() if err != nil { return err } + servicePath := filepath.Join(unitDir, "nugs-watch.service") + timerPath := filepath.Join(unitDir, "nugs-watch.timer") + backup := snapshotUnitFiles(servicePath, timerPath) + rollback := func() { + _ = systemctlUser("disable", "--now", "nugs-watch.timer") + restoreUnitFiles(backup) + _ = systemctlUser("daemon-reload") + } if err := writeWatchUnitFiles(cfg, unitDir, binPath); err != nil { + restoreUnitFiles(backup) return err } if err := systemctlUser("daemon-reload"); err != nil { + rollback() return fmt.Errorf("daemon-reload failed: %w", err) } if err := systemctlUser("enable", "--now", "nugs-watch.timer"); err != nil { + rollback() return fmt.Errorf("failed to enable timer: %w", err) } systemdInterval, _ := toSystemdDuration(watchIntervalOrDefault(cfg)) - servicePath := filepath.Join(unitDir, "nugs-watch.service") - timerPath := filepath.Join(unitDir, "nugs-watch.timer") ui.PrintSuccess("Nugs watch timer enabled") ui.PrintKeyValue("Binary", binPath, ui.ColorCyan) @@ -97,7 +127,14 @@ func writeWatchUnitFiles(cfg *model.Config, unitDir, binPath string) error { } servicePath := filepath.Join(unitDir, "nugs-watch.service") - if err := writeUnitFile(servicePath, serviceTemplate, struct{ BinaryPath string }{BinaryPath: binPath}); err != nil { + if !filepath.IsAbs(binPath) { + return fmt.Errorf("binary path must be absolute: %s", binPath) + } + templateData := struct { + BinaryPath string + Path string + }{BinaryPath: binPath, Path: os.Getenv("PATH")} + if err := writeUnitFile(servicePath, serviceTemplate, templateData); err != nil { return fmt.Errorf("failed to write service unit: %w", err) } @@ -144,8 +181,11 @@ func toSystemdDuration(s string) (string, error) { // WatchDisable stops and disables the nugs-watch systemd timer and removes unit files. func WatchDisable() error { - // Stop and disable — ignore errors if units aren't loaded (they may not exist yet). - _ = systemctlUser("disable", "--now", "nugs-watch.timer") + // A failed disable leaves the unit files intact so the operation can be + // retried without losing the installed configuration. + if err := systemctlUser("disable", "--now", "nugs-watch.timer"); err != nil { + return fmt.Errorf("failed to disable timer: %w", err) + } _ = systemctlUser("stop", "nugs-watch.service") unitDir, err := systemdUserDir() @@ -155,10 +195,12 @@ func WatchDisable() error { servicePath := filepath.Join(unitDir, "nugs-watch.service") timerPath := filepath.Join(unitDir, "nugs-watch.timer") + backup := snapshotUnitFiles(servicePath, timerPath) removed := 0 for _, p := range []string{servicePath, timerPath} { if err := os.Remove(p); err != nil && !os.IsNotExist(err) { + restoreUnitFiles(backup) return fmt.Errorf("failed to remove %s: %w", p, err) } else if err == nil { removed++ @@ -166,7 +208,9 @@ func WatchDisable() error { } if err := systemctlUser("daemon-reload"); err != nil { - ui.PrintWarning(fmt.Sprintf("daemon-reload failed after removing unit files: %v", err)) + restoreUnitFiles(backup) + _ = systemctlUser("daemon-reload") + return fmt.Errorf("daemon-reload failed after removing unit files: %w", err) } if removed == 0 { @@ -177,6 +221,31 @@ func WatchDisable() error { return nil } +type unitSnapshot map[string][]byte + +func snapshotUnitFiles(paths ...string) unitSnapshot { + snapshot := make(unitSnapshot, len(paths)) + for _, path := range paths { + data, err := os.ReadFile(path) + if err == nil { + snapshot[path] = data + } else { + snapshot[path] = nil + } + } + return snapshot +} + +func restoreUnitFiles(snapshot unitSnapshot) { + for path, data := range snapshot { + if data == nil { + _ = os.Remove(path) + continue + } + _ = cache.WriteFileAtomic(path, data, 0644) + } +} + // systemdUserDir returns ~/.config/systemd/user, creating it if needed. func systemdUserDir() (string, error) { homeDir, err := os.UserHomeDir() @@ -203,13 +272,8 @@ func writeUnitFile(path, tmplStr string, data any) error { return fmt.Errorf("failed to render template for %s: %w", path, err) } - tmpPath := path + ".tmp" - if err := os.WriteFile(tmpPath, buf.Bytes(), 0644); err != nil { - return fmt.Errorf("failed to write temp unit file %s: %w", tmpPath, err) - } - if err := os.Rename(tmpPath, path); err != nil { - _ = os.Remove(tmpPath) - return fmt.Errorf("failed to rename unit file to %s: %w", path, err) + if err := cache.WriteFileAtomic(path, buf.Bytes(), 0644); err != nil { + return fmt.Errorf("failed to atomically write unit file %s: %w", path, err) } return nil } diff --git a/internal/catalog/watch_test.go b/internal/catalog/watch_test.go index f27ac38..04a2329 100644 --- a/internal/catalog/watch_test.go +++ b/internal/catalog/watch_test.go @@ -246,8 +246,9 @@ func TestWatchCheck_CatalogUpdateFailureIsNonFatal(t *testing.T) { OutPath: t.TempDir(), } err := WatchCheck(context.Background(), cfg, &model.StreamParams{}, "", model.MediaTypeUnknown, deps) - if err != nil { - t.Fatalf("WatchCheck() returned error on catalog failure: %v", err) + var outcome *WatchOutcomeError + if !errors.As(err, &outcome) || !errors.Is(err, fetchErr) { + t.Fatalf("WatchCheck() error = %v, want degraded outcome wrapping fetch error", err) } if !artistMetaCalled { t.Error("GetArtistMetaCached should be called even when catalog update fails") @@ -281,8 +282,9 @@ func TestWatchCheck_PerArtistFailureContinues(t *testing.T) { OutPath: t.TempDir(), } err := WatchCheck(context.Background(), cfg, &model.StreamParams{}, "", model.MediaTypeUnknown, deps) - if err != nil { - t.Fatalf("WatchCheck() returned error when per-artist failure should be non-fatal: %v", err) + var outcome *WatchOutcomeError + if !errors.As(err, &outcome) || len(outcome.ArtistErrors) != 2 { + t.Fatalf("WatchCheck() error = %v, want outcome with two artist errors", err) } // Both artists must have been attempted. @@ -348,16 +350,16 @@ func TestToSystemdDuration(t *testing.T) { }{ {"1h", "1h", false}, {"6h", "6h", false}, - {"30m", "30min", false}, // critical: Go 'm' → systemd 'min', not months - {"90m", "1h30min", false}, // mixed hours + minutes + {"30m", "30min", false}, // critical: Go 'm' → systemd 'min', not months + {"90m", "1h30min", false}, // mixed hours + minutes {"45m", "45min", false}, {"3600s", "1h", false}, {"1h30m", "1h30min", false}, {"30s", "30s", false}, - {"0s", "", true}, // zero duration rejected - {"-1h", "", true}, // negative rejected - {"2x", "", true}, // invalid format - {"every day", "", true}, // completely invalid + {"0s", "", true}, // zero duration rejected + {"-1h", "", true}, // negative rejected + {"2x", "", true}, // invalid format + {"every day", "", true}, // completely invalid } for _, tc := range tests { @@ -407,10 +409,13 @@ func TestWatchEnableUnitContent(t *testing.T) { if err != nil { t.Fatalf("failed to read service file: %v", err) } - wantExecStart := "ExecStart=" + fakeBin + " watch check" + wantExecStart := "ExecStart=\"" + fakeBin + "\" watch check" if !strings.Contains(string(serviceContent), wantExecStart) { t.Errorf("service file missing %q:\n%s", wantExecStart, serviceContent) } + if !strings.Contains(string(serviceContent), `Environment="PATH=`) { + t.Errorf("service file missing explicit PATH environment:\n%s", serviceContent) + } // Verify timer file uses the configured interval (6h stays "6h" after conversion). timerPath := filepath.Join(unitDir, "nugs-watch.timer") @@ -443,8 +448,8 @@ func TestSendWatchSummary(t *testing.T) { wantCalled: false, }, { - name: "downloads only", - downloaded: 3, failed: 0, errs: nil, + name: "downloads only", + downloaded: 3, failed: 0, errs: nil, wantCalled: true, wantTitle: "Nugs Watch", wantPriority: 5, @@ -452,32 +457,32 @@ func TestSendWatchSummary(t *testing.T) { wantAbsent: []string{"failed", "error"}, }, { - name: "downloads with some failures", - downloaded: 2, failed: 1, errs: nil, + name: "downloads with some failures", + downloaded: 2, failed: 1, errs: nil, wantCalled: true, wantTitle: "Nugs Watch", wantPriority: 5, wantContains: []string{"2 new show(s)", "1 failed"}, }, { - name: "downloads with artist errors", - downloaded: 4, failed: 0, errs: []string{"1125: timeout"}, + name: "downloads with artist errors", + downloaded: 4, failed: 0, errs: []string{"1125: timeout"}, wantCalled: true, wantTitle: "Nugs Watch", wantPriority: 5, wantContains: []string{"4 new show(s)", "1 artist error(s)"}, }, { - name: "download failures only — no artist errors", - downloaded: 0, failed: 2, errs: nil, + name: "download failures only — no artist errors", + downloaded: 0, failed: 2, errs: nil, wantCalled: true, wantTitle: "Nugs Watch Error", wantPriority: 7, wantContains: []string{"2 download failure(s)"}, }, { - name: "artist errors only — no download failures", - downloaded: 0, failed: 0, errs: []string{"461: network error"}, + name: "artist errors only — no download failures", + downloaded: 0, failed: 0, errs: []string{"461: network error"}, wantCalled: true, wantTitle: "Nugs Watch Error", wantPriority: 7, @@ -486,8 +491,8 @@ func TestSendWatchSummary(t *testing.T) { wantAbsent: []string{"0 download failure(s)"}, }, { - name: "artist errors and download failures combined", - downloaded: 0, failed: 1, errs: []string{"461: timeout"}, + name: "artist errors and download failures combined", + downloaded: 0, failed: 1, errs: []string{"461: timeout"}, wantCalled: true, wantTitle: "Nugs Watch Error", wantPriority: 7, @@ -590,8 +595,8 @@ func TestSendArtistUpdate(t *testing.T) { // single-artist runs (to avoid double-notifying alongside the final summary). func TestWatchCheck_PerArtistNotification(t *testing.T) { tests := []struct { - name string - watchedArtists []string + name string + watchedArtists []string // downloadsByArtist controls how many downloads each artist "completes". // Uses album dep call count per artistID to drive successCount. artistDownloads map[string]int @@ -651,8 +656,9 @@ func TestWatchCheck_PerArtistNotification(t *testing.T) { OutPath: t.TempDir(), } - if err := WatchCheck(context.Background(), cfg, &model.StreamParams{}, "", model.MediaTypeUnknown, deps); err != nil { - t.Fatalf("WatchCheck() error = %v", err) + var outcome *WatchOutcomeError + if err := WatchCheck(context.Background(), cfg, &model.StreamParams{}, "", model.MediaTypeUnknown, deps); !errors.As(err, &outcome) { + t.Fatalf("WatchCheck() error = %v, want empty-catalog outcome", err) } // With emptyArtistMeta (no shows), no artist has downloads — no per-artist @@ -699,3 +705,27 @@ func TestWatchEnableUnitContent_MinutesConversion(t *testing.T) { t.Errorf("timer file should use '30min' for minutes:\n%s", content) } } + +func TestRestoreUnitFilesRollsBackCreatedAndReplacedFiles(t *testing.T) { + dir := t.TempDir() + existing := filepath.Join(dir, "existing.service") + created := filepath.Join(dir, "created.timer") + if err := os.WriteFile(existing, []byte("old"), 0644); err != nil { + t.Fatal(err) + } + snapshot := snapshotUnitFiles(existing, created) + if err := os.WriteFile(existing, []byte("new"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(created, []byte("new"), 0644); err != nil { + t.Fatal(err) + } + restoreUnitFiles(snapshot) + data, err := os.ReadFile(existing) + if err != nil || string(data) != "old" { + t.Fatalf("existing unit not restored: %q, %v", data, err) + } + if _, err := os.Stat(created); !os.IsNotExist(err) { + t.Fatalf("new unit not removed: %v", err) + } +} diff --git a/internal/config/atomic_test.go b/internal/config/atomic_test.go new file mode 100644 index 0000000..df79ac1 --- /dev/null +++ b/internal/config/atomic_test.go @@ -0,0 +1,52 @@ +package config + +import ( + "bytes" + "os" + "path/filepath" + "sync" + "testing" +) + +func TestAtomicWriteFileConcurrentWriters(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "config.json") + values := [][]byte{[]byte(`{"writer":1}`), []byte(`{"writer":2}`), []byte(`{"writer":3}`)} + var wg sync.WaitGroup + for _, value := range values { + value := value + wg.Add(1) + go func() { + defer wg.Done() + if err := atomicWriteFile(target, value, 0600); err != nil { + t.Errorf("atomicWriteFile: %v", err) + } + }() + } + wg.Wait() + got, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + valid := false + for _, value := range values { + valid = valid || bytes.Equal(got, value) + } + if !valid { + t.Fatalf("partial or unexpected result: %q", got) + } + info, err := os.Stat(target) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0600 { + t.Fatalf("mode = %o, want 600", info.Mode().Perm()) + } + matches, err := filepath.Glob(filepath.Join(dir, ".config.json.*.tmp")) + if err != nil { + t.Fatal(err) + } + if len(matches) != 0 { + t.Fatalf("orphan temp files: %v", matches) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index 0c1e041..a5facd7 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -241,14 +241,43 @@ func promptRclone(scanner *bufio.Scanner) (rcloneSettings, error) { // atomicWriteFile writes data to a temp file then atomically renames it to the target path. func atomicWriteFile(targetPath string, data []byte, perm os.FileMode) error { - tmpPath := targetPath + ".tmp" - if err := os.WriteFile(tmpPath, data, perm); err != nil { - return fmt.Errorf("failed to write temp file %s: %w", tmpPath, err) + dir := filepath.Dir(targetPath) + tmp, err := os.CreateTemp(dir, "."+filepath.Base(targetPath)+".*.tmp") + if err != nil { + return fmt.Errorf("create temp file: %w", err) + } + tmpPath := tmp.Name() + committed := false + defer func() { + _ = tmp.Close() + if !committed { + _ = os.Remove(tmpPath) + } + }() + if err := tmp.Chmod(perm); err != nil { + return fmt.Errorf("chmod temp file: %w", err) + } + if _, err := tmp.Write(data); err != nil { + return fmt.Errorf("write temp file: %w", err) + } + if err := tmp.Sync(); err != nil { + return fmt.Errorf("sync temp file: %w", err) + } + if err := tmp.Close(); err != nil { + return fmt.Errorf("close temp file: %w", err) } if err := os.Rename(tmpPath, targetPath); err != nil { - // Clean up temp file on rename failure - os.Remove(tmpPath) - return fmt.Errorf("failed to rename %s to %s: %w", tmpPath, targetPath, err) + return fmt.Errorf("rename temp file: %w", err) + } + committed = true + dirFile, err := os.Open(dir) + if err != nil { + return fmt.Errorf("open config directory for sync: %w", err) + } + syncErr := dirFile.Sync() + closeErr := dirFile.Close() + if err := errors.Join(syncErr, closeErr); err != nil { + return fmt.Errorf("sync config directory: %w", err) } return nil } @@ -420,45 +449,23 @@ func ParseCfg() (*model.Config, error) { func ResolveFfmpegBinary(cfg *model.Config) (string, error) { preferred := strings.TrimSpace(cfg.FfmpegNameStr) - // Respect explicit non-default binary names/paths from config. - if preferred != "" && preferred != "./ffmpeg" && preferred != "ffmpeg" { + // Explicit filesystem paths must be absolute. This prevents a repository or + // download directory from silently supplying the executable. + if preferred != "" && preferred != "ffmpeg" { + if strings.ContainsAny(preferred, `/\\`) && !filepath.IsAbs(preferred) { + return "", fmt.Errorf("configured ffmpeg path must be absolute: %s", preferred) + } if resolved, err := exec.LookPath(preferred); err == nil { return resolved, nil } - if info, err := os.Stat(preferred); err == nil && !info.IsDir() { - return preferred, nil - } return "", fmt.Errorf("configured ffmpeg binary not found: %s", preferred) } - if cfg.UseFfmpegEnvVar || preferred == "ffmpeg" { - if resolved, err := exec.LookPath("ffmpeg"); err == nil { - return resolved, nil - } - return "", errors.New("ffmpeg not found in PATH (install ffmpeg or set ffmpegNameStr to an absolute/local binary path)") - } - - // Backward-compatible default: local ./ffmpeg first. - candidates := []string{"./ffmpeg"} - if exePath, err := os.Executable(); err == nil { - exeDir := filepath.Dir(exePath) - exeLocal := filepath.Join(exeDir, "ffmpeg") - if exeLocal != "./ffmpeg" { - candidates = append(candidates, exeLocal) - } - } - for _, candidate := range candidates { - if info, err := os.Stat(candidate); err == nil && !info.IsDir() { - return candidate, nil - } - } - - // Fallback: use system ffmpeg if available. + // Defaults and the legacy environment flag both resolve only through PATH. if resolved, err := exec.LookPath("ffmpeg"); err == nil { return resolved, nil } - - return "", errors.New("ffmpeg binary not found (checked ./ffmpeg and PATH)") + return "", errors.New("ffmpeg not found in PATH (install ffmpeg or configure an absolute ffmpegNameStr path)") } var showCountFilterRegex = regexp.MustCompile(`^(>=|<=|>|<|=)\d+$`) @@ -590,9 +597,35 @@ func ReadConfig() (*model.Config, error) { return &obj, nil } +// Args owns the CLI argument contract. Keeping it in config avoids making the +// foundation model package depend behaviorally on package-main initialization. +type Args struct { + Urls []string `arg:"positional"` + Format int `arg:"-f" default:"-1" help:"Track format (1-5)"` + VideoFormat int `arg:"-F" default:"-1" help:"Video format (1-5)"` + OutPath string `arg:"-o" help:"Download directory"` + ForceVideo bool `arg:"--force-video" help:"Deprecated: use a video media modifier"` + SkipVideos bool `arg:"--skip-videos" help:"Deprecated: use an audio media modifier"` + SkipChapters bool `arg:"--skip-chapters" help:"Skip video chapters"` +} + +func (Args) Description() string { + return `Download music and videos from Nugs.net. + +Commands: + nugs grab [audio|video|both] + nugs latest|full [audio|video|both] + nugs list [artists|] + nugs catalog update|cache|stats|latest|list|gaps|coverage|config + nugs watch add|remove|list|check|enable|disable + nugs status|cancel|version + +Use README.md or docs/COMMANDS.md for complete examples.` +} + // ParseArgs parses CLI arguments using go-arg. -func ParseArgs() *model.Args { - var args model.Args +func ParseArgs() *Args { + var args Args arg.MustParse(&args) return &args } diff --git a/internal/download/audio.go b/internal/download/audio.go index 399d31e..f241c4a 100644 --- a/internal/download/audio.go +++ b/internal/download/audio.go @@ -29,6 +29,9 @@ import ( var bitrateRegex = regexp.MustCompile(`[\w]+(?:_(\d+)k_v\d+)`) +// ErrEmptyHLSMaster identifies a syntactically valid master playlist without variants. +var ErrEmptyHLSMaster = errors.New("HLS master playlist has no variants") + var trackFallback = map[int]int{ 1: 2, 2: 5, @@ -85,11 +88,14 @@ func DownloadTrack(ctx context.Context, trackPath, _url string, onProgress func( req.Header.Add("Referer", api.PlayerURL) req.Header.Add("User-Agent", api.UserAgent) req.Header.Add("Range", "bytes=0-") - do, err := api.Client.Do(req) + do, err := api.Do(req) if err != nil { return err } - defer do.Body.Close() + defer func() { + _, _ = io.Copy(io.Discard, io.LimitReader(do.Body, 64<<10)) + _ = do.Body.Close() + }() if do.StatusCode != http.StatusOK && do.StatusCode != http.StatusPartialContent { return errors.New(do.Status) } @@ -97,7 +103,6 @@ func DownloadTrack(ctx context.Context, trackPath, _url string, onProgress func( if err != nil { return err } - defer f.Close() totalBytes := do.ContentLength totalStr := model.UnknownSizeLabelLower if totalBytes > 0 { @@ -113,10 +118,11 @@ func DownloadTrack(ctx context.Context, trackPath, _url string, onProgress func( deps: deps, } _, err = io.Copy(f, io.TeeReader(do.Body, counter)) + closeErr := f.Close() if printNewline { fmt.Println("") } - if err != nil { + if err = errors.Join(err, closeErr); err != nil { os.Remove(trackPath) return err } @@ -144,7 +150,16 @@ func ExtractBitrate(manUrl string) string { // ParseHlsMaster parses an HLS master playlist and selects the best variant. func ParseHlsMaster(qual *model.Quality) error { - req, err := api.Client.Get(qual.URL) + return ParseHlsMasterContext(context.Background(), qual) +} + +// ParseHlsMasterContext fetches and parses a master playlist with cancellation. +func ParseHlsMasterContext(ctx context.Context, qual *model.Quality) error { + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, qual.URL, nil) + if err != nil { + return err + } + req, err := api.Do(httpReq) if err != nil { return err } @@ -162,7 +177,7 @@ func ParseHlsMaster(qual *model.Quality) error { return errors.New("expected HLS master playlist but got media playlist") } if len(master.Variants) == 0 { - return errors.New("HLS master playlist has no variants") + return ErrEmptyHLSMaster } sort.Slice(master.Variants, func(x, y int) bool { return master.Variants[x].Bandwidth > master.Variants[y].Bandwidth @@ -183,7 +198,16 @@ func ParseHlsMaster(qual *model.Quality) error { // GetKey retrieves an AES encryption key from the given URL. func GetKey(keyUrl string) ([]byte, error) { - req, err := api.Client.Get(keyUrl) + return GetKeyContext(context.Background(), keyUrl) +} + +// GetKeyContext retrieves an AES key with cancellation. +func GetKeyContext(ctx context.Context, keyUrl string) ([]byte, error) { + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, keyUrl, nil) + if err != nil { + return nil, err + } + req, err := api.Do(httpReq) if err != nil { return nil, err } @@ -234,14 +258,18 @@ func DecryptTrack(tempPath string, key, iv []byte) ([]byte, error) { // TsToAac converts a decrypted TS stream to AAC using ffmpeg. func TsToAac(decData []byte, outPath, ffmpegNameStr string) error { + return TsToAacContext(context.Background(), decData, outPath, ffmpegNameStr) +} + +// TsToAacContext converts a decrypted TS stream and cancels ffmpeg with ctx. +func TsToAacContext(ctx context.Context, decData []byte, outPath, ffmpegNameStr string) error { var errBuffer bytes.Buffer - cmd := exec.Command(ffmpegNameStr, "-i", "pipe:", "-c:a", "copy", outPath) + cmd := exec.CommandContext(ctx, ffmpegNameStr, "-i", "pipe:", "-c:a", "copy", outPath) cmd.Stdin = bytes.NewReader(decData) cmd.Stderr = &errBuffer err := cmd.Run() if err != nil { - errString := fmt.Sprintf("%s\n%s", err, errBuffer.String()) - return errors.New(errString) + return fmt.Errorf("ffmpeg AAC conversion: %w: %s", err, errBuffer.String()) } return nil } @@ -322,18 +350,27 @@ func DecryptTrackToFile(tempPath, decryptedPath string, key, iv []byte) error { // TsFileToAac converts a decrypted TS file to AAC using ffmpeg. func TsFileToAac(decPath, outPath, ffmpegNameStr string) error { + return TsFileToAacContext(context.Background(), decPath, outPath, ffmpegNameStr) +} + +// TsFileToAacContext converts a file and cancels ffmpeg with ctx. +func TsFileToAacContext(ctx context.Context, decPath, outPath, ffmpegNameStr string) error { var errBuffer bytes.Buffer - cmd := exec.Command(ffmpegNameStr, "-i", decPath, "-c:a", "copy", outPath) + cmd := exec.CommandContext(ctx, ffmpegNameStr, "-i", decPath, "-c:a", "copy", outPath) cmd.Stderr = &errBuffer if err := cmd.Run(); err != nil { - return fmt.Errorf("%s\n%s", err, errBuffer.String()) + return fmt.Errorf("ffmpeg AAC conversion: %w: %s", err, errBuffer.String()) } return nil } // HlsOnly downloads an HLS-only track (encrypted), decrypts it, and converts to AAC. func HlsOnly(ctx context.Context, trackPath, manUrl, ffmpegNameStr string, onProgress func(downloaded, total, speed int64), printNewline bool, deps *Deps) error { - req, err := api.Client.Get(manUrl) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, manUrl, nil) + if err != nil { + return err + } + req, err := api.Do(httpReq) if err != nil { return err } @@ -365,7 +402,7 @@ func HlsOnly(ctx context.Context, trackPath, manUrl, ffmpegNameStr string, onPro tsUrl := manBase + media.Segments[0].URI + q key := media.Key - keyBytes, err := GetKey(manBase + key.URI) + keyBytes, err := GetKeyContext(ctx, manBase+key.URI) if err != nil { return err } @@ -398,7 +435,7 @@ func HlsOnly(ctx context.Context, trackPath, manUrl, ffmpegNameStr string, onPro if err := DecryptTrackToFile(tempPath, decryptedPath, keyBytes, iv); err != nil { return err } - return TsFileToAac(decryptedPath, trackPath, ffmpegNameStr) + return TsFileToAacContext(ctx, decryptedPath, trackPath, ffmpegNameStr) } // CheckIfHlsOnly checks if all qualities are HLS-only streams. @@ -423,7 +460,7 @@ func selectTrackQuality(ctx context.Context, track *model.Track, streamParams *m isHlsOnly := strings.Contains(track.TrackURL, ".m3u8?") if isHlsOnly { ui.PrintInfo("HLS-only track. Only AAC is available, tags currently unsupported") - if err := ParseHlsMaster(qual); err != nil { + if err := ParseHlsMasterContext(ctx, qual); err != nil { return nil, true, wantFmt, err } } @@ -443,7 +480,7 @@ func selectTrackQuality(ctx context.Context, track *model.Track, streamParams *m } quality := api.QueryQuality(streamUrl) if quality == nil { - ui.PrintError(fmt.Sprintf("The API returned an unsupported format: %s", streamUrl)) + ui.PrintError(fmt.Sprintf("The API returned an unsupported format: %s", helpers.RedactURL(streamUrl))) continue } quals = append(quals, quality) @@ -457,7 +494,7 @@ func selectTrackQuality(ctx context.Context, track *model.Track, streamParams *m if isHlsOnly { ui.PrintInfo("HLS-only track. Only AAC is available, tags currently unsupported") chosenQual := quals[0] - if err := ParseHlsMaster(chosenQual); err != nil { + if err := ParseHlsMasterContext(ctx, chosenQual); err != nil { return nil, true, wantFmt, err } return chosenQual, true, wantFmt, nil @@ -668,9 +705,13 @@ func PreCalculateShowSize(ctx context.Context, tracks []model.Track, streamParam return } - streamURL, err := api.GetStreamMeta(ctx, track.TrackID, 0, 1, streamParams) - if err != nil || streamURL == "" { - continue + streamURL := track.TrackURL + if streamURL == "" { + var err error + streamURL, err = api.GetStreamMeta(ctx, track.TrackID, 0, 1, streamParams) + if err != nil || streamURL == "" { + continue + } } reqCtx, reqCancel := context.WithTimeout(ctx, model.PreCalcPerRequestTimeout) @@ -679,7 +720,7 @@ func PreCalculateShowSize(ctx context.Context, tracks []model.Track, streamParam reqCancel() continue } - resp, err := api.Client.Do(req) + resp, err := api.Do(req) reqCancel() if err != nil { continue @@ -689,7 +730,8 @@ func PreCalculateShowSize(ctx context.Context, tracks []model.Track, streamParam totalSize += resp.ContentLength mu.Unlock() } - resp.Body.Close() + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) + _ = resp.Body.Close() } } }() @@ -846,6 +888,7 @@ func prepareAlbumProgressBox(meta *model.AlbArtResp, cfg *model.Config, batchSta func downloadAlbumAudio(ctx context.Context, tracks []model.Track, albumPath, artistFolder string, cfg *model.Config, streamParams *model.StreamParams, progressBox *model.ProgressBoxState, downloadVideo bool, deps *Deps) error { trackTotal := len(tracks) + var failures []error for trackNum, track := range tracks { if deps.WaitIfPausedOrCancelled != nil { if err := deps.WaitIfPausedOrCancelled(); err != nil { @@ -863,35 +906,41 @@ func downloadAlbumAudio(ctx context.Context, tracks []model.Track, albumPath, ar progressBox.ErrorTracks++ progressBox.Mu.Unlock() } - helpers.HandleErr("Track failed.", err, false) + helpers.ReportErr("Track failed.", err) + failures = append(failures, fmt.Errorf("track %d (%s): %w", trackNum, track.SongTitle, err)) } } if progressBox != nil { - progressBox.IsComplete = true + progressBox.Mu.Lock() + progressBox.IsComplete = len(failures) == 0 progressBox.CompletionTime = time.Now() progressBox.TotalDuration = time.Since(progressBox.StartTime) + progressBox.Mu.Unlock() } if cfg.RcloneEnabled { if err := deps.UploadPath(ctx, albumPath, artistFolder, cfg, progressBox, false); err != nil { - helpers.HandleErr("Upload failed.", err, false) + helpers.ReportErr("Upload failed.", err) + failures = append(failures, fmt.Errorf("upload album: %w", err)) } } if !downloadVideo && deps.RenderCompletionSummary != nil { deps.RenderCompletionSummary(progressBox) fmt.Println("") } - return nil + return errors.Join(failures...) } -func downloadAlbumVideo(ctx context.Context, albumID string, cfg *model.Config, streamParams *model.StreamParams, meta *model.AlbArtResp, progressBox *model.ProgressBoxState, hadAudio bool, deps *Deps) { +func downloadAlbumVideo(ctx context.Context, albumID string, cfg *model.Config, streamParams *model.StreamParams, meta *model.AlbArtResp, progressBox *model.ProgressBoxState, hadAudio bool, deps *Deps) error { if hadAudio { fmt.Println("") ui.PrintInfo("Downloading video...") } err := Video(ctx, albumID, "", cfg, streamParams, meta, false, progressBox, deps) if err != nil { - helpers.HandleErr("Video download failed.", err, false) + helpers.ReportErr("Video download failed.", err) + return fmt.Errorf("video download: %w", err) } + return nil } // Album downloads an album or show from Nugs.net using the provided albumID. @@ -926,15 +975,18 @@ func Album(ctx context.Context, albumID string, cfg *model.Config, streamParams } }() } + var failures []error if downloadAudio && trackTotal > 0 { if err := downloadAlbumAudio(ctx, tracks, albumPath, artistFolder, cfg, streamParams, progressBox, downloadVideo, deps); err != nil { - return err + failures = append(failures, err) } } if downloadVideo && skuID != 0 { - downloadAlbumVideo(ctx, albumID, cfg, streamParams, meta, progressBox, downloadAudio && trackTotal > 0, deps) + if err := downloadAlbumVideo(ctx, albumID, cfg, streamParams, meta, progressBox, downloadAudio && trackTotal > 0, deps); err != nil { + failures = append(failures, err) + } } - return nil + return errors.Join(failures...) } // GetAlbumTotal counts the total number of containers across all artist metadata pages. diff --git a/internal/download/batch.go b/internal/download/batch.go index 1ebe728..0039cca 100644 --- a/internal/download/batch.go +++ b/internal/download/batch.go @@ -16,17 +16,18 @@ import ( // Artist downloads all albums for an artist. func Artist(ctx context.Context, artistId string, cfg *model.Config, streamParams *model.StreamParams, deps *Deps) error { - // Use availType=2 to get complete catalog (both audio and video shows) - meta, err := api.GetArtistMetaWithAvailType(ctx, artistId, 2) + // AVAILABLE is the complete downloadable catalog. PREORDER is upcoming + // content and must never drive a download batch. + meta, err := api.GetArtistMetaWithAvailType(ctx, artistId, model.AvailableCatalogView) if err != nil { ui.PrintError("Failed to get artist metadata") return err } if len(meta) == 0 { - return errors.New("The API didn't return any artist metadata.") + return errors.New("the API did not return any artist metadata") } if len(meta[0].Response.Containers) == 0 { - return errors.New("The API didn't return any containers for this artist.") + return errors.New("the API did not return any containers for this artist") } fmt.Println(meta[0].Response.Containers[0].ArtistName) @@ -158,6 +159,7 @@ func Playlist(ctx context.Context, plistId, legacyToken string, cfg *model.Confi err = deps.UploadPath(ctx, plistPath, "", cfg, progressBox, false) if err != nil { ui.PrintError(fmt.Sprintf("Playlist upload failed (%s): %v", plistName, err)) + return fmt.Errorf("playlist upload failed (%s): %w", plistName, err) } } @@ -167,6 +169,7 @@ func Playlist(ctx context.Context, plistId, legacyToken string, cfg *model.Confi // processPlaylistTracks iterates over playlist tracks, downloading each with pause/cancel support. func processPlaylistTracks(ctx context.Context, tracks []*model.Track, plistPath string, cfg *model.Config, streamParams *model.StreamParams, progressBox *model.ProgressBoxState, deps *Deps) error { trackTotal := len(tracks) + var failures []error for trackNum, track := range tracks { if deps.WaitIfPausedOrCancelled != nil { if err := deps.WaitIfPausedOrCancelled(); err != nil { @@ -180,9 +183,10 @@ func processPlaylistTracks(ctx context.Context, tracks []*model.Track, plistPath } ui.PrintError(fmt.Sprintf("Track %d/%d failed (%s): %v", trackNum, trackTotal, track.SongTitle, err)) + failures = append(failures, fmt.Errorf("track %d/%d (%s): %w", trackNum, trackTotal, track.SongTitle, err)) } } - return nil + return errors.Join(failures...) } // PaidLstream downloads a paid livestream video. diff --git a/internal/download/deps.go b/internal/download/deps.go index 21b3130..6d4a3b2 100644 --- a/internal/download/deps.go +++ b/internal/download/deps.go @@ -5,10 +5,13 @@ package download import ( "context" + "errors" "github.com/jmagar/nugs-cli/internal/model" ) +var errStorageDependencyMissing = errors.New("rclone is enabled but no storage provider is configured") + // Deps holds callbacks to root-package functions that the download package needs. // These cannot be imported directly because they live in the root (main) package. // The root package wires these up before calling any download functions. @@ -60,6 +63,9 @@ func (d *Deps) UploadPath(ctx context.Context, localPath, artistFolder string, c IsVideo: isVideo, }, model.StorageHooks{}) } + if cfg != nil && cfg.RcloneEnabled { + return errStorageDependencyMissing + } return nil } @@ -71,5 +77,8 @@ func (d *Deps) CheckRemotePathExists(ctx context.Context, remotePath string, cfg if d != nil && d.Storage != nil { return d.Storage.PathExists(ctx, cfg, remotePath, isVideo) } + if cfg != nil && cfg.RcloneEnabled { + return false, errStorageDependencyMissing + } return false, nil } diff --git a/internal/download/deps_test.go b/internal/download/deps_test.go index f653ec9..0af7a38 100644 --- a/internal/download/deps_test.go +++ b/internal/download/deps_test.go @@ -2,6 +2,7 @@ package download import ( "context" + "errors" "testing" "github.com/jmagar/nugs-cli/internal/model" @@ -16,6 +17,17 @@ type fakeStorageProvider struct { pathExists bool } +func TestDepsRejectsEnabledRcloneWithoutStorageDependency(t *testing.T) { + cfg := &model.Config{RcloneEnabled: true} + + if err := (*Deps)(nil).UploadPath(context.Background(), "/tmp/album", "artist", cfg, nil, false); !errors.Is(err, errStorageDependencyMissing) { + t.Fatalf("UploadPath error = %v, want %v", err, errStorageDependencyMissing) + } + if _, err := (*Deps)(nil).CheckRemotePathExists(context.Background(), "artist/show", cfg, false); !errors.Is(err, errStorageDependencyMissing) { + t.Fatalf("CheckRemotePathExists error = %v, want %v", err, errStorageDependencyMissing) + } +} + func (f *fakeStorageProvider) Upload(_ context.Context, _ *model.Config, req model.UploadRequest, _ model.StorageHooks) error { f.uploadCalls++ f.lastUpload = req diff --git a/internal/download/precalculate_test.go b/internal/download/precalculate_test.go new file mode 100644 index 0000000..14e4de1 --- /dev/null +++ b/internal/download/precalculate_test.go @@ -0,0 +1,37 @@ +package download + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/jmagar/nugs-cli/internal/api" + "github.com/jmagar/nugs-cli/internal/model" +) + +func TestPreCalculateShowSizeReusesTrackURL(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.Method != http.MethodHead { + t.Errorf("method = %s, want HEAD", r.Method) + } + w.Header().Set("Content-Length", "123") + })) + defer server.Close() + + ctx := api.WithHTTPClient(context.Background(), server.Client()) + + size, err := PreCalculateShowSize(ctx, []model.Track{{TrackID: 42, TrackURL: server.URL + "/track"}}, &model.StreamParams{}, &model.Config{}) + if err != nil { + t.Fatal(err) + } + if size != 123 { + t.Fatalf("size = %d, want 123", size) + } + if requests.Load() != 1 { + t.Fatalf("requests = %d, want one metadata URL HEAD", requests.Load()) + } +} diff --git a/internal/download/regression_test.go b/internal/download/regression_test.go index 05ea5e8..9faa42e 100644 --- a/internal/download/regression_test.go +++ b/internal/download/regression_test.go @@ -29,9 +29,6 @@ func httpResponse(status int, body string) *http.Response { } } -// NOTE: Tests in this file mutate the global api.Client and restore it via t.Cleanup. -// They MUST NOT use t.Parallel() — concurrent subtests would race on the shared client. - func TestProcessTrack_QualityFallback_NoHang(t *testing.T) { tests := []struct { name string @@ -61,10 +58,8 @@ func TestProcessTrack_QualityFallback_NoHang(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - oldClient := api.Client - t.Cleanup(func() { api.Client = oldClient }) - - api.Client = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + t.Parallel() + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { switch { case strings.Contains(req.URL.Path, "/bigriver/subPlayer.aspx"): return httpResponse(http.StatusOK, fmt.Sprintf(`{"streamLink":%q}`, tc.streamURL)), nil @@ -74,6 +69,7 @@ func TestProcessTrack_QualityFallback_NoHang(t *testing.T) { return nil, fmt.Errorf("unexpected request: %s", req.URL.String()) } })} + ctx := api.WithHTTPClient(context.Background(), client) dir := t.TempDir() cfg := &model.Config{Format: tc.wantFmt} @@ -82,7 +78,7 @@ func TestProcessTrack_QualityFallback_NoHang(t *testing.T) { errCh := make(chan error, 1) go func() { - errCh <- ProcessTrack(context.Background(), dir, 1, 1, cfg, track, streamParams, nil, &Deps{}) + errCh <- ProcessTrack(ctx, dir, 1, 1, cfg, track, streamParams, nil, &Deps{}) }() select { @@ -130,15 +126,14 @@ func TestHlsOnly_ValidationErrors_NoPanic(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - oldClient := api.Client - t.Cleanup(func() { api.Client = oldClient }) - - api.Client = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + t.Parallel() + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { if strings.HasSuffix(req.URL.Path, "/manifest.m3u8") { return httpResponse(http.StatusOK, tc.playlist), nil } return nil, fmt.Errorf("unexpected request: %s", req.URL.String()) })} + ctx := api.WithHTTPClient(context.Background(), client) panicked := false var gotErr error @@ -148,7 +143,7 @@ func TestHlsOnly_ValidationErrors_NoPanic(t *testing.T) { panicked = true } }() - gotErr = HlsOnly(context.Background(), filepath.Join(t.TempDir(), "out.m4a"), "https://stream.test/manifest.m3u8", "ffmpeg", nil, false, &Deps{}) + gotErr = HlsOnly(ctx, filepath.Join(t.TempDir(), "out.m4a"), "https://stream.test/manifest.m3u8", "ffmpeg", nil, false, &Deps{}) }() if panicked { @@ -165,10 +160,7 @@ func TestHlsOnly_ValidationErrors_NoPanic(t *testing.T) { } func TestChooseVariant_InvalidResolution_NoPanic(t *testing.T) { - oldClient := api.Client - t.Cleanup(func() { api.Client = oldClient }) - - api.Client = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { if strings.HasSuffix(req.URL.Path, "/master.m3u8") { playlist := "#EXTM3U\n" + "#EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=bad-resolution\n" + @@ -177,6 +169,7 @@ func TestChooseVariant_InvalidResolution_NoPanic(t *testing.T) { } return nil, fmt.Errorf("unexpected request: %s", req.URL.String()) })} + ctx := api.WithHTTPClient(context.Background(), client) panicked := false var gotErr error @@ -186,7 +179,7 @@ func TestChooseVariant_InvalidResolution_NoPanic(t *testing.T) { panicked = true } }() - _, _, gotErr = ChooseVariant("https://video.test/master.m3u8", "2160") + _, _, gotErr = ChooseVariant(ctx, "https://video.test/master.m3u8", "2160") }() if panicked { diff --git a/internal/download/video.go b/internal/download/video.go index ff45ff2..decb088 100644 --- a/internal/download/video.go +++ b/internal/download/video.go @@ -126,10 +126,20 @@ func FormatRes(res string) string { } // ChooseVariant selects the best video variant from a manifest URL. -func ChooseVariant(manifestUrl, wantRes string) (*m3u8.Variant, string, error) { +func ChooseVariant(ctx context.Context, manifestUrl, wantRes string) (*m3u8.Variant, string, error) { + return ChooseVariantContext(ctx, manifestUrl, wantRes) +} + +// ChooseVariantContext selects a variant with cancellation. +func ChooseVariantContext(ctx context.Context, manifestUrl, wantRes string) (*m3u8.Variant, string, error) { origWantRes := wantRes var wantVariant *m3u8.Variant - req, err := api.Client.Get(manifestUrl) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, manifestUrl, nil) + if err != nil { + return nil, "", err + } + httpReq = httpReq.WithContext(ctx) + req, err := api.Do(httpReq) if err != nil { return nil, "", err } @@ -145,6 +155,9 @@ func ChooseVariant(manifestUrl, wantRes string) (*m3u8.Variant, string, error) { if !ok { return nil, "", errors.New("expected HLS master playlist but got media playlist") } + if len(master.Variants) == 0 { + return nil, "", ErrEmptyHLSMaster + } sort.Slice(master.Variants, func(x, y int) bool { return master.Variants[x].Bandwidth > master.Variants[y].Bandwidth }) @@ -211,8 +224,17 @@ func GetManifestBase(manifestUrl string) (string, string, error) { // GetSegUrls retrieves segment URLs from an HLS media playlist. func GetSegUrls(manifestUrl, query string) ([]string, error) { + return GetSegUrlsContext(context.Background(), manifestUrl, query) +} + +// GetSegUrlsContext retrieves media segments with cancellation. +func GetSegUrlsContext(ctx context.Context, manifestUrl, query string) ([]string, error) { var segUrls []string - req, err := api.Client.Get(manifestUrl) + httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, manifestUrl, nil) + if err != nil { + return nil, err + } + req, err := api.Do(httpReq) if err != nil { return nil, err } @@ -256,7 +278,7 @@ func DownloadVideoFile(ctx context.Context, videoPath, _url string, onProgress f return err } req.Header.Add("Range", fmt.Sprintf("bytes=%d-", startByte)) - do, err := api.Client.Do(req) + do, err := api.Do(req) if err != nil { return err } @@ -295,6 +317,7 @@ func DownloadVideoFile(ctx context.Context, videoPath, _url string, onProgress f // Use a simple writer that doesn't need deps (video has its own progress callbacks) _, err = io.Copy(f, io.TeeReader(do.Body, &simpleWriteCounter{wc: counter})) + err = errors.Join(err, f.Close()) if onProgress == nil { fmt.Println("") } @@ -334,7 +357,7 @@ func DownloadLstream(ctx context.Context, videoPath, baseUrl string, segUrls []s if err != nil { return err } - do, err := api.Client.Do(req) + do, err := api.Do(req) if err != nil { return err } @@ -351,7 +374,7 @@ func DownloadLstream(ctx context.Context, videoPath, baseUrl string, segUrls []s if onProgress == nil { fmt.Println("") } - return nil + return f.Close() } // ExtractDuration extracts the duration string from ffmpeg output. @@ -379,9 +402,14 @@ func ParseDuration(dur string) (int, error) { // GetDuration gets the duration of a TS file using ffmpeg. func GetDuration(tsPath, ffmpegNameStr string) (int, error) { + return GetDurationContext(context.Background(), tsPath, ffmpegNameStr) +} + +// GetDurationContext reads duration while allowing ffmpeg cancellation. +func GetDurationContext(ctx context.Context, tsPath, ffmpegNameStr string) (int, error) { var errBuffer bytes.Buffer args := []string{"-hide_banner", "-i", tsPath} - cmd := exec.Command(ffmpegNameStr, args...) + cmd := exec.CommandContext(ctx, ffmpegNameStr, args...) cmd.Stderr = &errBuffer // Return code's always 1 as we're not providing any output files. err := cmd.Run() @@ -400,7 +428,7 @@ func GetDuration(tsPath, ffmpegNameStr string) (int, error) { } dur := ExtractDuration(errStr) if dur == "" { - return 0, errors.New("No regex match.") + return 0, errors.New("no duration found in ffmpeg output") } durSecs, err := ParseDuration(dur) if err != nil { @@ -487,6 +515,11 @@ func WriteChapsFile(chapters []any, dur int) (string, error) { // TsToMp4 converts a TS file to MP4 using ffmpeg, optionally including chapters. // chapsFilePath is the path to the chapters metadata file (empty if no chapters). func TsToMp4(vidPathTs, vidPath, ffmpegNameStr, chapsFilePath string) error { + return TsToMp4Context(context.Background(), vidPathTs, vidPath, ffmpegNameStr, chapsFilePath) +} + +// TsToMp4Context converts a TS file and cancels ffmpeg with ctx. +func TsToMp4Context(ctx context.Context, vidPathTs, vidPath, ffmpegNameStr, chapsFilePath string) error { var ( errBuffer bytes.Buffer args []string @@ -499,12 +532,11 @@ func TsToMp4(vidPathTs, vidPath, ffmpegNameStr, chapsFilePath string) error { } else { args = []string{"-hide_banner", "-i", vidPathTs, "-c", "copy", vidPath} } - cmd := exec.Command(ffmpegNameStr, args...) + cmd := exec.CommandContext(ctx, ffmpegNameStr, args...) cmd.Stderr = &errBuffer err := cmd.Run() if err != nil { - errString := fmt.Sprintf("%s\n%s", err, errBuffer.String()) - return errors.New(errString) + return fmt.Errorf("ffmpeg MP4 conversion: %w: %s", err, errBuffer.String()) } return nil } @@ -561,9 +593,9 @@ func PrepareVideoProgressBox(meta *model.AlbArtResp, cfg *model.Config, progress RcloneEnabled: cfg.RcloneEnabled, StartTime: time.Now(), RenderInterval: model.DefaultProgressRenderInterval, + CurrentPhase: model.PhaseDownload, } - box.SetPhase(model.PhaseVerify) if deps.SetCurrentProgressBox != nil { deps.SetCurrentProgressBox(box) } @@ -617,7 +649,7 @@ func resolveManifestAndVariant(ctx context.Context, meta *model.AlbArtResp, vide if manifestUrl == "" { return "", nil, "", errors.New("the api didn't return a video manifest url") } - variant, retRes, err := ChooseVariant(manifestUrl, cfg.WantRes) + variant, retRes, err := ChooseVariantContext(ctx, manifestUrl, cfg.WantRes) if err != nil { ui.PrintError("Failed to get video master manifest") return "", nil, "", err @@ -713,7 +745,7 @@ func downloadVideoContent(ctx context.Context, vidPathTs, manBaseUrl string, seg func convertAndUploadVideo(ctx context.Context, vidPathTs, vidPath, artistFolder string, meta *model.AlbArtResp, cfg *model.Config, chapsAvail bool, progressBox *model.ProgressBoxState, deps *Deps) error { var chapsFilePath string if chapsAvail { - dur, getDurErr := GetDuration(vidPathTs, cfg.FfmpegNameStr) + dur, getDurErr := GetDurationContext(ctx, vidPathTs, cfg.FfmpegNameStr) if getDurErr != nil { ui.PrintError("Failed to get TS duration") return getDurErr @@ -731,13 +763,15 @@ func convertAndUploadVideo(ctx context.Context, vidPathTs, vidPath, artistFolder } ui.PrintInfo("Putting into MP4 container...") if progressBox != nil { - progressBox.SetPhase(model.PhaseVerify) + if err := progressBox.SetPhase(model.PhaseVerify); err != nil { + return err + } progressBox.SetMessage(model.MessagePriorityStatus, model.VideoConvertStatusLabel, model.StatusMessageDuration) if deps.RenderProgressBox != nil { deps.RenderProgressBox(progressBox) } } - if err := TsToMp4(vidPathTs, vidPath, cfg.FfmpegNameStr, chapsFilePath); err != nil { + if err := TsToMp4Context(ctx, vidPathTs, vidPath, cfg.FfmpegNameStr, chapsFilePath); err != nil { ui.PrintError("Failed to put TS into MP4 container") return err } @@ -746,14 +780,16 @@ func convertAndUploadVideo(ctx context.Context, vidPathTs, vidPath, artistFolder } if cfg.RcloneEnabled { if progressBox != nil { - progressBox.SetPhase(model.PhaseUpload) + if err := progressBox.SetPhase(model.PhaseUpload); err != nil { + return err + } progressBox.SetMessage(model.MessagePriorityStatus, model.VideoUploadStatusLabel, model.StatusMessageDuration) if deps.RenderProgressBox != nil { deps.RenderProgressBox(progressBox) } } if err := deps.UploadPath(ctx, vidPath, artistFolder, cfg, progressBox, true); err != nil { - helpers.HandleErr("Upload failed.", err, false) + return fmt.Errorf("upload video: %w", err) } } return nil @@ -797,7 +833,7 @@ func Video(ctx context.Context, videoID, uguID string, cfg *model.Config, stream ui.PrintError("Failed to get video manifest base URL") return err } - segUrls, err := GetSegUrls(manBaseUrl+variant.URI, query) + segUrls, err := GetSegUrlsContext(ctx, manBaseUrl+variant.URI, query) if err != nil { ui.PrintError("Failed to get video segment URLs") return err diff --git a/internal/helpers/errors.go b/internal/helpers/errors.go index ac00853..31d7ed4 100644 --- a/internal/helpers/errors.go +++ b/internal/helpers/errors.go @@ -19,20 +19,14 @@ var ( ErrScanTextFile = errors.New("failed to scan text file") ) -// HandleErr prints an error to stderr and optionally exits. When fatal is true, -// the process exits with code 1 after printing. When false, execution continues -// and callers are responsible for checking err themselves before calling. -// -// Deprecated: Prefer returning errors to callers instead of printing directly. -func HandleErr(errText string, err error, fatal bool) { +// ReportErr prints a non-fatal error to stderr. Prefer returning errors when the +// caller can make a meaningful decision about the failure. +func ReportErr(errText string, err error) { if err == nil { return } errString := errText + "\n" + err.Error() fmt.Fprintln(os.Stderr, errString) - if fatal { - os.Exit(1) - } } // WasRunFromSrc checks if the binary was run from a Go build temp directory. diff --git a/internal/helpers/paths.go b/internal/helpers/paths.go index 2938ff4..f5a7700 100644 --- a/internal/helpers/paths.go +++ b/internal/helpers/paths.go @@ -3,6 +3,7 @@ package helpers import ( "errors" "fmt" + "net/url" "os" "path" "path/filepath" @@ -42,7 +43,37 @@ func NewConfigPathResolver(cfg *model.Config) PathResolver { // Sanitise cleans a filename by replacing invalid characters. func Sanitise(filename string) string { san := sanRegex.ReplaceAllString(filename, "_") - return strings.TrimSpace(san) + san = strings.TrimSpace(san) + if san == "" || san == "." || san == ".." { + return "_" + } + return san +} + +// JoinWithinRoot joins path elements while proving the result remains beneath root. +func JoinWithinRoot(root string, elems ...string) (string, error) { + rootAbs, err := filepath.Abs(root) + if err != nil { + return "", err + } + candidate := filepath.Join(append([]string{rootAbs}, elems...)...) + rel, err := filepath.Rel(rootAbs, candidate) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("%w: %q", ErrPathTraversalDetected, candidate) + } + return candidate, nil +} + +// RedactURL removes credentials, query parameters, and fragments before logging. +func RedactURL(raw string) string { + u, err := url.Parse(raw) + if err != nil { + return "" + } + u.User = nil + u.RawQuery = "" + u.Fragment = "" + return u.String() } // BuildAlbumFolderName constructs a sanitized folder name for an album diff --git a/internal/helpers/paths_security_test.go b/internal/helpers/paths_security_test.go new file mode 100644 index 0000000..2136b7c --- /dev/null +++ b/internal/helpers/paths_security_test.go @@ -0,0 +1,40 @@ +package helpers + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/jmagar/nugs-cli/internal/model" +) + +func TestSanitiseReservedPathComponents(t *testing.T) { + for _, input := range []string{"", ".", "..", " .. "} { + if got := Sanitise(input); got == "" || got == "." || got == ".." { + t.Errorf("Sanitise(%q) = %q", input, got) + } + } +} + +func TestLocalShowPathConfinedToConfiguredRoot(t *testing.T) { + root := t.TempDir() + resolver := NewConfigPathResolver(&model.Config{OutPath: root}) + got := resolver.LocalShowPath(&model.AlbArtResp{ArtistName: "..", ContainerInfo: "../escape"}, model.MediaTypeAudio) + rel, err := filepath.Rel(root, got) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + t.Fatalf("path escaped root: %q (rel %q, err %v)", got, rel, err) + } +} + +func TestJoinWithinRootRejectsTraversal(t *testing.T) { + if _, err := JoinWithinRoot(t.TempDir(), "..", "escape"); err == nil { + t.Fatal("JoinWithinRoot accepted traversal") + } +} + +func TestRedactURLRemovesSignedQuery(t *testing.T) { + got := RedactURL("https://media.example/track.m3u8?token=secret#frag") + if strings.Contains(got, "secret") || strings.Contains(got, "?") || strings.Contains(got, "#") { + t.Fatalf("RedactURL leaked sensitive URL data: %q", got) + } +} diff --git a/internal/list/list.go b/internal/list/list.go index b60df13..a1d7a21 100644 --- a/internal/list/list.go +++ b/internal/list/list.go @@ -301,12 +301,7 @@ func ListArtists(ctx context.Context, jsonLevel string, showFilter string, deps Total: len(artists), } for i, artist := range artists { - output.Artists[i] = model.ArtistOutput{ - ArtistID: artist.ArtistID, - ArtistName: artist.ArtistName, - NumShows: artist.NumShows, - NumAlbums: artist.NumAlbums, - } + output.Artists[i] = model.ArtistOutput(artist) } jsonData, err := json.MarshalIndent(output, "", " ") if err != nil { @@ -433,11 +428,7 @@ func ListArtistShows(ctx context.Context, artistId string, jsonLevel string, dep if jsonLevel == "" { ui.PrintInfo("Fetching artist shows...") } - availType := 1 - if mf == model.MediaTypeVideo || mf == model.MediaTypeBoth { - availType = 2 - } - allMeta, err := api.GetArtistMetaWithAvailType(ctx, artistId, availType) + allMeta, err := api.GetArtistMetaWithAvailType(ctx, artistId, model.AvailableCatalogView) if err != nil { ui.PrintError("Failed to get artist metadata") return err @@ -525,7 +516,7 @@ func ListArtistShowsByVenue(ctx context.Context, artistId string, venueFilter st fmt.Printf("Fetching shows at venues matching \"%s\"...\n", venueFilter) } - allMeta, err := api.GetArtistMetaWithAvailType(ctx, artistId, 2) + allMeta, err := api.GetArtistMetaWithAvailType(ctx, artistId, model.AvailableCatalogView) if err != nil { ui.PrintError("Failed to get artist metadata") return err @@ -609,7 +600,7 @@ func ListArtistLatestShows(ctx context.Context, artistId string, limit int, json fmt.Printf("Fetching latest %d shows...\n", limit) } - allMeta, err := api.GetArtistMetaWithAvailType(ctx, artistId, 2) + allMeta, err := api.GetArtistMetaWithAvailType(ctx, artistId, model.AvailableCatalogView) if err != nil { ui.PrintError("Failed to get artist metadata") return err @@ -673,7 +664,7 @@ func ResolveCatPlistID(ctx context.Context, plistUrl string) (string, error) { if err != nil { return "", err } - resp, err := api.Client.Do(httpReq) + resp, err := api.Do(httpReq) if err != nil { return "", err } diff --git a/internal/model/constants.go b/internal/model/constants.go index 80dd653..eaa678f 100644 --- a/internal/model/constants.go +++ b/internal/model/constants.go @@ -55,6 +55,8 @@ const ( LiveHDVideoFormatLabel = "LIVE HD VIDEO" AvailableAvailabilityType = "AVAILABLE" ShowContainerType = "Show" + AvailableCatalogView = 1 + PreorderCatalogView = 2 Res2160 = "2160" Res4K = "4K" diff --git a/internal/model/progress_box.go b/internal/model/progress_box.go index 7724066..030c0b6 100644 --- a/internal/model/progress_box.go +++ b/internal/model/progress_box.go @@ -139,7 +139,8 @@ func (s *ProgressBoxState) isValidTransition(from, to string) bool { case PhaseUpload: return to == PhaseVerify || to == PhaseComplete || to == PhasePaused case PhaseVerify: - return to == PhaseComplete || to == PhaseError + // Verification/conversion may precede an optional remote upload. + return to == PhaseUpload || to == PhaseComplete || to == PhaseError case PhasePaused: return to == PhaseDownload || to == PhaseUpload || to == PhaseVerify case PhaseComplete, PhaseError: diff --git a/internal/model/progress_box_test.go b/internal/model/progress_box_test.go index 52c5596..33e8b0e 100644 --- a/internal/model/progress_box_test.go +++ b/internal/model/progress_box_test.go @@ -89,3 +89,16 @@ func TestProgressBoxState_ShouldRender_ConcurrentAccess_NoDeadlock(t *testing.T) t.Fatal("progress box concurrent render operations timed out") } } + +func TestProgressBoxState_VideoConversionCanTransitionToUpload(t *testing.T) { + state := &ProgressBoxState{CurrentPhase: PhaseDownload} + if err := state.SetPhase(PhaseVerify); err != nil { + t.Fatalf("download -> verify: %v", err) + } + if err := state.SetPhase(PhaseUpload); err != nil { + t.Fatalf("verify -> upload: %v", err) + } + if err := state.SetPhase(PhaseComplete); err != nil { + t.Fatalf("upload -> complete: %v", err) + } +} diff --git a/internal/model/types.go b/internal/model/types.go index f046896..5f8493f 100644 --- a/internal/model/types.go +++ b/internal/model/types.go @@ -36,34 +36,6 @@ type Config struct { SkipSizePreCalculation bool `json:"skipSizePreCalculation,omitempty"` } -// ArgsDescriptionFunc is set by package main's init() (in cmd/nugs/model_aliases.go) -// to provide colored help text. This is a startup-time side effect that wires the -// root package's argsDescription() into the model layer. Tests that call -// Description() should set this explicitly or accept the empty-string default. -// If nil, Description() returns an empty string (go-arg will use default help). -var ArgsDescriptionFunc func() string - -// Args holds CLI arguments parsed by go-arg. -type Args struct { - Urls []string `arg:"positional"` - Format int `arg:"-f" default:"-1" help:"Track download format.\n\t\t\t 1 = 16-bit / 44.1 kHz ALAC\n\t\t\t 2 = 16-bit / 44.1 kHz FLAC\n\t\t\t 3 = 24-bit / 48 kHz MQA\n\t\t\t 4 = 360 Reality Audio / best available\n\t\t\t 5 = 150 Kbps AAC"` - VideoFormat int `arg:"-F" default:"-1" help:"Video download format.\n\t\t\t 1 = 480p\n\t\t\t 2 = 720p\n\t\t\t 3 = 1080p\n\t\t\t 4 = 1440p\n\t\t\t 5 = 4K / best available"` - OutPath string `arg:"-o" help:"Where to download to. Path will be made if it doesn't already exist."` - ForceVideo bool `arg:"--force-video" help:"[Deprecated] Use 'nugs grab video' or set defaultOutputs in config."` - SkipVideos bool `arg:"--skip-videos" help:"[Deprecated] Use 'nugs grab audio' or set defaultOutputs in config."` - SkipChapters bool `arg:"--skip-chapters" help:"Skips chapters for videos."` -} - -// Description provides custom help text for go-arg. -// It delegates to ArgsDescriptionFunc if set, allowing the root package -// to inject colored output without model depending on UI code. -func (Args) Description() string { - if ArgsDescriptionFunc != nil { - return ArgsDescriptionFunc() - } - return "" -} - // Transport is used as a custom HTTP transport. type Transport struct{} diff --git a/internal/notify/gotify.go b/internal/notify/gotify.go index 68504dc..b09a418 100644 --- a/internal/notify/gotify.go +++ b/internal/notify/gotify.go @@ -6,12 +6,39 @@ import ( "context" "encoding/json" "fmt" + "io" + "net" "net/http" + "net/url" "strings" "time" ) -var httpClient = &http.Client{Timeout: 5 * time.Second} +var httpClient = &http.Client{ + Timeout: 5 * time.Second, + CheckRedirect: func(_ *http.Request, _ []*http.Request) error { + // The token is a credential. Never replay it to a redirect target. + return http.ErrUseLastResponse + }, +} + +func validateServerURL(raw string) (*url.URL, error) { + u, err := url.Parse(raw) + if err != nil || u.Hostname() == "" { + return nil, fmt.Errorf("gotify: invalid server URL") + } + if u.User != nil { + return nil, fmt.Errorf("gotify: server URL must not contain userinfo") + } + if u.Scheme == "https" { + return u, nil + } + ip := net.ParseIP(u.Hostname()) + if u.Scheme == "http" && (u.Hostname() == "localhost" || ip != nil && ip.IsLoopback()) { + return u, nil + } + return nil, fmt.Errorf("gotify: HTTPS is required for non-loopback servers") +} // Send posts a message to a Gotify server. // Returns nil immediately if url or token are empty. @@ -20,7 +47,11 @@ func Send(ctx context.Context, serverURL, token, title, message string, priority return nil } - url := strings.TrimRight(serverURL, "/") + "/message" + base, err := validateServerURL(serverURL) + if err != nil { + return err + } + requestURL := strings.TrimRight(base.String(), "/") + "/message" body, err := json.Marshal(map[string]any{ "title": title, @@ -31,7 +62,7 @@ func Send(ctx context.Context, serverURL, token, title, message string, priority return fmt.Errorf("gotify: marshal failed: %w", err) } - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body)) + req, err := http.NewRequestWithContext(ctx, http.MethodPost, requestURL, bytes.NewReader(body)) if err != nil { return fmt.Errorf("gotify: create request failed: %w", err) } @@ -42,7 +73,10 @@ func Send(ctx context.Context, serverURL, token, title, message string, priority if err != nil { return fmt.Errorf("gotify: send failed: %w", err) } - defer resp.Body.Close() + defer func() { + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) + _ = resp.Body.Close() + }() if resp.StatusCode < 200 || resp.StatusCode >= 300 { return fmt.Errorf("gotify: server returned %d", resp.StatusCode) diff --git a/internal/notify/gotify_security_test.go b/internal/notify/gotify_security_test.go new file mode 100644 index 0000000..4be0358 --- /dev/null +++ b/internal/notify/gotify_security_test.go @@ -0,0 +1,41 @@ +package notify + +import ( + "context" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" +) + +func TestSendRejectsPlainHTTPOffLoopback(t *testing.T) { + err := Send(context.Background(), "http://example.com", "secret", "title", "body", 1) + if err == nil { + t.Fatal("Send accepted plaintext HTTP for a non-loopback server") + } +} + +func TestSendDoesNotForwardTokenAcrossRedirect(t *testing.T) { + var redirected atomic.Bool + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + redirected.Store(true) + if got := r.Header.Get("X-Gotify-Token"); got != "" { + t.Errorf("redirect target received token %q", got) + } + w.WriteHeader(http.StatusOK) + })) + defer target.Close() + + source := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusTemporaryRedirect) + })) + defer source.Close() + + err := Send(context.Background(), source.URL, "secret", "title", "body", 1) + if err == nil { + t.Fatal("Send accepted a redirect response") + } + if redirected.Load() { + t.Fatal("redirect target was contacted") + } +} diff --git a/internal/rclone/concurrency_test.go b/internal/rclone/concurrency_test.go new file mode 100644 index 0000000..00141b2 --- /dev/null +++ b/internal/rclone/concurrency_test.go @@ -0,0 +1,73 @@ +package rclone + +import ( + "context" + "errors" + "os/exec" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/jmagar/nugs-cli/internal/model" +) + +func TestListArtistFoldersPreservesLastGoodBulkIndex(t *testing.T) { + adapter := NewStorageAdapter() + adapter.validatePath = func(string) error { return nil } + adapter.commandContext = func(context.Context, string, ...string) *exec.Cmd { return exec.Command("true") } + calls := 0 + adapter.outputCommand = func(*exec.Cmd) ([]byte, error) { + calls++ + if calls == 1 { + return []byte("show-one/\nshow-two/\n"), nil + } + return nil, errors.New("transient remote failure") + } + adapter.exitCode = func(error) (int, bool) { return 0, false } + cfg := &model.Config{RcloneEnabled: true, RcloneRemote: "remote-" + t.Name()} + first, err := adapter.ListArtistFolders(context.Background(), cfg, "artist", false) + if err != nil || len(first) != 2 { + t.Fatalf("first listing = %v, %v", first, err) + } + stale, err := adapter.ListArtistFolders(context.Background(), cfg, "artist", false) + if err != nil || len(stale) != 2 { + t.Fatalf("stale listing = %v, %v", stale, err) + } +} + +func TestRcloneProcessesAreGloballyBounded(t *testing.T) { + adapter := NewStorageAdapter() + adapter.validatePath = func(string) error { return nil } + adapter.commandContext = func(context.Context, string, ...string) *exec.Cmd { return exec.Command("true") } + var active atomic.Int32 + var peak atomic.Int32 + release := make(chan struct{}) + adapter.runCommand = func(*exec.Cmd) error { + current := active.Add(1) + for current > peak.Load() && !peak.CompareAndSwap(peak.Load(), current) { + } + <-release + active.Add(-1) + return nil + } + cfg := &model.Config{RcloneEnabled: true, RcloneRemote: "remote"} + const calls = maxRcloneProcesses + 6 + var wg sync.WaitGroup + for i := 0; i < calls; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = adapter.PathExists(context.Background(), cfg, "artist/show", false) + }() + } + deadline := time.Now().Add(time.Second) + for peak.Load() < maxRcloneProcesses && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := peak.Load(); got > maxRcloneProcesses { + t.Fatalf("peak rclone processes = %d, max = %d", got, maxRcloneProcesses) + } + close(release) + wg.Wait() +} diff --git a/internal/rclone/rclone.go b/internal/rclone/rclone.go index cffce80..60a0ab0 100644 --- a/internal/rclone/rclone.go +++ b/internal/rclone/rclone.go @@ -30,22 +30,19 @@ var ( speedPattern = regexp.MustCompile(`(?:^|,)\s*@?\s*([^,]*?/s)\s*(?:,|$)`) ) +// defaultStorageProvider is immutable. Alternate providers are injected through +// download.Deps instead of mutating process-global state. var defaultStorageProvider model.StorageProvider = NewStorageAdapter() -// SetStorageProvider swaps the default storage provider used by rclone wrappers. -// Passing nil resets it to the standard rclone adapter. -func SetStorageProvider(provider model.StorageProvider) { - if provider == nil { - defaultStorageProvider = NewStorageAdapter() - return - } - defaultStorageProvider = provider -} - // CheckRcloneAvailable verifies rclone is installed and available in PATH. func CheckRcloneAvailable(quiet bool) error { cmd := exec.Command("rclone", "version") - output, err := cmd.Output() + var output []byte + err := WithProcessSlot(context.Background(), func() error { + var runErr error + output, runErr = cmd.Output() + return runErr + }) if err != nil { return fmt.Errorf("rclone is not installed or not available in PATH: %w\n"+ "Please install rclone from https://rclone.org/downloads/ or disable rclone in config.json", err) @@ -73,7 +70,7 @@ func CheckRclonePathOnline(cfg *model.Config) string { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() cmd := exec.CommandContext(ctx, "rclone", "lsf", target) - err := cmd.Run() + err := WithProcessSlot(ctx, cmd.Run) if err == nil { return "Online" } @@ -338,19 +335,24 @@ func RunRcloneWithProgress(cmd *exec.Cmd, onProgress func(percent int, speed, up // BuildRcloneVerifyCommand constructs the rclone check command for upload verification. func BuildRcloneVerifyCommand(localPath, remoteFullPath string) (*exec.Cmd, error) { + return BuildRcloneVerifyCommandContext(context.Background(), localPath, remoteFullPath) +} + +// BuildRcloneVerifyCommandContext constructs a cancellable rclone check command. +func BuildRcloneVerifyCommandContext(ctx context.Context, localPath, remoteFullPath string) (*exec.Cmd, error) { localInfo, err := os.Stat(localPath) if err != nil { return nil, fmt.Errorf("failed to stat local path for verification: %w", err) } if localInfo.IsDir() { - return exec.Command("rclone", "check", "--one-way", localPath, remoteFullPath), nil + return exec.CommandContext(ctx, "rclone", "check", "--one-way", localPath, remoteFullPath), nil } localDir := filepath.Dir(localPath) remoteDir := path.Dir(remoteFullPath) fileName := filepath.Base(localPath) - return exec.Command("rclone", "check", "--one-way", "--include", fileName, localDir, remoteDir), nil + return exec.CommandContext(ctx, "rclone", "check", "--one-way", "--include", fileName, localDir, remoteDir), nil } // RemotePathExists checks if a path exists on the configured rclone remote. diff --git a/internal/rclone/storage_adapter.go b/internal/rclone/storage_adapter.go index bb847cc..29704a6 100644 --- a/internal/rclone/storage_adapter.go +++ b/internal/rclone/storage_adapter.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "strings" + "sync" "time" "github.com/jmagar/nugs-cli/internal/helpers" @@ -15,6 +16,37 @@ import ( "github.com/jmagar/nugs-cli/internal/ui" ) +const maxRcloneProcesses = 8 + +var ( + rcloneProcessSlots = make(chan struct{}, maxRcloneProcesses) + folderCacheMu sync.RWMutex + folderCache = make(map[string]map[string]struct{}) +) + +func acquireRcloneSlot(ctx context.Context) (func(), error) { + select { + case rcloneProcessSlots <- struct{}{}: + return func() { <-rcloneProcessSlots }, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// WithProcessSlot runs fn while holding a package-global rclone subprocess +// slot. Catalog bulk scans use this too, so every rclone spawn shares one cap. +func WithProcessSlot(ctx context.Context, fn func() error) error { + if ctx == nil { + ctx = context.Background() + } + release, err := acquireRcloneSlot(ctx) + if err != nil { + return err + } + defer release() + return fn() +} + // StorageAdapter is a storage provider backed by the rclone CLI. // Function fields are injected to keep the adapter unit-testable. type StorageAdapter struct { @@ -24,7 +56,7 @@ type StorageAdapter struct { removeAll func(path string) error buildUploadCommand func(ctx context.Context, localPath, artistFolder string, cfg *model.Config, transfers int, isVideo bool) (*exec.Cmd, string, error) - buildVerifyCommand func(localPath, remoteFullPath string) (*exec.Cmd, error) + buildVerifyCommand func(ctx context.Context, localPath, remoteFullPath string) (*exec.Cmd, error) runWithProgress func(cmd *exec.Cmd, onProgress UploadProgressFunc) error runCommand func(cmd *exec.Cmd) error outputCommand func(cmd *exec.Cmd) ([]byte, error) @@ -41,7 +73,7 @@ func NewStorageAdapter() *StorageAdapter { calculateLocal: helpers.CalculateLocalSize, removeAll: os.RemoveAll, buildUploadCommand: BuildRcloneUploadCommandContext, - buildVerifyCommand: BuildRcloneVerifyCommand, + buildVerifyCommand: BuildRcloneVerifyCommandContext, runWithProgress: func(cmd *exec.Cmd, onProgress UploadProgressFunc) error { return RunRcloneWithProgress(cmd, onProgress) }, @@ -83,6 +115,11 @@ func (a *StorageAdapter) Upload(ctx context.Context, cfg *model.Config, req mode if ctx == nil { ctx = context.Background() } + release, err := acquireRcloneSlot(ctx) + if err != nil { + return fmt.Errorf("waiting for rclone process slot: %w", err) + } + defer release() cmd, remoteFullPath, err := a.buildUploadCommand(ctx, req.LocalPath, req.ArtistFolder, cfg, transfers, req.IsVideo) if err != nil { return err @@ -131,7 +168,7 @@ func (a *StorageAdapter) Upload(ctx context.Context, cfg *model.Config, req mode ui.PrintInfo("Verifying upload integrity...") } - verifyCmd, err := a.buildVerifyCommand(req.LocalPath, remoteFullPath) + verifyCmd, err := a.buildVerifyCommand(ctx, req.LocalPath, remoteFullPath) if err != nil { return fmt.Errorf("failed to build upload verification command: %w", err) } @@ -176,9 +213,14 @@ func (a *StorageAdapter) PathExists(ctx context.Context, cfg *model.Config, remo } timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() + release, err := acquireRcloneSlot(timeoutCtx) + if err != nil { + return false, fmt.Errorf("waiting for rclone process slot: %w", err) + } + defer release() cmd := a.commandContext(timeoutCtx, "rclone", "lsf", fullPath) - err := a.runCommand(cmd) + err = a.runCommand(cmd) if err == nil { return true, nil } @@ -208,12 +250,23 @@ func (a *StorageAdapter) ListArtistFolders(ctx context.Context, cfg *model.Confi } timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() + release, err := acquireRcloneSlot(timeoutCtx) + if err != nil { + return nil, fmt.Errorf("waiting for rclone process slot: %w", err) + } + defer release() cmd := a.commandContext(timeoutCtx, "rclone", "lsf", fullPath, "--dirs-only") output, err := a.outputCommand(cmd) if err != nil { if code, ok := a.exitCode(err); ok && code == 3 { return folders, nil } + folderCacheMu.RLock() + stale, ok := folderCache[fullPath] + folderCacheMu.RUnlock() + if ok { + return cloneFolderSet(stale), nil + } return nil, fmt.Errorf("failed to list remote artist folders: %w", err) } @@ -224,5 +277,16 @@ func (a *StorageAdapter) ListArtistFolders(ctx context.Context, cfg *model.Confi } folders[trimmed] = struct{}{} } + folderCacheMu.Lock() + folderCache[fullPath] = cloneFolderSet(folders) + folderCacheMu.Unlock() return folders, nil } + +func cloneFolderSet(source map[string]struct{}) map[string]struct{} { + clone := make(map[string]struct{}, len(source)) + for value := range source { + clone[value] = struct{}{} + } + return clone +} diff --git a/internal/rclone/storage_adapter_test.go b/internal/rclone/storage_adapter_test.go index e609882..d98f312 100644 --- a/internal/rclone/storage_adapter_test.go +++ b/internal/rclone/storage_adapter_test.go @@ -35,7 +35,7 @@ func TestStorageAdapterUploadWithHooksAndDelete(t *testing.T) { onProgress(55, "8 MiB/s", "440 MiB", "800 MiB") return nil } - adapter.buildVerifyCommand = func(localPath, remoteFullPath string) (*exec.Cmd, error) { + adapter.buildVerifyCommand = func(_ context.Context, localPath, remoteFullPath string) (*exec.Cmd, error) { if remoteFullPath != "remote:path/artist/local" { t.Fatalf("verify remote path = %q", remoteFullPath) } diff --git a/internal/runtime/detach_unix.go b/internal/runtime/detach_unix.go index 6c8c0ee..7c117bc 100644 --- a/internal/runtime/detach_unix.go +++ b/internal/runtime/detach_unix.go @@ -25,7 +25,7 @@ func SpawnDetached(args []string) (int, string, error) { return 0, "", err } logPath := filepath.Join(cacheDir, "runtime.log") - logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) if err != nil { return 0, "", err } diff --git a/internal/runtime/status.go b/internal/runtime/status.go index 00ecbff..bf2c532 100644 --- a/internal/runtime/status.go +++ b/internal/runtime/status.go @@ -230,12 +230,28 @@ func WriteRuntimeControl(control model.RuntimeControl) error { // WriteFileAtomic writes data to a file atomically using a temp file and rename. func WriteFileAtomic(path string, data []byte, mode os.FileMode) error { - tmpPath := path + ".tmp" - if err := os.WriteFile(tmpPath, data, mode); err != nil { + tmp, err := os.CreateTemp(filepath.Dir(path), ".nugs-runtime-*.tmp") + if err != nil { + return err + } + tmpPath := tmp.Name() + defer os.Remove(tmpPath) + if err := tmp.Chmod(mode); err != nil { + _ = tmp.Close() + return err + } + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + return err + } + if err := tmp.Close(); err != nil { return err } if err := os.Rename(tmpPath, path); err != nil { - _ = os.Remove(tmpPath) return err } return nil diff --git a/scripts/check-docs.py b/scripts/check-docs.py new file mode 100644 index 0000000..8570d53 --- /dev/null +++ b/scripts/check-docs.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 +"""Validate local Markdown links, portable paths, and Config field coverage.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path +from urllib.parse import unquote + + +ROOT = Path(__file__).resolve().parent.parent +PUBLIC_DOCS = [ROOT / "README.md", ROOT / "CLAUDE.md", ROOT / "CHANGELOG.md"] +PUBLIC_DOCS.extend(sorted((ROOT / "docs").glob("*.md"))) +PUBLIC_DOCS.append(ROOT / "INCREMENTAL_CATALOG_UPDATE.md") + +LINK_RE = re.compile(r"(? list[str]: + errors: list[str] = [] + for raw_target in LINK_RE.findall(text): + target = raw_target.strip().split(maxsplit=1)[0].strip("<>") + if not target or target.startswith(("#", "http://", "https://", "mailto:")): + continue + file_part = unquote(target.split("#", 1)[0]) + resolved = (path.parent / file_part).resolve() + try: + resolved.relative_to(ROOT) + except ValueError: + errors.append(f"{path.relative_to(ROOT)}: link escapes repository: {target}") + continue + if not resolved.exists(): + errors.append(f"{path.relative_to(ROOT)}: missing local link target: {target}") + return errors + + +def config_coverage_errors() -> list[str]: + model_text = (ROOT / "internal/model/types.go").read_text(encoding="utf-8") + config_text = (ROOT / "docs/CONFIG.md").read_text(encoding="utf-8") + config_struct = model_text.split("type Config struct {", 1)[1].split("\n}", 1)[0] + fields = {tag for tag in JSON_TAG_RE.findall(config_struct) if tag != "-"} + missing = sorted(field for field in fields if f"`{field}`" not in config_text) + if missing: + return ["docs/CONFIG.md: undocumented Config JSON fields: " + ", ".join(missing)] + return [] + + +def main() -> int: + errors: list[str] = [] + for path in PUBLIC_DOCS: + text = path.read_text(encoding="utf-8") + errors.extend(local_link_errors(path, text)) + for line_no, line in enumerate(text.splitlines(), 1): + if re.search(r"/home/[A-Za-z0-9._-]+/", line): + errors.append( + f"{path.relative_to(ROOT)}:{line_no}: developer-home absolute path" + ) + errors.extend(config_coverage_errors()) + + if errors: + print("Documentation validation failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + return 1 + print("Documentation links, portability, and Config field coverage are valid.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/openwiki/package-lock.json b/tools/openwiki/package-lock.json new file mode 100644 index 0000000..c5fb2f5 --- /dev/null +++ b/tools/openwiki/package-lock.json @@ -0,0 +1,2683 @@ +{ + "name": "nugs-openwiki-tooling", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "nugs-openwiki-tooling", + "version": "0.0.0", + "dependencies": { + "openwiki": "0.2.0" + }, + "engines": { + "node": "22.x" + } + }, + "node_modules/@alcalzone/ansi-tokenize": { + "version": "0.1.3", + "resolved": "https://registry.npmjs.org/@alcalzone/ansi-tokenize/-/ansi-tokenize-0.1.3.tgz", + "integrity": "sha512-3yWxPTq3UQ/FY9p1ErPxIyfT64elWaMvM9lIHnaqpyft63tkxodF5aUElYHrdisWve5cETkh1+KBw1yJuW0aRw==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "is-fullwidth-code-point": "^4.0.0" + }, + "engines": { + "node": ">=14.13.1" + } + }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.112.3", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.112.3.tgz", + "integrity": "sha512-wjcozJlitVIuBEw9cj/xBuRznwkhcLmXmNzlFoeHbh4AvrDG3HGZrdvEOTTmobcbhjGkfOpKbmDTCQ4s9LQvCg==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1", + "standardwebhooks": "^1.0.0" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@anthropic-ai/vertex-sdk": { + "version": "0.19.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/vertex-sdk/-/vertex-sdk-0.19.0.tgz", + "integrity": "sha512-Ja5NkDAmdCcvCJCvkY/6uJ+9krOiXFN56dzb+8n5apElHrYpUMlOCHCVRuLLsJCVWTsN8w60sgN9RSXZ+LPqNA==", + "license": "MIT", + "dependencies": { + "@anthropic-ai/sdk": ">=0.50.3 <1", + "google-auth-library": "^10.2.0" + } + }, + "node_modules/@aws-sdk/client-bedrock-agent-runtime": { + "version": "3.1090.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-agent-runtime/-/client-bedrock-agent-runtime-3.1090.0.tgz", + "integrity": "sha512-Rr1+ezVE/Oifm4ecgo2rUeyA33lDNAA5PJkVnUJSP/hZYYbhoq/VM7A2BudRGMzBV6zo2xiKFKkr0MLXHpsOkA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-node": "^3.972.70", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-bedrock-runtime": { + "version": "3.1090.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-bedrock-runtime/-/client-bedrock-runtime-3.1090.0.tgz", + "integrity": "sha512-uFo62YTKik92NxlnExaTU2rH2WQCseW8XvZnBiSab+YWU9pl1URhVsnxY8whUe4EVOuxdAAlgCuChTJo6gZIfQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-node": "^3.972.70", + "@aws-sdk/eventstream-handler-node": "^3.972.29", + "@aws-sdk/middleware-eventstream": "^3.972.24", + "@aws-sdk/middleware-websocket": "^3.972.41", + "@aws-sdk/token-providers": "3.1090.0", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/client-kendra": { + "version": "3.1090.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/client-kendra/-/client-kendra-3.1090.0.tgz", + "integrity": "sha512-zps/px25bLHgQA+6kAXqzAoIUMhyw1e2koqFdhYKnIpRzCSnUNw7AP8E//vddfQQqXwE7vPrtADheb2SectHiQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-node": "^3.972.70", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/core": { + "version": "3.975.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/core/-/core-3.975.3.tgz", + "integrity": "sha512-7ur3kCKuvPLqlsZ2XlvnNBVQ7KkpSu6Y6dOTwSPHLrFpTEfZM8isLBJc4cgv96WB7GifeVM436mpycwxBd2vEA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@aws-sdk/xml-builder": "^3.972.36", + "@aws/lambda-invoke-store": "^0.3.0", + "@smithy/core": "^3.29.4", + "@smithy/signature-v4": "^5.6.5", + "@smithy/types": "^4.16.1", + "bowser": "^2.11.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-env": { + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-env/-/credential-provider-env-3.972.59.tgz", + "integrity": "sha512-Ny5e4Mfh3QPmiAc0AiUe+cbTXDlxkU3Rc+EpWOfyWeWEy6yp7Fa1KmfNeCc+1a8by9zQ9gtohmiQUkMPScF3ng==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-http": { + "version": "3.972.61", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-http/-/credential-provider-http-3.972.61.tgz", + "integrity": "sha512-8jAjgStl5Ytq4+HF3X/9f+EmRinaRbGRRtQGktlPfBRVx73H+R1y48vIeXerQtYGFaUqkEp3fT6jP854rVO2yQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-ini": { + "version": "3.973.4", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-ini/-/credential-provider-ini-3.973.4.tgz", + "integrity": "sha512-e6ZvVsj90aRALf1kHP+J4iqC1496ZpVgqI/+u0LJ5HL7q7ATauGy4gdDvRCP13L1pN/fMiZLah162PGIYkbUVQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/credential-provider-env": "^3.972.59", + "@aws-sdk/credential-provider-http": "^3.972.61", + "@aws-sdk/credential-provider-login": "^3.972.66", + "@aws-sdk/credential-provider-process": "^3.972.59", + "@aws-sdk/credential-provider-sso": "^3.973.3", + "@aws-sdk/credential-provider-web-identity": "^3.972.65", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/credential-provider-imds": "^4.4.9", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-login": { + "version": "3.972.66", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-login/-/credential-provider-login-3.972.66.tgz", + "integrity": "sha512-g2fsqm87r/nKthLZ0VkkDBElkGg0PvSa8d97HQ6EilMbJTZ6hxa8FxkSZyJfgPfFdZn0TTmkOffQmTSUcAHIng==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-node": { + "version": "3.972.70", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-node/-/credential-provider-node-3.972.70.tgz", + "integrity": "sha512-3xzvkGdykBunxqh8WudmUpSyLWvIhfI6aBQo1b5rb3mDO5mNLadK+0hiI0qBQBMVynJbfLO+Ajy9dztMwy9O8w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/credential-provider-env": "^3.972.59", + "@aws-sdk/credential-provider-http": "^3.972.61", + "@aws-sdk/credential-provider-ini": "^3.973.4", + "@aws-sdk/credential-provider-process": "^3.972.59", + "@aws-sdk/credential-provider-sso": "^3.973.3", + "@aws-sdk/credential-provider-web-identity": "^3.972.65", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/credential-provider-imds": "^4.4.9", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-process": { + "version": "3.972.59", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-process/-/credential-provider-process-3.972.59.tgz", + "integrity": "sha512-DlZF2/MhLlatDdlrIy3CUCpfdbLrKx+3SMjVo+WyHnPpwzkc/M3vwAHw4OVJf7DMvO+4vfRqSCMc/E9I1auN0g==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso": { + "version": "3.973.3", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-sso/-/credential-provider-sso-3.973.3.tgz", + "integrity": "sha512-hmdDHoy2G5Es2e8IgelNMYUuSQI6uCIAKZMJ2u2PdKDhxvbk1uWD/g4+R7R5c/tJfKEB1+KjjWiaoCr/S+ZTiQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/token-providers": "3.1088.0", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-sso/node_modules/@aws-sdk/token-providers": { + "version": "3.1088.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1088.0.tgz", + "integrity": "sha512-4ObatWt2qpJg5FBk4LOOKrTQYzaqeewAtdO3r9ZO8lH9YqLtpTzLyIdy0mJ+nVdfYOnqISkKNfmzP22bNDhwyw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/credential-provider-web-identity": { + "version": "3.972.65", + "resolved": "https://registry.npmjs.org/@aws-sdk/credential-provider-web-identity/-/credential-provider-web-identity-3.972.65.tgz", + "integrity": "sha512-gHQb/Kt0chjk/JQDa/GJDqmAvEuVn8n7z10wK2h0LFM9TUDRkohgOO4aEF+s2sBLM0br7Cl5W6P7phgjrrJvLQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/eventstream-handler-node": { + "version": "3.972.29", + "resolved": "https://registry.npmjs.org/@aws-sdk/eventstream-handler-node/-/eventstream-handler-node-3.972.29.tgz", + "integrity": "sha512-t3tKQRTVXsI2QNPE3CaNjHl0wRO9Xi3acZkAyti2RQsiFmZ9Gi0kArX2ighlRJ1BtDVuul413gThAgzyTfgmWA==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-eventstream": { + "version": "3.972.24", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-eventstream/-/middleware-eventstream-3.972.24.tgz", + "integrity": "sha512-oykin4mDWxNOuYQ7SF1cHzgYeuFEkF4cdRwgvjFFbIklkx09qIFBiOgsORafG9sXZFO3TayMmQuAQYgADXhI8w==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/middleware-websocket": { + "version": "3.972.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/middleware-websocket/-/middleware-websocket-3.972.41.tgz", + "integrity": "sha512-LSbGvvYmjc4Br9BPYI2dTLnIclmrSiQbahkP4D6nRGVEv4qsCZ8csVuKBPVEEFCVD+EEngGh8ROls6XpumtwMg==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/signature-v4": "^5.6.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">= 14.0.0" + } + }, + "node_modules/@aws-sdk/nested-clients": { + "version": "3.997.33", + "resolved": "https://registry.npmjs.org/@aws-sdk/nested-clients/-/nested-clients-3.997.33.tgz", + "integrity": "sha512-dVZOroI/r3/ENvqNGgjMPul+jjlz9GddfVusgTXlVjfZj5isibOxecLkGQbRPp8XOuX+RAfjXLFgPkD1JS5xrw==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/signature-v4-multi-region": "^3.996.41", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/fetch-http-handler": "^5.6.6", + "@smithy/node-http-handler": "^4.9.6", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/signature-v4-multi-region": { + "version": "3.996.41", + "resolved": "https://registry.npmjs.org/@aws-sdk/signature-v4-multi-region/-/signature-v4-multi-region-3.996.41.tgz", + "integrity": "sha512-QMUytg+FQMGouc8gHS00KoYih3+N6cqmVI/pQGOIo7Nr7OpQaiXjSYOuL+vsPZ1tymY4LAQ8MYcHJmws5LRxng==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/types": "^3.974.2", + "@smithy/signature-v4": "^5.6.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/token-providers": { + "version": "3.1090.0", + "resolved": "https://registry.npmjs.org/@aws-sdk/token-providers/-/token-providers-3.1090.0.tgz", + "integrity": "sha512-uwPWr8zRBL1YLiWpIWoEOVj0bgGJ0M4Gfi151lEJroQq/7Lc/phrJVIEnCTEHHa1BIse525BfunmOZLMBfD9OQ==", + "license": "Apache-2.0", + "dependencies": { + "@aws-sdk/core": "^3.975.3", + "@aws-sdk/nested-clients": "^3.997.33", + "@aws-sdk/types": "^3.974.2", + "@smithy/core": "^3.29.4", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/types": { + "version": "3.974.2", + "resolved": "https://registry.npmjs.org/@aws-sdk/types/-/types-3.974.2.tgz", + "integrity": "sha512-3W6IUtSxFbH6X7Wb7DzGCV5QiFQsd0g8bOfntpmDxQlzBoKWUMBu/JPQR0DwkE+Hpnxd6db1tXbOwdeHddG6cA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws-sdk/xml-builder": { + "version": "3.972.36", + "resolved": "https://registry.npmjs.org/@aws-sdk/xml-builder/-/xml-builder-3.972.36.tgz", + "integrity": "sha512-RdGmS1GLrtaTOLE1ElSluMldNrpk9Emq6uYs8SS8iHlu5xTAmM9rRkM91o48+rIRryBtyO9t+uLYCoMG6jVMVA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@aws/lambda-invoke-store": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/@aws/lambda-invoke-store/-/lambda-invoke-store-0.3.0.tgz", + "integrity": "sha512-sl4Bm6yiMNYrZKkqqDFWN0UfnWhlS8ivKxrYl+6t0gCLrqr8y3B2IqZZbFRkfaVVp7C/baApyh71P+LeE1A2sQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@babel/runtime": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz", + "integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==", + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@cfworker/json-schema": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@cfworker/json-schema/-/json-schema-4.1.1.tgz", + "integrity": "sha512-gAmrUZSGtKc3AiBL71iNWxDsyUC5uMaKKGdvzYsBoTW/xi42JQHl7eKV2OYzCUqvc+D2RCcf7EXY2iCyFIk6og==", + "license": "MIT" + }, + "node_modules/@langchain/anthropic": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-1.5.1.tgz", + "integrity": "sha512-j92zCCd5BFH3rHMRzc2wBmSKDoVpinof1oh8aFiAz9TWbSOc4tGU4n6bqwy/wP0GH1uO96zZHLGCHBMPgrxTNw==", + "license": "MIT", + "dependencies": { + "@anthropic-ai/sdk": "^0.103.0", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@langchain/core": "^1.2.1" + } + }, + "node_modules/@langchain/anthropic/node_modules/@anthropic-ai/sdk": { + "version": "0.103.0", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.103.0.tgz", + "integrity": "sha512-1uG7RNgoHTUxzOXqSCODKt0UTVlxWiHk/2Tt2/uQJiPW7XzBeKVuJyd3Aw6T3LPyvZV/jDTnPLX7SaM70WLLjA==", + "license": "MIT", + "dependencies": { + "json-schema-to-ts": "^3.1.1", + "standardwebhooks": "^1.0.0" + }, + "bin": { + "anthropic-ai-sdk": "bin/cli" + }, + "peerDependencies": { + "zod": "^3.25.0 || ^4.0.0" + }, + "peerDependenciesMeta": { + "zod": { + "optional": true + } + } + }, + "node_modules/@langchain/aws": { + "version": "1.4.3", + "resolved": "https://registry.npmjs.org/@langchain/aws/-/aws-1.4.3.tgz", + "integrity": "sha512-X3oNXI1/pLizW6D4Wd1ojWTTPGnRgli8S+rm+CAhFAevIP3UdFfTDY6xih7xrQQ/0sO7po/eD0jHeZoLRKhhBg==", + "license": "MIT", + "dependencies": { + "@aws-sdk/client-bedrock-agent-runtime": "^3.1078.0", + "@aws-sdk/client-bedrock-runtime": "^3.1078.0", + "@aws-sdk/client-kendra": "^3.1078.0", + "@aws-sdk/credential-provider-node": "^3.972.61" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@langchain/core": "^1.0.0" + } + }, + "node_modules/@langchain/core": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-1.2.3.tgz", + "integrity": "sha512-F+L5SsciykwDl7eDxacnhDTcWe1IF6jetzfkvI5PPfq6ogWHO7xcjU90SGh/3lqbbS0tgun+qF01KIqxawrCsA==", + "license": "MIT", + "dependencies": { + "@cfworker/json-schema": "^4.0.2", + "@standard-schema/spec": "^1.1.0", + "js-tiktoken": "^1.0.12", + "langsmith": ">=0.5.0 <1.0.0", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + } + }, + "node_modules/@langchain/google": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/@langchain/google/-/google-0.2.1.tgz", + "integrity": "sha512-9WrE2wnb4mNwK4deGQZDCHdwd5E84rlIu1tHGStI61MqOWnGJe2KApAeG+49kVya+aG0k71LrvtNxoJcOgXP4g==", + "license": "MIT", + "dependencies": { + "eventsource-parser": "^3.1.0", + "google-auth-library": "^10.6.2", + "jose": "^6.2.3" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@langchain/core": "^1.0.0" + } + }, + "node_modules/@langchain/langgraph": { + "version": "1.4.8", + "resolved": "https://registry.npmjs.org/@langchain/langgraph/-/langgraph-1.4.8.tgz", + "integrity": "sha512-DN1Np1XefdBEbp1qBKlt39cwoL743AAGpR5Ipja0gY2YbWvsoQnOTIrjnj/orSAhaUYsdTKS8VSWdFzsHZo6Ig==", + "license": "MIT", + "dependencies": { + "@langchain/langgraph-checkpoint": "^1.1.3", + "@langchain/langgraph-sdk": "~1.9.26", + "@langchain/protocol": "^0.0.18", + "@standard-schema/spec": "1.1.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "zod": "^3.25.32 || ^4.2.0" + } + }, + "node_modules/@langchain/langgraph-checkpoint": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint/-/langgraph-checkpoint-1.1.3.tgz", + "integrity": "sha512-wgzdQNeEsdw1e+4lvlj0tdq/RYR/k1vPin10g0ymGoehZDDgd9nvIllGXSXN4TFgF9sf5qQP/KTkOcLfeseIhA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48" + } + }, + "node_modules/@langchain/langgraph-checkpoint-sqlite": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-checkpoint-sqlite/-/langgraph-checkpoint-sqlite-1.0.3.tgz", + "integrity": "sha512-odOy8z45HvbWerx4t9g//fA38QNq0Rr6cw8UeHbx1LDOXbarMsfJ5CrmmJs5FlhK16MTlsyaF2m+JJ2cXkTLPg==", + "license": "MIT", + "dependencies": { + "better-sqlite3": "^12.10.0" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@langchain/core": "^1.1.44", + "@langchain/langgraph-checkpoint": "^1.0.0" + } + }, + "node_modules/@langchain/langgraph-sdk": { + "version": "1.9.27", + "resolved": "https://registry.npmjs.org/@langchain/langgraph-sdk/-/langgraph-sdk-1.9.27.tgz", + "integrity": "sha512-EHGFsxawniLw2pGVUDLSSdMriASIP4oz0Mihytb50iRsIqhmODzGBWW7iLWA/0hqavggNpqjat4WD7LjmxiXSQ==", + "license": "MIT", + "dependencies": { + "@langchain/protocol": "^0.0.18", + "@types/json-schema": "^7.0.15", + "p-queue": "^9.0.1", + "p-retry": "^7.1.1" + }, + "peerDependencies": { + "@langchain/core": "^1.1.48", + "react": "^18 || ^19", + "react-dom": "^18 || ^19", + "svelte": "^4.0.0 || ^5.0.0", + "vue": "^3.0.0" + }, + "peerDependenciesMeta": { + "react": { + "optional": true + }, + "react-dom": { + "optional": true + }, + "svelte": { + "optional": true + }, + "vue": { + "optional": true + } + } + }, + "node_modules/@langchain/langgraph-sdk/node_modules/eventemitter3": { + "version": "5.0.4", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.4.tgz", + "integrity": "sha512-mlsTRyGaPBjPedk6Bvw+aqbsXDtoAyAzm5MO7JgU+yVRyMQ5O8bD4Kcci7BS85f93veegeCPkL8R4GLClnjLFw==", + "license": "MIT" + }, + "node_modules/@langchain/langgraph-sdk/node_modules/p-queue": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-9.3.1.tgz", + "integrity": "sha512-POWdiIPmsUPGwb4FeQ4OBg46aqmcInSWe45CKDsGHiOBiVQM9chqfQTuqhuTzcg2Vz9faTI65at0KkVyVEiCHw==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^5.0.4", + "p-timeout": "^7.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@langchain/langgraph-sdk/node_modules/p-timeout": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-7.0.1.tgz", + "integrity": "sha512-AxTM2wDGORHGEkPCt8yqxOTMgpfbEHqF51f/5fJCmwFC3C/zNcGT63SymH2ttOAaiIws2zVg4+izQCjrakcwHg==", + "license": "MIT", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@langchain/openai": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-1.5.5.tgz", + "integrity": "sha512-wX7dwb9z4nf5FHXlIl/X2mk08pzonvRHCt1D4+s1zXLP0duYDC95j7dulPIQJ6fmhbyYQc9Ki8mEhY/D1lB8kw==", + "license": "MIT", + "dependencies": { + "js-tiktoken": "^1.0.12", + "openai": "^6.41.0", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@langchain/core": "^1.2.2" + } + }, + "node_modules/@langchain/openrouter": { + "version": "0.4.5", + "resolved": "https://registry.npmjs.org/@langchain/openrouter/-/openrouter-0.4.5.tgz", + "integrity": "sha512-WI5xpmwW2KqJ5pErFAAq5SUXHVQp2F4RVF/kQzXMqOhjd1UVVeIBrIb22AJGsQy4Tybh91ag4vp6/6f/2vRSwg==", + "license": "MIT", + "dependencies": { + "@langchain/openai": "1.5.5", + "eventsource-parser": "^3.1.0", + "openai": "^6.41.0" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@langchain/core": "^1.0.0" + } + }, + "node_modules/@langchain/protocol": { + "version": "0.0.18", + "resolved": "https://registry.npmjs.org/@langchain/protocol/-/protocol-0.0.18.tgz", + "integrity": "sha512-XW1egQtPfsGI41w2AMZNFZrUIwFSQHTjVMZs0OaTpCAvht/QLoaPN8FQcsysMVypOhupG28J29yOorrc70otBQ==", + "license": "MIT" + }, + "node_modules/@langchain/tavily": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/@langchain/tavily/-/tavily-1.2.0.tgz", + "integrity": "sha512-aPLPgtw8+b/Rnr3H+X8H8z98T/Y7JuCE4B5eqRDHoEWgZvMDUFF7divqwQqCTMq2deQttlVrm5bN5JbKaAR7/w==", + "license": "MIT", + "dependencies": { + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@langchain/core": "^1.0.0" + } + }, + "node_modules/@nodelib/fs.scandir": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", + "integrity": "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "2.0.5", + "run-parallel": "^1.1.9" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.stat": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/@nodelib/fs.stat/-/fs.stat-2.0.5.tgz", + "integrity": "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/@nodelib/fs.walk": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/@nodelib/fs.walk/-/fs.walk-1.2.8.tgz", + "integrity": "sha512-oGB+UxlgWcgQkgwo8GcEGwemoTFt3FIO9ababBmaGwXIoBKZ+GTy0pP185beGg7Llih/NSHSV2XAs1lnznocSg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.scandir": "2.1.5", + "fastq": "^1.6.0" + }, + "engines": { + "node": ">= 8" + } + }, + "node_modules/@posthog/core": { + "version": "1.43.1", + "resolved": "https://registry.npmjs.org/@posthog/core/-/core-1.43.1.tgz", + "integrity": "sha512-hGM8f5sp3we6Em/RQHXbmyYm554hUx9+9jhf92ZQgDS4/xW72KHyZiO9wcFye+qAx2cJAOhOCDIznmO1FV8IbA==", + "license": "MIT", + "dependencies": { + "@posthog/types": "^1.396.0" + } + }, + "node_modules/@posthog/types": { + "version": "1.397.0", + "resolved": "https://registry.npmjs.org/@posthog/types/-/types-1.397.0.tgz", + "integrity": "sha512-Pa7FtsBo3V0XrhY4y8Xquwp8U07syuZ2IGQdlsRyxhz0yejQLceFjI58UssCXE5zDCDj3km5l7H3ufD6rHChCg==", + "license": "MIT" + }, + "node_modules/@smithy/core": { + "version": "3.29.5", + "resolved": "https://registry.npmjs.org/@smithy/core/-/core-3.29.5.tgz", + "integrity": "sha512-i0dk2t5B+CwV/dcJdUHILYkOQF5lof8f44dFCfDWToGCxjT9YQ+CgHqTAvJxzc3+zqQwm2QtVoJ5IqiNar/CnQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/credential-provider-imds": { + "version": "4.4.10", + "resolved": "https://registry.npmjs.org/@smithy/credential-provider-imds/-/credential-provider-imds-4.4.10.tgz", + "integrity": "sha512-MJenAe4OKRZUo1LdYYFDCsSHxaHvInIU/z52GsheO9vl1/VSySVCr0zkyKD6TFiGkSUaWGxvKZ/70OvgUZR5HQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/fetch-http-handler": { + "version": "5.6.7", + "resolved": "https://registry.npmjs.org/@smithy/fetch-http-handler/-/fetch-http-handler-5.6.7.tgz", + "integrity": "sha512-3zpg8yqqyXzoK2TsRDdkqVOj2RDBFfLXwCczOZ5c7TWB4eiaebfSCsbMjDPYB3PJ9ihV62QaeadZ+wLadZtNGA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/node-http-handler": { + "version": "4.9.7", + "resolved": "https://registry.npmjs.org/@smithy/node-http-handler/-/node-http-handler-4.9.7.tgz", + "integrity": "sha512-wCU8HCLjAtAVqxxe0j2xff9LcEPw3yjBbg5IdQDIYFnxnPxbxcSLc7rgex7kqm9L/WYOnJEgaWQlfDkZleozMA==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/signature-v4": { + "version": "5.6.6", + "resolved": "https://registry.npmjs.org/@smithy/signature-v4/-/signature-v4-5.6.6.tgz", + "integrity": "sha512-efP6DN3UTFrzIsGO42/xcabv8jU7+9nwEdphFUH7yL0k010ERyAWaO41KFQIDLcFZLZ8xzIQr4wplFxNzslSGQ==", + "license": "Apache-2.0", + "dependencies": { + "@smithy/core": "^3.29.5", + "@smithy/types": "^4.16.1", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@smithy/types": { + "version": "4.16.1", + "resolved": "https://registry.npmjs.org/@smithy/types/-/types-4.16.1.tgz", + "integrity": "sha512-0JFs3V2y2M9tKW5na/qxe69Zv+uxLMO7QBbhxF/FHu/Gp2NFZAAL9tWl9PU02xxo07pb3G9FTyjNc6D5uZrJIg==", + "license": "Apache-2.0", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@stablelib/base64": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz", + "integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==", + "license": "MIT" + }, + "node_modules/@standard-schema/spec": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@standard-schema/spec/-/spec-1.1.0.tgz", + "integrity": "sha512-l2aFy5jALhniG5HgqrD6jXLi/rUWrKvqN/qJx6yoJsgKhblVd+iqqU4RCXavm/jPityDo5TCvKMnpjKnOriy0w==", + "license": "MIT" + }, + "node_modules/@types/json-schema": { + "version": "7.0.15", + "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz", + "integrity": "sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==", + "license": "MIT" + }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, + "node_modules/ansi-escapes": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/ansi-escapes/-/ansi-escapes-7.3.0.tgz", + "integrity": "sha512-BvU8nYgGQBxcmMuEeUEmNTvrMVjJNSH7RgW24vXexN4Ven6qCvy4TntnvlnwnMLTVlcRQQdbRY8NKnaIoeWDNg==", + "license": "MIT", + "dependencies": { + "environment": "^1.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ansi-regex": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.2.2.tgz", + "integrity": "sha512-Bq3SmSpyFHaWjPk8If9yc6svM8c56dB5BAtW4Qbw5jHTwwXXcTLoRMkpDJp6VL0XzlWaCHTXrkFURMYmD0sLqg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ansi-styles": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.3.tgz", + "integrity": "sha512-4Dj6M28JB+oAH8kFkTLUo+a2jwOFkuqb3yucU0CANcRRUbxS0cP0nZYCGjcc3BNXwRIsUVmDGgzawme7zvJHvg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/auto-bind": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/auto-bind/-/auto-bind-5.0.1.tgz", + "integrity": "sha512-ooviqdwwgfIfNmDwo94wlshcdzfO64XV0Cg6oDsDYBJfITDz1EngD2z7DkbvCWn+XIMsIqW27sEVF6qcpJrRcg==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", + "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "12.11.1", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.11.1.tgz", + "integrity": "sha512-dq9AtApgg5PGFtBzPFSBl3HZQjHok5gaQCM6zh2Yk0aSmDCs1CbnVI8/HgASQkNKsWFpseIO9beg5xxpYhbIfA==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + }, + "engines": { + "node": "20.x || 22.x || 23.x || 24.x || 25.x || 26.x" + } + }, + "node_modules/bignumber.js": { + "version": "9.3.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz", + "integrity": "sha512-Ko0uX15oIUS7wJ3Rb30Fs6SkVbLmPBAKdlm7q9+ak9bbIeFf0MwuBsQV6z7+X768/cHsfg+WlysDWJcmthjsjQ==", + "license": "MIT", + "engines": { + "node": "*" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", + "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/bowser": { + "version": "2.14.1", + "resolved": "https://registry.npmjs.org/bowser/-/bowser-2.14.1.tgz", + "integrity": "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg==", + "license": "MIT" + }, + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", + "license": "MIT", + "dependencies": { + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, + "node_modules/chalk": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.6.2.tgz", + "integrity": "sha512-7NzBL0rN6fMUW+f7A6Io4h40qQlG+xGmtMxfbnH/K7TAtt8JQWVQK+6g0UXKMeVJoyV5EkkNsErQ8pVD3bLHbA==", + "license": "MIT", + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", + "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "license": "ISC" + }, + "node_modules/ci-info": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/ci-info/-/ci-info-4.4.0.tgz", + "integrity": "sha512-77PSwercCZU2Fc4sX94eF8k8Pxte6JAwL4/ICZLFjJLqegs7kCuAsqqj/70NQF6TvDpgFjkubQB2FW2ZZddvQg==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/sibiraj-s" + } + ], + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/cli-boxes": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/cli-boxes/-/cli-boxes-3.0.0.tgz", + "integrity": "sha512-/lzGpEWL/8PfI0BmBOPRwp0c/wFNX1RdUML3jK/RcSBA9T8mZDdQpqYBKtCFTOfQbwPqWEOpjqW+Fnayc0969g==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-cursor": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-4.0.0.tgz", + "integrity": "sha512-VGtlMu3x/4DOtIUwEkRezxUZ2lBacNJCHash0N0WeZDBS+7Ux1dm3XWAgWYxLJFMMdOeXMHXorshEFhbMSGelg==", + "license": "MIT", + "dependencies": { + "restore-cursor": "^4.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-truncate": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-4.0.0.tgz", + "integrity": "sha512-nPdaFdQ0h/GEigbPClz11D0v/ZJEwxmeVZGeMo3Z5StPtUTkA9o1lD6QwoirYiSDzbcwn2XcjwmCp68W1IS4TA==", + "license": "MIT", + "dependencies": { + "slice-ansi": "^5.0.0", + "string-width": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/cli-truncate/node_modules/slice-ansi": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-5.0.0.tgz", + "integrity": "sha512-FC+lgizVPfie0kkhqUScwRu1O/lF6NOgJmlCgK+/LYxDCTk8sGelYaHDhFcDN+Sn3Cv+3VSa4Byeo+IMCzpMgQ==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.0.0", + "is-fullwidth-code-point": "^4.0.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/code-excerpt": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/code-excerpt/-/code-excerpt-4.0.0.tgz", + "integrity": "sha512-xxodCmBen3iy2i0WtAK8FlFNrRzjUqjRsMfho58xT/wvZU1YTM3fCnRjcy1gJPMepaRlgm/0e6w8SpWHpn3/cA==", + "license": "MIT", + "dependencies": { + "convert-to-spaces": "^2.0.1" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/convert-to-spaces": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/convert-to-spaces/-/convert-to-spaces-2.0.1.tgz", + "integrity": "sha512-rcQ1bsQO9799wq24uE5AM2tAILy4gXGIK/njFWcVQkGNZ96edlpY+A7bjwvzjYvLDyzmG1MmMLZhpcsb+klNMQ==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/cron-parser": { + "version": "5.6.1", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-5.6.1.tgz", + "integrity": "sha512-QBm4o1PwZiuY7KFbVvW7FLC8bozy7YWzv+Fz6KRS7sQghzcbDZCGxr/Bc5b6TQreAoSwuWVP491dIcK0THCX6A==", + "license": "MIT", + "dependencies": { + "luxon": "^3.7.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/cronstrue": { + "version": "3.24.0", + "resolved": "https://registry.npmjs.org/cronstrue/-/cronstrue-3.24.0.tgz", + "integrity": "sha512-t/Ji3Ur2c/pzhIAWNwC0ftl3JAE4dLfCjAdZoTZXmPDZwcispnS1PaMcMS4OmIIXyIVouAz+yw+mfQiE3hz5OQ==", + "license": "MIT", + "bin": { + "cronstrue": "bin/cli.js" + } + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", + "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", + "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/deepagents": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/deepagents/-/deepagents-1.11.1.tgz", + "integrity": "sha512-qBdQ+rmQteHsGgDjqubbBEaY6qcCyQdhKHYKs3f4G7FMnfGMWo/OvbzGncDGIjQLqSl1pF7tD/XkQmfVCU/fhQ==", + "license": "MIT", + "dependencies": { + "fast-glob": "^3.3.3", + "micromatch": "^4.0.8", + "yaml": "^2.8.2", + "zod": "^4.3.6" + }, + "peerDependencies": { + "@langchain/core": "^1.2.0", + "@langchain/langgraph": "^1.4.4", + "@langchain/langgraph-checkpoint": "^1.1.2", + "@langchain/langgraph-sdk": "^1.9.23", + "langchain": "^1.5.0", + "langsmith": "^0.7.1" + } + }, + "node_modules/detect-libc": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz", + "integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, + "node_modules/emoji-regex": { + "version": "10.6.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.6.0.tgz", + "integrity": "sha512-toUI84YS5YmxW219erniWD0CIVOo46xGKColeNQRgOzDorgBi1v4D71/OFzgD9GO2UGKIv1C3Sp8DAn0+j5w7A==", + "license": "MIT" + }, + "node_modules/end-of-stream": { + "version": "1.4.5", + "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", + "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/environment": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/environment/-/environment-1.1.0.tgz", + "integrity": "sha512-xUtoPkMggbz0MPyPiIWr1Kp4aeWJjDZ6SMvURhimjdZgsRuDplF5/s9hcgGhyXMhs+6vpnuoiZ2kFiu3FMnS8Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/es-toolkit": { + "version": "1.49.0", + "resolved": "https://registry.npmjs.org/es-toolkit/-/es-toolkit-1.49.0.tgz", + "integrity": "sha512-G5iZ6Pc/FNRY/soKZHC+TxGDD83rHUDXxzaWhGCX44vAv/tMs56WMusnm/KMNK+luUPsgA9U28cGr4RDlSzL2g==", + "license": "MIT", + "workspaces": [ + "docs", + "benchmarks" + ] + }, + "node_modules/escape-string-regexp": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-2.0.0.tgz", + "integrity": "sha512-UpzcLCXolUWcNu5HtVMHYdXJjArjsF9C0aNnquZYY4uW/Vu0miy5YoWvbV345HauVvcAUnpRuhMMcqTcGOY2+w==", + "license": "MIT", + "engines": { + "node": ">=8" + } + }, + "node_modules/eventemitter3": { + "version": "4.0.7", + "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-4.0.7.tgz", + "integrity": "sha512-8guHBZCwKnFhYdHr2ysuRWErTwhoN2X8XELRlrRwpmfeY2jjuUN4taQMsULKUVo1K4DvZl+0pgfyoysHxvmvEw==", + "license": "MIT" + }, + "node_modules/eventsource-parser": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/eventsource-parser/-/eventsource-parser-3.1.0.tgz", + "integrity": "sha512-kJezFj9YFAMLeORyi7aCLxLbD5/qWMQnoMVlVPyHIll7lgRJCc3JVln9Vgl9nwQi0YkMnhdGTMNn7CkRRAptMg==", + "license": "MIT", + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", + "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==", + "license": "MIT" + }, + "node_modules/fast-glob": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/fast-glob/-/fast-glob-3.3.3.tgz", + "integrity": "sha512-7MptL8U0cqcFdzIzwOTHoilX9x5BrNqye7Z/LuC7kCMRio1EMSyqRK3BEAUD7sXRq4iT4AzTVuZdhgQ2TCvYLg==", + "license": "MIT", + "dependencies": { + "@nodelib/fs.stat": "^2.0.2", + "@nodelib/fs.walk": "^1.2.3", + "glob-parent": "^5.1.2", + "merge2": "^1.3.0", + "micromatch": "^4.0.8" + }, + "engines": { + "node": ">=8.6.0" + } + }, + "node_modules/fast-sha256": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz", + "integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==", + "license": "Unlicense" + }, + "node_modules/fastq": { + "version": "1.20.1", + "resolved": "https://registry.npmjs.org/fastq/-/fastq-1.20.1.tgz", + "integrity": "sha512-GGToxJ/w1x32s/D2EKND7kTil4n8OVk/9mycTc4VDza13lOvpUZTGX3mFSCtV9ksdGBVzvsyAVLM6mHFThxXxw==", + "license": "ISC", + "dependencies": { + "reusify": "^1.0.4" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", + "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", + "license": "MIT" + }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", + "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "license": "MIT" + }, + "node_modules/gaxios": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-7.2.0.tgz", + "integrity": "sha512-CUVb4wcYe+771XevyH6HtGmXFAGGKkIC3kswAP8Z1JCe0j80JMaTPZH930DWFrvo0atjh18Arc0pEyUCWa5bfg==", + "license": "Apache-2.0", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^7.0.1", + "node-fetch": "^3.3.2" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/gcp-metadata": { + "version": "8.1.2", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-8.1.2.tgz", + "integrity": "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg==", + "license": "Apache-2.0", + "dependencies": { + "gaxios": "^7.0.0", + "google-logging-utils": "^1.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/get-east-asian-width": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/get-east-asian-width/-/get-east-asian-width-1.6.0.tgz", + "integrity": "sha512-QRbvDIbx6YklUe6RxeTeleMR0yv3cYH6PsPZHcnVn7xv7zO1BHN8r0XETu8n6Ye3Q+ahtSarc3WgtNWmehIBfA==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", + "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "license": "MIT" + }, + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", + "license": "ISC", + "dependencies": { + "is-glob": "^4.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/google-auth-library": { + "version": "10.9.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-10.9.0.tgz", + "integrity": "sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg==", + "license": "Apache-2.0", + "dependencies": { + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "gaxios": "^7.1.4", + "gcp-metadata": "8.1.2", + "google-logging-utils": "1.1.3", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/google-logging-utils": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", + "integrity": "sha512-eAmLkjDjAFCVXg7A1unxHsLf961m6y17QFqXqAXGj/gVkKFrEICfStRfwUlGNfeCEjNRa32JEWOUTlYXPyyKvA==", + "license": "Apache-2.0", + "engines": { + "node": ">=14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/ieee754": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", + "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/indent-string": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/indent-string/-/indent-string-5.0.0.tgz", + "integrity": "sha512-m6FAo/spmsW2Ab2fU35JTYwtOKa2yAwXSwgjSv1TJzh4Mh7mC3lzAOVLBprb72XsTrgkEIsl7YrFNAiDiRhIGg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", + "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "license": "ISC" + }, + "node_modules/ink": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/ink/-/ink-5.2.1.tgz", + "integrity": "sha512-BqcUyWrG9zq5HIwW6JcfFHsIYebJkWWb4fczNah1goUO0vv5vneIlfwuS85twyJ5hYR/y18FlAYUxrO9ChIWVg==", + "license": "MIT", + "dependencies": { + "@alcalzone/ansi-tokenize": "^0.1.3", + "ansi-escapes": "^7.0.0", + "ansi-styles": "^6.2.1", + "auto-bind": "^5.0.1", + "chalk": "^5.3.0", + "cli-boxes": "^3.0.0", + "cli-cursor": "^4.0.0", + "cli-truncate": "^4.0.0", + "code-excerpt": "^4.0.0", + "es-toolkit": "^1.22.0", + "indent-string": "^5.0.0", + "is-in-ci": "^1.0.0", + "patch-console": "^2.0.0", + "react-reconciler": "^0.29.0", + "scheduler": "^0.23.0", + "signal-exit": "^3.0.7", + "slice-ansi": "^7.1.0", + "stack-utils": "^2.0.6", + "string-width": "^7.2.0", + "type-fest": "^4.27.0", + "widest-line": "^5.0.0", + "wrap-ansi": "^9.0.0", + "ws": "^8.18.0", + "yoga-layout": "~3.2.1" + }, + "engines": { + "node": ">=18" + }, + "peerDependencies": { + "@types/react": ">=18.0.0", + "react": ">=18.0.0", + "react-devtools-core": "^4.19.1" + }, + "peerDependenciesMeta": { + "@types/react": { + "optional": true + }, + "react-devtools-core": { + "optional": true + } + } + }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-4.0.0.tgz", + "integrity": "sha512-O4L094N2/dZ7xqVdrXhh9r1KODPJpFms8B5sGdJLPy664AgvXsreZUyCQQNItZRDlYug4xStLjNp/sz3HvBowQ==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-in-ci": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-in-ci/-/is-in-ci-1.0.0.tgz", + "integrity": "sha512-eUuAjybVTHMYWm/U+vBO1sY/JOCgoPCXRxzdju0K+K0BiGW0SChEL1MLC0PoCIR1OlPo5YAp8HuQoUlsWEICwg==", + "license": "MIT", + "bin": { + "is-in-ci": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-network-error": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/is-network-error/-/is-network-error-1.3.2.tgz", + "integrity": "sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==", + "license": "MIT", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, + "node_modules/jose": { + "version": "6.2.3", + "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.3.tgz", + "integrity": "sha512-YYVDInQKFJfR/xa3ojUTl8c2KoTwiL1R5Wg9YCydwH0x0B9grbzlg5HC7mMjCtUJjbQ/YnGEZIhI5tCgfTb4Hw==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/panva" + } + }, + "node_modules/js-tiktoken": { + "version": "1.0.21", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.21.tgz", + "integrity": "sha512-biOj/6M5qdgx5TKjDnFT1ymSpM5tbd3ylwDtrQvFQSu0Z7bBYko2dF+W/aUkXUPuk6IVpRxk/3Q2sHOzGlS36g==", + "license": "MIT", + "dependencies": { + "base64-js": "^1.5.1" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "license": "MIT", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, + "node_modules/json-schema-to-ts": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz", + "integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==", + "license": "MIT", + "dependencies": { + "@babel/runtime": "^7.18.3", + "ts-algebra": "^2.0.0" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/langchain": { + "version": "1.5.3", + "resolved": "https://registry.npmjs.org/langchain/-/langchain-1.5.3.tgz", + "integrity": "sha512-YRYPy1xPq4CgKe0NdPoscYHnKBW5cViEyWiZsUKQfN3xL1X5DpGlIpaBQg1A/t6o9ePKE7J5b7nqrfLz8t7/Rw==", + "license": "MIT", + "dependencies": { + "@langchain/langgraph": "^1.4.7", + "@langchain/langgraph-checkpoint": "^1.1.3", + "langsmith": ">=0.5.0 <1.0.0", + "zod": "^3.25.76 || ^4" + }, + "engines": { + "node": ">=20" + }, + "peerDependencies": { + "@langchain/core": "^1.2.1" + } + }, + "node_modules/langsmith": { + "version": "0.7.17", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.7.17.tgz", + "integrity": "sha512-FfmxKOVp+qBkdbOWTwXxU7qeX2H40RMsLW9Us+53/4rZYl/QzfUeeBe/rE98mfuSSnEz1tOOMjoPJIYoMt1wnw==", + "license": "MIT", + "dependencies": { + "p-queue": "6.6.2" + }, + "peerDependencies": { + "@opentelemetry/api": "*", + "@opentelemetry/exporter-trace-otlp-proto": "*", + "@opentelemetry/sdk-trace-base": "*", + "openai": "*", + "ws": ">=7" + }, + "peerDependenciesMeta": { + "@opentelemetry/api": { + "optional": true + }, + "@opentelemetry/exporter-trace-otlp-proto": { + "optional": true + }, + "@opentelemetry/sdk-trace-base": { + "optional": true + }, + "openai": { + "optional": true + }, + "ws": { + "optional": true + } + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "license": "MIT", + "engines": { + "node": ">=12" + } + }, + "node_modules/marked": { + "version": "18.0.6", + "resolved": "https://registry.npmjs.org/marked/-/marked-18.0.6.tgz", + "integrity": "sha512-MrV5puXBfuiy6wl6DLaq3BtIJQAJToAd5zt/ZKhRfGRAuFPALE7/4Y7jnxRQoEgK/pBgurGqLyAuRgZ2xOjr6w==", + "license": "MIT", + "bin": { + "marked": "bin/marked.js" + }, + "engines": { + "node": ">= 20" + } + }, + "node_modules/merge2": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", + "integrity": "sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/micromatch": { + "version": "4.0.8", + "resolved": "https://registry.npmjs.org/micromatch/-/micromatch-4.0.8.tgz", + "integrity": "sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==", + "license": "MIT", + "dependencies": { + "braces": "^3.0.3", + "picomatch": "^2.3.1" + }, + "engines": { + "node": ">=8.6" + } + }, + "node_modules/mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", + "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", + "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "license": "MIT" + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "license": "MIT", + "bin": { + "mustache": "bin/mustache" + } + }, + "node_modules/napi-build-utils": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", + "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "license": "MIT" + }, + "node_modules/node-abi": { + "version": "3.94.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.94.0.tgz", + "integrity": "sha512-W5ZNO5KRPB5TkYmGVD9F6YqhsglXJzE6etpbmT+f6EQElhiX/UTG551cnsRGvLG3fyZEg9HwaDmNmj5nwJ4z9g==", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "deprecated": "Use your platform's native DOMException instead", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/once": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", + "integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "license": "MIT", + "dependencies": { + "mimic-fn": "^2.1.0" + }, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/openai": { + "version": "6.48.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-6.48.0.tgz", + "integrity": "sha512-KhVp+FyV50QrXNextvL9hIU5l6ox5HYuKQjGVk7lIqprgJol90+dQXWONV6S1lRWsKA1bXjrow8RsUT14M1hNA==", + "license": "Apache-2.0", + "peerDependencies": { + "@aws-sdk/credential-provider-node": ">=3.972.0 <4", + "@smithy/hash-node": ">=4.3.0 <5", + "@smithy/signature-v4": ">=5.4.0 <6", + "ws": "^8.18.0", + "zod": "^3.25 || ^4.0" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-provider-node": { + "optional": true + }, + "@smithy/hash-node": { + "optional": true + }, + "@smithy/signature-v4": { + "optional": true + }, + "ws": { + "optional": true + }, + "zod": { + "optional": true + } + } + }, + "node_modules/openwiki": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/openwiki/-/openwiki-0.2.0.tgz", + "integrity": "sha512-hLop7FDz4zwj7z5VCdXhyY0yJxYVOKtVrBZJj1cSkiMN8nbr1ywm9F6gDxP59kWkuaCs39DCU9QpyzxL7grxnw==", + "license": "MIT", + "dependencies": { + "@anthropic-ai/vertex-sdk": "^0.19.0", + "@aws-sdk/client-bedrock-runtime": "^3.1080.0", + "@langchain/anthropic": "^1.5.1", + "@langchain/aws": "^1.4.2", + "@langchain/core": "^1.2.1", + "@langchain/google": "^0.2.1", + "@langchain/langgraph-checkpoint-sqlite": "^1.0.3", + "@langchain/openai": "^1.5.5", + "@langchain/openrouter": "^0.4.3", + "@langchain/protocol": "^0.0.18", + "@langchain/tavily": "1.2.0", + "ci-info": "^4.4.0", + "cron-parser": "5.6.1", + "cronstrue": "3.24.0", + "deepagents": "^1.11.0", + "google-auth-library": "^10.9.0", + "ink": "^5.1.0", + "langchain": "^1.5.3", + "marked": "^18.0.5", + "posthog-node": "^5.39.4", + "react": "^18.3.1", + "yaml": "^2.9.0", + "zod": "^4.4.3" + }, + "bin": { + "openwiki": "dist/cli.js" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "license": "MIT", + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-7.1.1.tgz", + "integrity": "sha512-J5ApzjyRkkf601HpEeykoiCvzHQjWxPAHhyjFcEUP2SWq0+35NKh8TLhpLw+Dkq5TZBFvUM6UigdE9hIVYTl5w==", + "license": "MIT", + "dependencies": { + "is-network-error": "^1.1.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "license": "MIT", + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/patch-console": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/patch-console/-/patch-console-2.0.0.tgz", + "integrity": "sha512-0YNdUceMdaQwoKce1gatDScmMo5pu/tfABfnzEqeG0gtTmd7mh/WcwgUjtAeOU7N8nFFlbQBnFK2gXW5fGvmMA==", + "license": "MIT", + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + } + }, + "node_modules/picomatch": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, + "node_modules/posthog-node": { + "version": "5.45.2", + "resolved": "https://registry.npmjs.org/posthog-node/-/posthog-node-5.45.2.tgz", + "integrity": "sha512-QhHw/xL0ntMG/DzqA/qdiKu8JY8ChK5Obm8m1nMJor5sjJ7+pfVJSx1yAynC7iUAJfJ9V+OhBNhDCyi5IFWC0Q==", + "license": "MIT", + "dependencies": { + "@posthog/core": "^1.43.1" + }, + "engines": { + "node": "^20.20.0 || >=22.22.0" + }, + "peerDependencies": { + "rxjs": "^7.0.0" + }, + "peerDependenciesMeta": { + "rxjs": { + "optional": true + } + } + }, + "node_modules/prebuild-install": { + "version": "7.1.3", + "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", + "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", + "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^2.0.0", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/pump": { + "version": "3.0.4", + "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", + "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/queue-microtask": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/queue-microtask/-/queue-microtask-1.2.3.tgz", + "integrity": "sha512-NuaNSa6flKT5JaSYQzJok04JzTL1CA6aGhv5rfLW3PgqA+M2ChpZQnAC8h8i4ZFkBS8X5RqkDBHA7r4hej3K9A==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/rc": { + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", + "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-reconciler": { + "version": "0.29.2", + "resolved": "https://registry.npmjs.org/react-reconciler/-/react-reconciler-0.29.2.tgz", + "integrity": "sha512-zZQqIiYgDCTP/f1N/mAR10nJGrPD2ZR+jDSEsKWJHYC7Cm2wodlwbR3upZRdC3cjIjSlTLNVyO7Iu0Yy7t2AYg==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "engines": { + "node": ">=0.10.0" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/restore-cursor": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-4.0.0.tgz", + "integrity": "sha512-I9fPXU9geO9bHOt9pHHOhOkYerIMsmVaWB0rA2AI9ERh/+x/i7MV5HKBNrg+ljO5eoPVgCcnFuRjJ9uH6I/3eg==", + "license": "MIT", + "dependencies": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/reusify": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.1.0.tgz", + "integrity": "sha512-g6QUff04oZpHs0eG5p83rFLhHeV00ug/Yf9nZM6fLeUrPguBTkTQOdpAWWspMh55TZfVQDPaN3NQJfbVRAxdIw==", + "license": "MIT", + "engines": { + "iojs": ">=1.0.0", + "node": ">=0.10.0" + } + }, + "node_modules/run-parallel": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", + "integrity": "sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "queue-microtask": "^1.2.2" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/signal-exit": { + "version": "3.0.7", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-3.0.7.tgz", + "integrity": "sha512-wnD2ZE+l+SPC/uoS0vXeE9L1+0wuaMqKlfz9AMUo38JsyLSBWSFcHR1Rri62LZc12vLr1gb3jl7iwQhgwpAbGQ==", + "license": "ISC" + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", + "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", + "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/slice-ansi": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/slice-ansi/-/slice-ansi-7.1.2.tgz", + "integrity": "sha512-iOBWFgUX7caIZiuutICxVgX1SdxwAVFFKwt1EvMYYec/NWO5meOJ6K5uQxhrYBdQJne4KxiqZc+KptFOWFSI9w==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "is-fullwidth-code-point": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/slice-ansi?sponsor=1" + } + }, + "node_modules/slice-ansi/node_modules/is-fullwidth-code-point": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-5.1.0.tgz", + "integrity": "sha512-5XHYaSyiqADb4RnZ1Bdad6cPp8Toise4TzEjcOYDHZkTCbKgiUl7WTUCpNWHuxmDt91wnsZBc9xinNzopv3JMQ==", + "license": "MIT", + "dependencies": { + "get-east-asian-width": "^1.3.1" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/stack-utils": { + "version": "2.0.6", + "resolved": "https://registry.npmjs.org/stack-utils/-/stack-utils-2.0.6.tgz", + "integrity": "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ==", + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^2.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/standardwebhooks": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz", + "integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==", + "license": "MIT", + "dependencies": { + "@stablelib/base64": "^1.0.0", + "fast-sha256": "^1.3.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/string-width": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-7.2.0.tgz", + "integrity": "sha512-tsaTIkKW9b4N+AEj+SVA+WhJzV7/zMhcSu78mLKWSk7cXMOSHsBKFWUs0fWwq8QyK3MgJBQRX6Gbi4kYbdvGkQ==", + "license": "MIT", + "dependencies": { + "emoji-regex": "^10.3.0", + "get-east-asian-width": "^1.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/strip-ansi": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.2.0.tgz", + "integrity": "sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==", + "license": "MIT", + "dependencies": { + "ansi-regex": "^6.2.2" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", + "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar-fs": { + "version": "2.1.5", + "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.5.tgz", + "integrity": "sha512-OboTd8mmMhZDNPV+UjQcK9yKAatXu2aJ+r1w4im1Otd4M4fl2hwvdoXUxIYHFTHWK/3y3FarBP70v3vwmGlOxw==", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", + "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/ts-algebra": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz", + "integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==", + "license": "MIT" + }, + "node_modules/tslib": { + "version": "2.8.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", + "license": "0BSD" + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", + "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/type-fest": { + "version": "4.41.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-4.41.0.tgz", + "integrity": "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA==", + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "license": "MIT" + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "license": "MIT", + "engines": { + "node": ">= 8" + } + }, + "node_modules/widest-line": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/widest-line/-/widest-line-5.0.0.tgz", + "integrity": "sha512-c9bZp7b5YtRj2wOe6dlj32MK+Bx/M/d+9VB2SHM1OtsUHR0aV0tdP6DWh/iMt0kWi1t5g1Iudu6hQRNd1A4PVA==", + "license": "MIT", + "dependencies": { + "string-width": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/wrap-ansi": { + "version": "9.0.2", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-9.0.2.tgz", + "integrity": "sha512-42AtmgqjV+X1VpdOfyTGOYRi0/zsoLqtXQckTmqTeybT+BDIbM/Guxo7x3pE2vtpr1ok6xRqM9OpBe+Jyoqyww==", + "license": "MIT", + "dependencies": { + "ansi-styles": "^6.2.1", + "string-width": "^7.0.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrappy": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", + "integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==", + "license": "ISC" + }, + "node_modules/ws": { + "version": "8.21.1", + "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.1.tgz", + "integrity": "sha512-+0NTnW77fFN/DjQi6k/Sq/Yvk4Sgajw7urW8V+asjXnRgDs9gyGkdb7EzgfhA4goXsRIZKE28fzIXBHEzhuiWw==", + "license": "MIT", + "engines": { + "node": ">=10.0.0" + }, + "peerDependencies": { + "bufferutil": "^4.0.1", + "utf-8-validate": ">=5.0.2" + }, + "peerDependenciesMeta": { + "bufferutil": { + "optional": true + }, + "utf-8-validate": { + "optional": true + } + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + }, + "node_modules/yoga-layout": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/yoga-layout/-/yoga-layout-3.2.1.tgz", + "integrity": "sha512-0LPOt3AxKqMdFBZA3HBAt/t/8vIKq7VaQYbuA8WxCgung+p9TVyKRYdpvCb80HcdTN2NkbIKbhNwKUfm3tQywQ==", + "license": "MIT" + }, + "node_modules/zod": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/zod/-/zod-4.4.3.tgz", + "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/colinhacks" + } + } + } +} diff --git a/tools/openwiki/package.json b/tools/openwiki/package.json new file mode 100644 index 0000000..59ed5b2 --- /dev/null +++ b/tools/openwiki/package.json @@ -0,0 +1,11 @@ +{ + "name": "nugs-openwiki-tooling", + "private": true, + "version": "0.0.0", + "engines": { + "node": "22.x" + }, + "dependencies": { + "openwiki": "0.2.0" + } +} From 4fe0b7198b1acb6a8c9f0efa3c0e0cf4829b9fcc Mon Sep 17 00:00:00 2001 From: Jacob Magar Date: Sat, 18 Jul 2026 02:46:51 -0400 Subject: [PATCH 2/3] fix: address all PR review findings --- .github/workflows/ci.yml | 2 +- .github/workflows/openwiki-update.yml | 10 +- .github/workflows/release.yml | 4 +- .mise.toml | 2 +- CLAUDE.md | 2 +- Makefile | 1 + cmd/nugs/main.go | 37 ++--- cmd/nugs/main_flow_test.go | 22 +++ cmd/nugs/output.go | 5 +- cmd/nugs/rclone.go | 8 +- cmd/nugs/version_test.go | 26 +++- cmd/nugs/watch.go | 7 +- internal/api/apilog.go | 38 ++++- internal/api/apilog_test.go | 36 +++++ internal/api/circuit_test.go | 72 +++++++++ internal/api/client.go | 14 +- internal/cache/atomic.go | 6 + internal/cache/atomic_sync_unix.go | 16 ++ internal/cache/atomic_sync_windows.go | 9 ++ internal/cache/atomic_test.go | 28 ++++ internal/cache/cache.go | 86 +++++++---- internal/cache/filelock_unix.go | 2 + internal/cache/filelock_windows.go | 2 + internal/cache/full_catalog.go | 158 ++++++++++++++------ internal/cache/regression_test.go | 2 +- internal/cache/remediation_test.go | 113 +++++++++++++- internal/catalog/cache_status_test.go | 45 ++++++ internal/catalog/handlers.go | 3 +- internal/catalog/media_filter.go | 21 ++- internal/catalog/request_budget_test.go | 53 +++++++ internal/catalog/watch.go | 17 ++- internal/catalog/watch_systemd.go | 190 +++++++++++++++++++----- internal/catalog/watch_systemd_test.go | 157 ++++++++++++++++++++ internal/catalog/watch_test.go | 31 +--- internal/config/atomic_test.go | 6 + internal/config/config.go | 8 +- internal/config/sync_dir_unix.go | 16 ++ internal/config/sync_dir_windows.go | 9 ++ internal/download/audio.go | 22 +-- internal/download/audio_album_test.go | 62 ++++++++ internal/download/batch.go | 5 +- internal/download/regression_test.go | 18 +-- internal/download/video.go | 16 -- internal/helpers/paths.go | 9 +- internal/helpers/paths_security_test.go | 23 +++ internal/notify/gotify.go | 7 +- internal/notify/gotify_security_test.go | 26 ++++ internal/rclone/concurrency_test.go | 35 ++++- internal/rclone/rclone.go | 26 ++-- internal/rclone/storage_adapter.go | 30 +++- internal/rclone/storage_adapter_test.go | 10 +- internal/runtime/detach_unix.go | 14 +- internal/runtime/detach_unix_test.go | 30 ++++ scripts/check-docs.py | 35 ++++- scripts/test_check_docs.py | 52 +++++++ scripts/test_repository_contracts.py | 37 +++++ 56 files changed, 1424 insertions(+), 297 deletions(-) create mode 100644 internal/cache/atomic_sync_unix.go create mode 100644 internal/cache/atomic_sync_windows.go create mode 100644 internal/cache/atomic_test.go create mode 100644 internal/catalog/cache_status_test.go create mode 100644 internal/catalog/watch_systemd_test.go create mode 100644 internal/config/sync_dir_unix.go create mode 100644 internal/config/sync_dir_windows.go create mode 100644 internal/runtime/detach_unix_test.go create mode 100644 scripts/test_check_docs.py create mode 100644 scripts/test_repository_contracts.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5ca3c53..56e5afb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,7 @@ jobs: - name: Race tests run: go test -race -count=1 ./... - name: Documentation contracts - run: python3 scripts/check-docs.py + run: make docs-check - name: Static analysis run: go run honnef.co/go/tools/cmd/staticcheck@2025.1.1 ./... - name: Vulnerability analysis diff --git a/.github/workflows/openwiki-update.yml b/.github/workflows/openwiki-update.yml index e393b55..292ff6b 100644 --- a/.github/workflows/openwiki-update.yml +++ b/.github/workflows/openwiki-update.yml @@ -49,7 +49,10 @@ jobs: OPENAI_COMPATIBLE_BASE_URL: http://100.120.242.29:8317/v1 run: | set -euo pipefail - test -n "${OPENAI_COMPATIBLE_API_KEY:-}" + if [[ -z "${OPENAI_COMPATIBLE_API_KEY:-}" ]]; then + echo "::error title=Missing OpenWiki API key::Configure the OPENAI_COMPATIBLE_API_KEY repository secret" + exit 1 + fi response="$(mktemp)" trap 'rm -f "$response"' EXIT status="$(curl --silent --show-error --location --max-time 30 --connect-timeout 10 \ @@ -97,6 +100,9 @@ jobs: with: persist-credentials: false + - name: Clear previously generated documentation + run: rm -rf -- openwiki + - name: Download generated documentation uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 with: @@ -112,7 +118,7 @@ jobs: git config user.email 41898282+github-actions[bot]@users.noreply.github.com git fetch origin '+refs/heads/openwiki/update:refs/remotes/origin/openwiki/update' || true git switch -C openwiki/update - git add -- openwiki + git add -A -- openwiki if git diff --cached --quiet; then echo "No OpenWiki changes" exit 0 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b003e65..3eb44aa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,11 +27,11 @@ jobs: binary: nugs smoke: true - target: linux-arm64 - runner: ubuntu-latest + runner: ubuntu-24.04-arm goos: linux goarch: arm64 binary: nugs - smoke: false + smoke: true - target: darwin-amd64 runner: macos-15-intel goos: darwin diff --git a/.mise.toml b/.mise.toml index 50239b9..5652b91 100644 --- a/.mise.toml +++ b/.mise.toml @@ -1,3 +1,3 @@ [tools] # Exact Go follows go.mod's go directive. Do not change mise alone. -go = "1.24.12" +go = "1.25.12" diff --git a/CLAUDE.md b/CLAUDE.md index 0a9575d..92756f1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -18,7 +18,7 @@ This is the main development guide for contributors. For specialized topics, see - [Download Flow](#download-flow-critical-path) - [Testing](#testing) - [Common Gotchas](#common-gotchas) -- [Recent Improvements](#recent-improvements) +- [Current Features](#current-features) - [Troubleshooting](#troubleshooting) --- diff --git a/Makefile b/Makefile index dcb3bbb..2bd01eb 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,7 @@ test: @go test -race -count=1 ./... docs-check: + @python3 -m unittest discover -s scripts -p 'test_*.py' @python3 scripts/check-docs.py fmt-check: diff --git a/cmd/nugs/main.go b/cmd/nugs/main.go index cdfa2b8..891e5fa 100644 --- a/cmd/nugs/main.go +++ b/cmd/nugs/main.go @@ -194,12 +194,12 @@ func run(cfg *Config, jsonLevel string) (runErr error) { // Check if rclone is available when enabled if cfg.RcloneEnabled { - err = checkRcloneAvailable(jsonLevel != "") + err = checkRcloneAvailable(ctx, jsonLevel != "") if err != nil { return fmt.Errorf("rclone check failed: %w", err) } } - printStartupEnvironment(cfg, jsonLevel) + printStartupEnvironment(ctx, cfg, jsonLevel) // Show welcome screen if no arguments provided if len(cfg.Urls) == 0 { @@ -330,10 +330,7 @@ func handleListCommand(ctx context.Context, cfg *Config, jsonLevel string) (bool // Check for venue filter: list shows "venue" if len(remainingArgs) > 0 && remainingArgs[0] == "shows" { if len(remainingArgs) < 2 { - printInfo("Usage: nugs list shows \"\"") - fmt.Println("Or: list shows \"\"") - fmt.Println("Example: list 461 \"Red Rocks\"") - return true, nil + return true, errors.New("list shows requires a venue name: nugs list shows \"\"") } venueFilter := strings.Join(remainingArgs[1:], " ") err := listArtistShowsByVenue(ctx, artistId, venueFilter, jsonLevel) @@ -344,13 +341,11 @@ func handleListCommand(ctx context.Context, cfg *Config, jsonLevel string) (bool if len(remainingArgs) > 0 && remainingArgs[0] == "latest" { limit := 10 if len(remainingArgs) > 1 { - if parsedLimit, parseErr := strconv.Atoi(remainingArgs[1]); parseErr == nil { - if parsedLimit < 1 { - fmt.Println("Error: limit must be a positive number (got", parsedLimit, ")") - return true, nil - } - limit = parsedLimit + parsedLimit, parseErr := strconv.Atoi(remainingArgs[1]) + if parseErr != nil || parsedLimit < 1 { + return true, fmt.Errorf("list latest limit must be a positive integer: %q", remainingArgs[1]) } + limit = parsedLimit } err := listArtistLatestShows(ctx, artistId, limit, jsonLevel) return true, wrapCommandError("list latest shows", err) @@ -433,13 +428,11 @@ func handleCatalogCommand(ctx context.Context, cfg *Config, jsonLevel string) (b // Parse limit from remaining args if len(remainingArgs) > 0 { - if parsedLimit, err := strconv.Atoi(remainingArgs[0]); err == nil { - if parsedLimit < 1 { - fmt.Println("Error: limit must be a positive number (got", parsedLimit, ")") - return true, nil - } - limit = parsedLimit + parsedLimit, parseErr := strconv.Atoi(remainingArgs[0]) + if parseErr != nil || parsedLimit < 1 { + return true, fmt.Errorf("catalog latest limit must be a positive integer: %q", remainingArgs[0]) } + limit = parsedLimit } return true, wrapCommandError("catalog latest", catalogLatest(ctx, limit, jsonLevel)) case "gaps": @@ -465,8 +458,7 @@ func handleCatalogCommand(ctx context.Context, cfg *Config, jsonLevel string) (b } if len(artistIds) == 0 { - fmt.Println("Error: No artist IDs provided") - return true, nil + return true, errors.New("catalog gaps requires at least one artist ID") } return true, wrapCommandError("catalog gaps", catalogGaps(ctx, artistIds, cfg, jsonLevel, idsOnly, mediaFilter)) case "list": @@ -504,12 +496,11 @@ func handleCatalogCommand(ctx context.Context, cfg *Config, jsonLevel string) (b case "set": return true, wrapCommandError("configure auto-refresh", configureAutoRefresh(cfg)) default: - fmt.Printf("Unknown config action: %s\n", action) + return true, fmt.Errorf("unknown catalog config action: %s", action) } default: - fmt.Printf("Unknown catalog command: %s\n", subCmd) + return true, fmt.Errorf("unknown catalog command: %s", subCmd) } - return true, nil } // handleArtistShorthand handles " latest/full [media]" shortcuts. diff --git a/cmd/nugs/main_flow_test.go b/cmd/nugs/main_flow_test.go index b0bc2c7..ba08ab1 100644 --- a/cmd/nugs/main_flow_test.go +++ b/cmd/nugs/main_flow_test.go @@ -66,3 +66,25 @@ func TestArtistShorthandReturnsErrorInsteadOfExiting(t *testing.T) { t.Fatalf("handleArtistShorthand() = (%v, %v), want handled error", handled, err) } } + +func TestCommandValidationReturnsErrors(t *testing.T) { + tests := []struct { + name string + urls []string + call func(*Config) (bool, error) + }{ + {name: "invalid list limit", urls: []string{"list", "1125", "latest", "0"}, call: func(cfg *Config) (bool, error) { return handleListCommand(context.Background(), cfg, "") }}, + {name: "invalid catalog limit", urls: []string{"catalog", "latest", "0"}, call: func(cfg *Config) (bool, error) { return handleCatalogCommand(context.Background(), cfg, "") }}, + {name: "missing gaps artist", urls: []string{"catalog", "gaps", "--ids-only"}, call: func(cfg *Config) (bool, error) { return handleCatalogCommand(context.Background(), cfg, "") }}, + {name: "unknown catalog command", urls: []string{"catalog", "bogus"}, call: func(cfg *Config) (bool, error) { return handleCatalogCommand(context.Background(), cfg, "") }}, + {name: "missing watch add artist", urls: []string{"watch", "add"}, call: func(cfg *Config) (bool, error) { return handleWatchCommand(context.Background(), cfg, "") }}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + handled, err := tc.call(&Config{Urls: tc.urls}) + if !handled || err == nil { + t.Fatalf("command = handled %v error %v, want handled error", handled, err) + } + }) + } +} diff --git a/cmd/nugs/output.go b/cmd/nugs/output.go index 0cb03fb..55dd1d6 100644 --- a/cmd/nugs/output.go +++ b/cmd/nugs/output.go @@ -3,6 +3,7 @@ package main // Command adapters for terminal output. import ( + "context" "fmt" "strings" @@ -18,7 +19,7 @@ func describeAudioFormat(format int) string { return ui.DescribeAudioFormat func describeVideoFormat(videoFormat int) string { return ui.DescribeVideoFormat(videoFormat) } func describeAuthStatus(cfg *Config) string { return ui.DescribeAuthStatus(cfg) } -func printStartupEnvironment(cfg *Config, jsonLevel string) { +func printStartupEnvironment(ctx context.Context, cfg *Config, jsonLevel string) { if jsonLevel != "" { return } @@ -43,7 +44,7 @@ func printStartupEnvironment(cfg *Config, jsonLevel string) { printKeyValue("Rclone Audio Path", rcloneAudioPath, colorCyan) printKeyValue("Rclone Video Path", rcloneVideoPath, colorCyan) if cfg.RcloneEnabled { - printKeyValue("Rclone Status", checkRclonePathOnline(cfg), colorYellow) + printKeyValue("Rclone Status", checkRclonePathOnline(ctx, cfg), colorYellow) } else { printKeyValue("Rclone Status", "Disabled", colorYellow) } diff --git a/cmd/nugs/rclone.go b/cmd/nugs/rclone.go index f8caa62..e3f998b 100644 --- a/cmd/nugs/rclone.go +++ b/cmd/nugs/rclone.go @@ -18,8 +18,12 @@ const ( uploadCompleteVisibilityDelay = 500 * time.Millisecond ) -func checkRcloneAvailable(quiet bool) error { return rclone.CheckRcloneAvailable(quiet) } -func checkRclonePathOnline(cfg *Config) string { return rclone.CheckRclonePathOnline(cfg) } +func checkRcloneAvailable(ctx context.Context, quiet bool) error { + return rclone.CheckRcloneAvailable(ctx, quiet) +} +func checkRclonePathOnline(ctx context.Context, cfg *Config) string { + return rclone.CheckRclonePathOnline(ctx, cfg) +} func uploadToRclone(ctx context.Context, localPath string, artistFolder string, cfg *Config, progressBox *ProgressBoxState, isVideo bool) error { if progressBox != nil { diff --git a/cmd/nugs/version_test.go b/cmd/nugs/version_test.go index 61dc8bb..c063b44 100644 --- a/cmd/nugs/version_test.go +++ b/cmd/nugs/version_test.go @@ -8,15 +8,25 @@ import ( func TestIsVersionRequest(t *testing.T) { t.Parallel() - for _, arg := range []string{"version", "--version", "-v"} { - if !isVersionRequest([]string{arg}) { - t.Fatalf("expected %q to be a version request", arg) - } + tests := []struct { + name string + args []string + want bool + }{ + {name: "command", args: []string{"version"}, want: true}, + {name: "long flag", args: []string{"--version"}, want: true}, + {name: "short flag", args: []string{"-v"}, want: true}, + {name: "empty", args: nil, want: false}, + {name: "extra argument", args: []string{"version", "extra"}, want: false}, + {name: "help", args: []string{"--help"}, want: false}, } - for _, args := range [][]string{nil, {"version", "extra"}, {"--help"}} { - if isVersionRequest(args) { - t.Fatalf("did not expect %q to be a version request", args) - } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := isVersionRequest(tc.args); got != tc.want { + t.Fatalf("isVersionRequest(%q) = %v, want %v", tc.args, got, tc.want) + } + }) } } diff --git a/cmd/nugs/watch.go b/cmd/nugs/watch.go index 9d744e2..1b68122 100644 --- a/cmd/nugs/watch.go +++ b/cmd/nugs/watch.go @@ -2,6 +2,7 @@ package main import ( "context" + "errors" "fmt" "github.com/jmagar/nugs-cli/internal/catalog" @@ -67,16 +68,14 @@ func handleWatchCommand(ctx context.Context, cfg *Config, jsonLevel string) (boo switch subCmd { case "add": if len(cfg.Urls) < 3 { - printInfo("Usage: nugs watch add ") - return true, nil + return true, errors.New("watch add requires an artist ID") } if err := watchAdd(cfg, cfg.Urls[2]); err != nil { return true, fmt.Errorf("watch add failed: %w", err) } case "remove": if len(cfg.Urls) < 3 { - printInfo("Usage: nugs watch remove ") - return true, nil + return true, errors.New("watch remove requires an artist ID") } if err := watchRemove(cfg, cfg.Urls[2]); err != nil { return true, fmt.Errorf("watch remove failed: %w", err) diff --git a/internal/api/apilog.go b/internal/api/apilog.go index d7b00c1..aee1692 100644 --- a/internal/api/apilog.go +++ b/internal/api/apilog.go @@ -2,6 +2,7 @@ package api import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" @@ -38,6 +39,7 @@ var Logger *apiLogger var loggerMu sync.RWMutex var apiLogMaxBytes int64 = 5 << 20 +var apiLogRename = os.Rename const apiLogBackups = 3 @@ -105,17 +107,33 @@ func (l *apiLogger) rotateIfNeeded() error { if err := l.f.Close(); err != nil { return err } + l.f = nil + l.enc = nil for i := apiLogBackups - 1; i >= 1; i-- { - _ = os.Rename(fmt.Sprintf("%s.%d", l.path, i), fmt.Sprintf("%s.%d", l.path, i+1)) + _ = apiLogRename(fmt.Sprintf("%s.%d", l.path, i), fmt.Sprintf("%s.%d", l.path, i+1)) + } + if err := apiLogRename(l.path, l.path+".1"); err != nil && !os.IsNotExist(err) { + return errors.Join(err, l.openActive()) } - if err := os.Rename(l.path, l.path+".1"); err != nil && !os.IsNotExist(err) { + if err := l.openActive(); err != nil { + rollbackErr := apiLogRename(l.path+".1", l.path) + return errors.Join(err, rollbackErr, l.openActive()) + } + return nil +} + +func (l *apiLogger) openActive() error { + f, err := os.OpenFile(l.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { return err } - l.f, err = os.OpenFile(l.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) - if err == nil { - l.enc = json.NewEncoder(l.f) + if err := f.Chmod(0600); err != nil { + _ = f.Close() + return err } - return err + l.f = f + l.enc = json.NewEncoder(f) + return nil } // write is the internal append function. Failures are silently ignored — @@ -124,7 +142,13 @@ func (l *apiLogger) write(e APILogEntry) { e.Timestamp = time.Now().UTC().Format(time.RFC3339Nano) l.mu.Lock() defer l.mu.Unlock() - if l.f == nil || l.rotateIfNeeded() != nil { + if l.f == nil { + return + } + // A failed rotation is non-fatal when rotateIfNeeded recovered the active + // file; append the entry so transient filesystem errors do not disable logs. + _ = l.rotateIfNeeded() + if l.f == nil || l.enc == nil { return } _ = l.enc.Encode(e) diff --git a/internal/api/apilog_test.go b/internal/api/apilog_test.go index 3fbf396..7584448 100644 --- a/internal/api/apilog_test.go +++ b/internal/api/apilog_test.go @@ -1,8 +1,10 @@ package api import ( + "errors" "os" "path/filepath" + "strings" "testing" "time" ) @@ -49,3 +51,37 @@ func TestAPILoggerRotates(t *testing.T) { t.Fatalf("rotated log missing: %v", err) } } + +func TestAPILoggerContinuesAfterRotationRenameFailure(t *testing.T) { + defer CloseAPILogger() + oldMax := apiLogMaxBytes + apiLogMaxBytes = 1 + defer func() { apiLogMaxBytes = oldMax }() + + logPath := filepath.Join(t.TempDir(), "api.log") + if err := InitAPILogger(logPath); err != nil { + t.Fatal(err) + } + LogRequest("first", 200, 0, 0, "closed", nil) + + errRotate := errors.New("injected rotation rename failure") + oldRename := apiLogRename + apiLogRename = func(oldPath, newPath string) error { + if oldPath == logPath && newPath == logPath+".1" { + return errRotate + } + return oldRename(oldPath, newPath) + } + defer func() { apiLogRename = oldRename }() + + LogRequest("second", 200, 0, 0, "closed", nil) + contents, err := os.ReadFile(logPath) + if err != nil { + t.Fatal(err) + } + for _, label := range []string{`"label":"first"`, `"label":"second"`} { + if !strings.Contains(string(contents), label) { + t.Fatalf("active log after failed rotation does not contain %s: %s", label, contents) + } + } +} diff --git a/internal/api/circuit_test.go b/internal/api/circuit_test.go index 8d15ca1..0166aff 100644 --- a/internal/api/circuit_test.go +++ b/internal/api/circuit_test.go @@ -1,11 +1,20 @@ package api import ( + "context" + "errors" + "net/http" "sync" "testing" "time" ) +type failingRoundTripper func(*http.Request) (*http.Response, error) + +func (f failingRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + return f(req) +} + func TestCircuitBreakerAllowsOnlyOneHalfOpenProbe(t *testing.T) { cb := newCircuitBreaker(1, time.Nanosecond) cb.RecordFailure() @@ -44,3 +53,66 @@ func TestFailureDomainsAreScoped(t *testing.T) { t.Fatalf("catalog domain = %q", got) } } + +func TestRetryDoReopensHalfOpenCircuitWhenProbeCannotRun(t *testing.T) { + oldLimiter := RateLimiter + RateLimiter = newRateLimiter(1000, 10) + t.Cleanup(func() { RateLimiter = oldLimiter }) + + tests := []struct { + name string + makeReq func(context.Context) func() (*http.Request, error) + }{ + { + name: "request construction error", + makeReq: func(context.Context) func() (*http.Request, error) { + return func() (*http.Request, error) { + return nil, errors.New("construct request") + } + }, + }, + { + name: "transport error", + makeReq: func(ctx context.Context) func() (*http.Request, error) { + client := &http.Client{Transport: failingRoundTripper(func(*http.Request) (*http.Response, error) { + return nil, errors.New("transport failed") + })} + ctx = WithHTTPClient(ctx, client) + return func() (*http.Request, error) { + return http.NewRequestWithContext(ctx, http.MethodGet, "https://example.test", nil) + } + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cb := newCircuitBreaker(1, time.Hour) + cb.RecordFailure() + cb.mu.Lock() + cb.openedAt = time.Now().Add(-2 * time.Hour) + cb.mu.Unlock() + + circuitMu.Lock() + oldCircuits := circuits + circuits = map[string]*circuitBreaker{"catalog": cb} + circuitMu.Unlock() + t.Cleanup(func() { + circuitMu.Lock() + circuits = oldCircuits + circuitMu.Unlock() + }) + + ctx := context.Background() + if _, err := retryDo(ctx, "catalog.probe", tc.makeReq(ctx)); err == nil { + t.Fatal("retryDo returned nil error") + } + if got := cb.State(); got != circuitOpen { + t.Fatalf("circuit state = %s, want open", got) + } + if _, allowed := cb.Allow(); allowed { + t.Fatal("failed half-open probe did not restart the open timeout") + } + }) + } +} diff --git a/internal/api/client.go b/internal/api/client.go index e11ad96..01a5dc0 100644 --- a/internal/api/client.go +++ b/internal/api/client.go @@ -99,6 +99,14 @@ func breakerFor(label string) *circuitBreaker { return cb } +func failHalfOpenProbe(breaker *circuitBreaker, label string, state circuitState) { + if state != circuitHalfOpen { + return + } + newState := breaker.RecordFailure() + LogCircuitStateChange("circuit_opened", label, state.String(), newState.String()) +} + func drainAndClose(body io.ReadCloser) error { if body == nil { return nil @@ -153,6 +161,7 @@ func retryDo(ctx context.Context, label string, makeReq func() (*http.Request, e // 3. Build and execute the request. req, err := makeReq() if err != nil { + failHalfOpenProbe(breaker, label, cbState) return nil, err } start := time.Now() @@ -160,8 +169,9 @@ func retryDo(ctx context.Context, label string, makeReq func() (*http.Request, e duration := time.Since(start) if err != nil { - // Network-level error: log but do NOT trip the circuit breaker. - // Network hiccups are distinct from the API being overloaded. + // Network-level errors do not trip a closed circuit, but a failed + // half-open recovery probe must reopen it so the probe slot cannot wedge. + failHalfOpenProbe(breaker, label, cbState) LogRequest(label, 0, duration, attempt, cbState.String(), err) return nil, err } diff --git a/internal/cache/atomic.go b/internal/cache/atomic.go index 74e9b9c..e864577 100644 --- a/internal/cache/atomic.go +++ b/internal/cache/atomic.go @@ -4,8 +4,11 @@ import ( "fmt" "os" "path/filepath" + "sync" ) +var cacheProcessMu sync.Mutex + // WriteFileAtomic durably writes data to a unique temporary file in the target // directory and then atomically replaces targetPath. func WriteFileAtomic(targetPath string, data []byte, mode os.FileMode) error { @@ -40,6 +43,9 @@ func WriteFileAtomic(targetPath string, data []byte, mode os.FileMode) error { return fmt.Errorf("failed to rename temp file: %w", err) } ok = true + if err := syncParentDirectory(filepath.Dir(targetPath)); err != nil { + return fmt.Errorf("failed to sync target directory: %w", err) + } return nil } diff --git a/internal/cache/atomic_sync_unix.go b/internal/cache/atomic_sync_unix.go new file mode 100644 index 0000000..7033ae5 --- /dev/null +++ b/internal/cache/atomic_sync_unix.go @@ -0,0 +1,16 @@ +//go:build !windows + +package cache + +import ( + "errors" + "os" +) + +func syncParentDirectory(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + return errors.Join(dir.Sync(), dir.Close()) +} diff --git a/internal/cache/atomic_sync_windows.go b/internal/cache/atomic_sync_windows.go new file mode 100644 index 0000000..c2986e3 --- /dev/null +++ b/internal/cache/atomic_sync_windows.go @@ -0,0 +1,9 @@ +//go:build windows + +package cache + +// Windows cannot fsync an opened directory. WriteFileAtomic still flushes the +// file before rename and uses this no-op only for the unsupported directory step. +func syncParentDirectory(_ string) error { + return nil +} diff --git a/internal/cache/atomic_test.go b/internal/cache/atomic_test.go new file mode 100644 index 0000000..bb1e1ee --- /dev/null +++ b/internal/cache/atomic_test.go @@ -0,0 +1,28 @@ +package cache + +import ( + "os" + "path/filepath" + "testing" +) + +func TestWriteFileAtomicSyncsParentAndReplacesTarget(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "state.json") + if err := os.WriteFile(target, []byte("old"), 0644); err != nil { + t.Fatal(err) + } + if err := WriteFileAtomic(target, []byte("new"), 0600); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + if string(got) != "new" { + t.Fatalf("target = %q, want new", got) + } + if err := syncParentDirectory(dir); err != nil { + t.Fatalf("syncParentDirectory() error = %v", err) + } +} diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 2ffa990..578778e 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -5,7 +5,6 @@ import ( "fmt" "os" "path/filepath" - "sort" "strings" "time" @@ -19,6 +18,7 @@ const ( type catalogManifest struct { Generation string `json:"generation"` + Previous string `json:"previous,omitempty"` } func catalogArtifactPath(cacheDir, name string) (string, error) { @@ -36,6 +36,38 @@ func catalogArtifactPath(cacheDir, name string) (string, error) { return filepath.Join(cacheDir, catalogGenerationDir, manifest.Generation, name), nil } +func readCatalogArtifact(cacheDir, name string) ([]byte, error) { + var data []byte + err := WithCacheLock(func() error { + path, err := catalogArtifactPath(cacheDir, name) + if err != nil { + return err + } + data, err = os.ReadFile(path) + return err + }) + return data, err +} + +// StatCatalogData returns metadata for catalog.json in the manifest-selected +// generation while holding the cache lock across resolution and stat. +func StatCatalogData() (os.FileInfo, error) { + cacheDir, err := GetCacheDir() + if err != nil { + return nil, err + } + var info os.FileInfo + err = WithCacheLock(func() error { + path, err := catalogArtifactPath(cacheDir, "catalog.json") + if err != nil { + return err + } + info, err = os.Stat(path) + return err + }) + return info, err +} + // GetCacheDir returns the cache directory path, creating it if needed. func GetCacheDir() (string, error) { homeDir, err := os.UserHomeDir() @@ -59,12 +91,7 @@ func ReadCacheMeta() (*model.CacheMeta, error) { if err != nil { return nil, err } - metaPath, err := catalogArtifactPath(cacheDir, "catalog-meta.json") - if err != nil { - return nil, err - } - - data, err := os.ReadFile(metaPath) + data, err := readCatalogArtifact(cacheDir, "catalog-meta.json") if err != nil { if os.IsNotExist(err) { return nil, nil // No cache yet @@ -86,12 +113,7 @@ func ReadCatalogCache() (*model.LatestCatalogResp, error) { if err != nil { return nil, err } - catalogPath, err := catalogArtifactPath(cacheDir, "catalog.json") - if err != nil { - return nil, err - } - - data, err := os.ReadFile(catalogPath) + data, err := readCatalogArtifact(cacheDir, "catalog.json") if err != nil { if os.IsNotExist(err) { return nil, fmt.Errorf("no cache found - run 'nugs catalog update' first") @@ -221,7 +243,19 @@ func writePreparedCatalogCacheWithFailpoint(cacheDir string, prepared *preparedC return err } } - manifestData, err := json.Marshal(catalogManifest{Generation: filepath.Base(generation)}) + // Persist the generation directory entry before publishing a manifest that + // points at it. Individual artifact writes already sync the generation itself. + if err := syncParentDirectory(root); err != nil { + return fmt.Errorf("failed to sync catalog generation directory: %w", err) + } + previous := "" + if manifestData, readErr := os.ReadFile(filepath.Join(cacheDir, catalogManifestFile)); readErr == nil { + var old catalogManifest + if json.Unmarshal(manifestData, &old) == nil && filepath.Base(old.Generation) == old.Generation { + previous = old.Generation + } + } + manifestData, err := json.Marshal(catalogManifest{Generation: filepath.Base(generation), Previous: previous}) if err != nil { return err } @@ -229,22 +263,24 @@ func writePreparedCatalogCacheWithFailpoint(cacheDir string, prepared *preparedC return err } committed = true - pruneCatalogGenerations(root, filepath.Base(generation)) + pruneCatalogGenerations(root, filepath.Base(generation), previous) return nil } -func pruneCatalogGenerations(root, current string) { +func pruneCatalogGenerations(root, current, previous string) { entries, err := os.ReadDir(root) if err != nil { return } - sort.Slice(entries, func(i, j int) bool { return entries[i].Name() > entries[j].Name() }) - kept := 0 + keep := map[string]struct{}{current: {}} + if previous != "" { + keep[previous] = struct{}{} + } for _, entry := range entries { - if !entry.IsDir() || entry.Name() == current || kept < 1 { - if entry.IsDir() && entry.Name() != current { - kept++ - } + if !entry.IsDir() { + continue + } + if _, ok := keep[entry.Name()]; ok { continue } _ = os.RemoveAll(filepath.Join(root, entry.Name())) @@ -258,11 +294,7 @@ func ReadContainersIndex() (*model.ContainersIndex, error) { if err != nil { return nil, err } - indexPath, err := catalogArtifactPath(cacheDir, "containers_index.json") - if err != nil { - return nil, err - } - data, err := os.ReadFile(indexPath) + data, err := readCatalogArtifact(cacheDir, "containers_index.json") if err != nil { if os.IsNotExist(err) { return &model.ContainersIndex{Containers: map[int]model.ContainerIndexEntry{}}, nil diff --git a/internal/cache/filelock_unix.go b/internal/cache/filelock_unix.go index b4604ba..a2216b2 100644 --- a/internal/cache/filelock_unix.go +++ b/internal/cache/filelock_unix.go @@ -77,6 +77,8 @@ func (fl *FileLock) Release() error { // WithCacheLock executes a function with the catalog cache lock acquired. func WithCacheLock(fn func() error) error { + cacheProcessMu.Lock() + defer cacheProcessMu.Unlock() cacheDir, err := GetCacheDir() if err != nil { return err diff --git a/internal/cache/filelock_windows.go b/internal/cache/filelock_windows.go index 6d4374f..c0d857a 100644 --- a/internal/cache/filelock_windows.go +++ b/internal/cache/filelock_windows.go @@ -67,6 +67,8 @@ func (fl *FileLock) Release() error { // WithCacheLock executes a function with the catalog cache lock acquired. func WithCacheLock(fn func() error) error { + cacheProcessMu.Lock() + defer cacheProcessMu.Unlock() cacheDir, err := GetCacheDir() if err != nil { return err diff --git a/internal/cache/full_catalog.go b/internal/cache/full_catalog.go index 6e6b1e4..4dd1339 100644 --- a/internal/cache/full_catalog.go +++ b/internal/cache/full_catalog.go @@ -6,75 +6,149 @@ import ( "fmt" "os" "path/filepath" + "sort" "time" "github.com/jmagar/nugs-cli/internal/model" ) -const fullCatalogIndexFile = "full-catalog-index.json" - -// FullCatalogIndex is the durable, API-independent source used by catalog -// coverage and gap analysis after the first complete crawl. -type FullCatalogIndex struct { - Version int `json:"version"` - UpdatedAt time.Time `json:"updatedAt"` - Artists map[string]FullArtist `json:"artists"` -} +const ( + fullCatalogShardDir = "full-catalog-artists" + legacyFullCatalogIndex = "full-catalog-index.json" + fullCatalogShardMaxAge = 30 * 24 * time.Hour + fullCatalogShardMaxBytes = int64(256 << 20) +) -type FullArtist struct { +type fullCatalogArtistShard struct { + Version int `json:"version"` + UpdatedAt time.Time `json:"updatedAt"` + ArtistID string `json:"artistID"` ArtistName string `json:"artistName"` Pages []*model.ArtistMeta `json:"pages"` } -func ReadFullCatalogArtist(artistID string) ([]*model.ArtistMeta, string, error) { +func fullCatalogArtistPath(cacheDir, artistID string) (string, error) { + safeID := filepath.Base(artistID) + if safeID != artistID || safeID == "." || safeID == ".." || safeID == "" { + return "", fmt.Errorf("invalid artist ID: %q", artistID) + } + return filepath.Join(cacheDir, fullCatalogShardDir, "artist_"+safeID+".json"), nil +} + +// ReadFullCatalogArtist reads one durable artist shard without loading the +// rest of the catalog into memory. +func ReadFullCatalogArtist(artistID string) ([]*model.ArtistMeta, string, time.Time, error) { cacheDir, err := GetCacheDir() if err != nil { - return nil, "", err + return nil, "", time.Time{}, err + } + path, err := fullCatalogArtistPath(cacheDir, artistID) + if err != nil { + return nil, "", time.Time{}, err } - data, err := os.ReadFile(filepath.Join(cacheDir, fullCatalogIndexFile)) + data, err := os.ReadFile(path) if err != nil { - return nil, "", err + return nil, "", time.Time{}, err } - var index FullCatalogIndex - if err := json.Unmarshal(data, &index); err != nil { - return nil, "", fmt.Errorf("failed to parse full catalog index: %w", err) + var shard fullCatalogArtistShard + if err := json.Unmarshal(data, &shard); err != nil { + return nil, "", time.Time{}, fmt.Errorf("failed to parse full catalog artist shard: %w", err) } - artist, ok := index.Artists[artistID] - if !ok { - return nil, "", os.ErrNotExist + if shard.Version != 1 || shard.ArtistID != artistID || shard.UpdatedAt.IsZero() { + return nil, "", time.Time{}, fmt.Errorf("invalid full catalog artist shard for %s", artistID) } - return artist.Pages, artist.ArtistName, nil + return shard.Pages, shard.ArtistName, shard.UpdatedAt, nil +} + +// FullCatalogArtistFresh reports whether a shard is recent enough and was +// written after the latest catalog generation. A catalog update therefore +// invalidates older artist shards so watch/gap checks cannot hide new shows. +func FullCatalogArtistFresh(updatedAt time.Time, maxAge time.Duration) bool { + if updatedAt.IsZero() || maxAge <= 0 || time.Since(updatedAt) > maxAge { + return false + } + meta, err := ReadCacheMeta() + if err != nil || meta == nil { + return err == nil + } + return !updatedAt.Before(meta.LastUpdated) } func updateFullCatalogArtistLocked(cacheDir, artistID string, pages []*model.ArtistMeta) error { - path := filepath.Join(cacheDir, fullCatalogIndexFile) - index := FullCatalogIndex{Version: 1, Artists: make(map[string]FullArtist)} - if data, err := os.ReadFile(path); err == nil { - if err := json.Unmarshal(data, &index); err != nil { - return fmt.Errorf("failed to parse full catalog index: %w", err) - } - } else if !errors.Is(err, os.ErrNotExist) { + path, err := fullCatalogArtistPath(cacheDir, artistID) + if err != nil { return err } - if index.Artists == nil { - index.Artists = make(map[string]FullArtist) - } name := "" for _, page := range pages { - if page != nil { - if value, ok := page.Response.ArtistName.(string); ok && value != "" { - name = value - } - if name == "" && len(page.Response.Containers) > 0 && page.Response.Containers[0] != nil { - name = page.Response.Containers[0].ArtistName - } + if page == nil { + continue + } + if value, ok := page.Response.ArtistName.(string); ok && value != "" { + name = value + } + if name == "" && len(page.Response.Containers) > 0 && page.Response.Containers[0] != nil { + name = page.Response.Containers[0].ArtistName } } - index.Artists[artistID] = FullArtist{ArtistName: name, Pages: pages} - index.UpdatedAt = time.Now() - data, err := json.Marshal(index) + shard := fullCatalogArtistShard{ + Version: 1, + UpdatedAt: time.Now(), + ArtistID: artistID, + ArtistName: name, + Pages: pages, + } + data, err := json.Marshal(shard) + if err != nil { + return err + } + if err := WriteFileAtomic(path, data, 0644); err != nil { + return err + } + // The former monolithic file is neither bounded nor efficiently readable. + _ = os.Remove(filepath.Join(cacheDir, legacyFullCatalogIndex)) + return pruneFullCatalogShardsLocked(filepath.Dir(path), fullCatalogShardMaxAge, fullCatalogShardMaxBytes, path) +} + +func pruneFullCatalogShardsLocked(dir string, maxAge time.Duration, maxBytes int64, preserve string) error { + entries, err := os.ReadDir(dir) + if errors.Is(err, os.ErrNotExist) { + return nil + } if err != nil { return err } - return WriteFileAtomic(path, data, 0644) + type candidate struct { + path string + info os.FileInfo + } + files := make([]candidate, 0, len(entries)) + var total int64 + now := time.Now() + for _, entry := range entries { + if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" { + continue + } + path := filepath.Join(dir, entry.Name()) + info, statErr := entry.Info() + if statErr != nil { + continue + } + if path != preserve && maxAge > 0 && now.Sub(info.ModTime()) > maxAge { + _ = os.Remove(path) + continue + } + files = append(files, candidate{path: path, info: info}) + total += info.Size() + } + sort.Slice(files, func(i, j int) bool { return files[i].info.ModTime().Before(files[j].info.ModTime()) }) + for _, file := range files { + if maxBytes <= 0 || total <= maxBytes || file.path == preserve { + continue + } + if err := os.Remove(file.path); err == nil { + total -= file.info.Size() + } + } + return nil } diff --git a/internal/cache/regression_test.go b/internal/cache/regression_test.go index 695d8f9..2de3c23 100644 --- a/internal/cache/regression_test.go +++ b/internal/cache/regression_test.go @@ -69,7 +69,7 @@ func TestWriteCatalogCache_ConcurrentWriters_Consistency(t *testing.T) { select { case <-done: - case <-time.After(5 * time.Second): + case <-time.After(30 * time.Second): t.Fatal("concurrent cache writes timed out") } close(errCh) diff --git a/internal/cache/remediation_test.go b/internal/cache/remediation_test.go index 017a6fb..e6421c6 100644 --- a/internal/cache/remediation_test.go +++ b/internal/cache/remediation_test.go @@ -1,9 +1,11 @@ package cache import ( + "encoding/json" "errors" "os" "path/filepath" + "sync" "testing" "time" @@ -42,6 +44,112 @@ func TestCatalogGenerationFailureKeepsPreviousSnapshot(t *testing.T) { } } +func TestCatalogManifestRetainsActualPreviousGeneration(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + if err := WriteCatalogCache(buildCatalog(1, 1, "one"), 0, time.Duration.String); err != nil { + t.Fatal(err) + } + cacheDir, _ := GetCacheDir() + readManifest := func() catalogManifest { + data, err := os.ReadFile(filepath.Join(cacheDir, catalogManifestFile)) + if err != nil { + t.Fatal(err) + } + var manifest catalogManifest + if err := json.Unmarshal(data, &manifest); err != nil { + t.Fatal(err) + } + return manifest + } + first := readManifest() + if err := WriteCatalogCache(buildCatalog(2, 2, "two"), 0, time.Duration.String); err != nil { + t.Fatal(err) + } + second := readManifest() + if second.Previous != first.Generation { + t.Fatalf("previous generation = %q, want %q", second.Previous, first.Generation) + } + for _, generation := range []string{second.Generation, second.Previous} { + if _, err := os.Stat(filepath.Join(cacheDir, catalogGenerationDir, generation)); err != nil { + t.Fatalf("retained generation %q: %v", generation, err) + } + } +} + +func TestCatalogGenerationConcurrentReadersAndWriters(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + if err := WriteCatalogCache(buildCatalog(1, 1, "initial"), 0, time.Duration.String); err != nil { + t.Fatal(err) + } + const writers = 16 + const readers = 6 + start := make(chan struct{}) + errs := make(chan error, writers+readers*120) + var wg sync.WaitGroup + for i := 0; i < writers; i++ { + i := i + wg.Add(1) + go func() { + defer wg.Done() + <-start + errs <- WriteCatalogCache(buildCatalog(100+i, 200+i, "writer"), 0, time.Duration.String) + }() + } + for i := 0; i < readers; i++ { + wg.Add(1) + go func() { + defer wg.Done() + <-start + for range 40 { + if _, err := ReadCatalogCache(); err != nil { + errs <- err + } + if _, err := ReadCacheMeta(); err != nil { + errs <- err + } + if _, err := ReadContainersIndex(); err != nil { + errs <- err + } + } + }() + } + close(start) + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent catalog access failed: %v", err) + } + } +} + +func TestFullCatalogUsesBoundedArtistShards(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + page := &model.ArtistMeta{} + for _, artistID := range []string{"1", "2", "3"} { + page.Response.ArtistName = "Artist " + artistID + if err := WriteArtistMetaCache(artistID, []*model.ArtistMeta{page}); err != nil { + t.Fatal(err) + } + } + cacheDir, _ := GetCacheDir() + dir := filepath.Join(cacheDir, fullCatalogShardDir) + entries, err := os.ReadDir(dir) + if err != nil || len(entries) != 3 { + t.Fatalf("artist shards = %d, err=%v; want 3", len(entries), err) + } + if err := WithCacheLock(func() error { return pruneFullCatalogShardsLocked(dir, 0, 1, "") }); err != nil { + t.Fatal(err) + } + entries, err = os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("bounded shard prune retained %d files, want 0", len(entries)) + } +} + func TestArtistCacheEvictionPreservesDurableFullIndex(t *testing.T) { t.Setenv("HOME", t.TempDir()) page := &model.ArtistMeta{} @@ -60,12 +168,13 @@ func TestArtistCacheEvictionPreservesDurableFullIndex(t *testing.T) { if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { t.Fatalf("artist cache was not evicted: %v", err) } - pages, name, err := ReadFullCatalogArtist("1") + pages, name, _, err := ReadFullCatalogArtist("1") if err != nil || len(pages) != 1 || name != "Artist" { t.Fatalf("durable full index lost: pages=%d name=%q err=%v", len(pages), name, err) } cacheDir, _ := GetCacheDir() - if _, err := os.Stat(filepath.Join(cacheDir, fullCatalogIndexFile)); err != nil { + shardPath, _ := fullCatalogArtistPath(cacheDir, "1") + if _, err := os.Stat(shardPath); err != nil { t.Fatal(err) } } diff --git a/internal/catalog/cache_status_test.go b/internal/catalog/cache_status_test.go new file mode 100644 index 0000000..b623774 --- /dev/null +++ b/internal/catalog/cache_status_test.go @@ -0,0 +1,45 @@ +package catalog + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jmagar/nugs-cli/internal/cache" + "github.com/jmagar/nugs-cli/internal/testutil" +) + +func TestCatalogCacheStatusUsesManifestGeneration(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + if err := cache.WriteCatalogCache(buildUpdateCatalog([]showSpec{{1, 2, "Artist", "2025-01-01", "Show"}}), 0, time.Duration.String); err != nil { + t.Fatal(err) + } + cacheDir, err := cache.GetCacheDir() + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(cacheDir, "catalog.json")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("test requires no legacy catalog path, stat err = %v", err) + } + + var callErr error + stdout := testutil.CaptureStdout(t, func() { + callErr = CatalogCacheStatus("normal", &Deps{FormatDuration: time.Duration.String}) + }) + if callErr != nil { + t.Fatal(callErr) + } + var payload struct { + Exists bool `json:"exists"` + FileSizeBytes int64 `json:"fileSizeBytes"` + } + if err := json.Unmarshal([]byte(stdout), &payload); err != nil { + t.Fatalf("invalid status JSON %q: %v", stdout, err) + } + if !payload.Exists || payload.FileSizeBytes <= 0 { + t.Fatalf("unexpected cache status: %+v", payload) + } +} diff --git a/internal/catalog/handlers.go b/internal/catalog/handlers.go index a8bb931..8292896 100644 --- a/internal/catalog/handlers.go +++ b/internal/catalog/handlers.go @@ -278,8 +278,7 @@ func CatalogCacheStatus(jsonLevel string, deps *Deps) error { } cacheDir, _ := cache.GetCacheDir() - catalogPath := filepath.Join(cacheDir, "catalog.json") - fileInfo, err := os.Stat(catalogPath) + fileInfo, err := cache.StatCatalogData() if err != nil { return fmt.Errorf("failed to stat cache file: %w", err) } diff --git a/internal/catalog/media_filter.go b/internal/catalog/media_filter.go index e9e42fe..5a42290 100644 --- a/internal/catalog/media_filter.go +++ b/internal/catalog/media_filter.go @@ -220,14 +220,21 @@ func AnalyzeArtistCatalogMediaAware(ctx context.Context, artistID string, cfg *m return nil, fmt.Errorf("GetArtistMetaCached callback not configured") } - // The complete local index is authoritative for warm catalog analysis. It - // deliberately has no TTL: catalog update/crawl refreshes it, while normal - // coverage and gap commands have a zero-network request budget. - artistMetas, _, localErr := cache.ReadFullCatalogArtist(artistID) - cacheUsed, cacheStaleUse := localErr == nil, false - if localErr != nil { + // A fresh durable shard is authoritative for warm catalog analysis and keeps + // normal coverage/gap commands within their zero-network request budget. + // A catalog update invalidates older shards so new releases are fetched. + artistMetas, _, shardUpdatedAt, localErr := cache.ReadFullCatalogArtist(artistID) + localFresh := localErr == nil && cache.FullCatalogArtistFresh(shardUpdatedAt, ArtistMetaCacheTTL) + cacheUsed, cacheStaleUse := localFresh, false + if !localFresh { var err error - artistMetas, cacheUsed, cacheStaleUse, err = deps.GetArtistMetaCached(ctx, artistID, ArtistMetaCacheTTL) + requestTTL := ArtistMetaCacheTTL + if localErr == nil { + // A stale durable shard implies the ordinary artist cache contains + // the same stale pages. Force the callback through to the API. + requestTTL = 0 + } + artistMetas, cacheUsed, cacheStaleUse, err = deps.GetArtistMetaCached(ctx, artistID, requestTTL) if err != nil { return nil, fmt.Errorf("failed to get artist metadata: %w", err) } diff --git a/internal/catalog/request_budget_test.go b/internal/catalog/request_budget_test.go index 01eb94a..ea10b49 100644 --- a/internal/catalog/request_budget_test.go +++ b/internal/catalog/request_budget_test.go @@ -46,3 +46,56 @@ func TestAnalyzeArtistWarmIndexUsesZeroAPIRequests(t *testing.T) { t.Fatalf("warm analysis made %d requests, want zero additional", requests) } } + +func TestAnalyzeArtistRefreshesShardOlderThanCatalogGeneration(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + oldPage := artistPageWithShow("Artist", 10, "Old Show") + if err := cache.WriteArtistMetaCache("1", []*model.ArtistMeta{oldPage}); err != nil { + t.Fatal(err) + } + latest := &model.LatestCatalogResp{} + if err := cache.WriteCatalogCache(latest, 0, time.Duration.String); err != nil { + t.Fatal(err) + } + + freshPage := artistPageWithShow("Artist", 20, "New Show") + requests := 0 + deps := &Deps{ + GetArtistMetaCached: func(_ context.Context, _ string, ttl time.Duration) ([]*model.ArtistMeta, bool, bool, error) { + requests++ + if ttl != 0 { + t.Fatalf("refresh TTL = %s, want 0 to bypass stale artist cache", ttl) + } + if err := cache.WriteArtistMetaCache("1", []*model.ArtistMeta{freshPage}); err != nil { + return nil, false, false, err + } + return []*model.ArtistMeta{freshPage}, false, false, nil + }, + GetShowMediaType: func(*model.AlbArtResp) model.MediaType { return model.MediaTypeAudio }, + } + cfg := &model.Config{OutPath: t.TempDir(), DefaultOutputs: "audio"} + analysis, err := AnalyzeArtistCatalogMediaAware(context.Background(), "1", cfg, "", model.MediaTypeAudio, deps) + if err != nil { + t.Fatal(err) + } + if requests != 1 { + t.Fatalf("metadata refresh requests = %d, want 1", requests) + } + if len(analysis.Shows) != 1 || analysis.Shows[0].Show.ContainerID != 20 { + t.Fatalf("analysis used stale shard: %+v", analysis.Shows) + } +} + +func artistPageWithShow(name string, containerID int, title string) *model.ArtistMeta { + page := &model.ArtistMeta{} + page.Response.ArtistName = name + page.Response.Containers = []*model.AlbArtResp{{ + ArtistName: name, + ContainerID: containerID, + ContainerInfo: title, + PerformanceDate: "2025-01-01", + AvailabilityTypeStr: model.AvailableAvailabilityType, + ProductFormatList: []*model.ProductFormatList{{}}, + }} + return page +} diff --git a/internal/catalog/watch.go b/internal/catalog/watch.go index af3b95f..d82546e 100644 --- a/internal/catalog/watch.go +++ b/internal/catalog/watch.go @@ -203,11 +203,11 @@ func WatchCheck(ctx context.Context, cfg *model.Config, streamParams *model.Stre } } - sendWatchSummary(ctx, deps.Notify, totalDownloaded, totalFailed, artistErrors) + sendWatchSummary(ctx, deps.Notify, totalDownloaded, totalFailed, artistErrors, catalogUpdateErr) if catalogUpdateErr != nil || totalFailed > 0 || len(artistErrors) > 0 { outcome := &WatchOutcomeError{Downloaded: totalDownloaded, Failed: totalFailed, ArtistErrors: artistErrors, CatalogUpdate: catalogUpdateErr} if catalogUpdateErr != nil { - return fmt.Errorf("%w: catalog update: %v", outcome, catalogUpdateErr) + return fmt.Errorf("%w: catalog update: %w", outcome, catalogUpdateErr) } return outcome } @@ -231,12 +231,23 @@ func sendArtistUpdate(ctx context.Context, notify func(ctx context.Context, titl // sendWatchSummary fires a single Gotify notification summarising the watch check outcome. // Sends nothing if notify is nil or there is nothing to report (all up-to-date, no errors). -func sendWatchSummary(ctx context.Context, notify func(ctx context.Context, title, message string, priority int) error, downloaded, failed int, errs []string) { +func sendWatchSummary(ctx context.Context, notify func(ctx context.Context, title, message string, priority int) error, downloaded, failed int, errs []string, catalogUpdateErr error) { if notify == nil { return } switch { + case catalogUpdateErr != nil: + parts := []string{fmt.Sprintf("Catalog update failed: %v", catalogUpdateErr)} + if downloaded > 0 { + parts = append(parts, fmt.Sprintf("%d new show(s) downloaded from cached catalog", downloaded)) + } + if failed > 0 { + parts = append(parts, fmt.Sprintf("%d download failure(s)", failed)) + } + parts = append(parts, errs...) + _ = notify(ctx, "Nugs Watch Error", strings.Join(parts, "\n"), 7) + case downloaded > 0: msg := fmt.Sprintf("%d new show(s) downloaded", downloaded) if failed > 0 { diff --git a/internal/catalog/watch_systemd.go b/internal/catalog/watch_systemd.go index 11bf1ad..4caaccd 100644 --- a/internal/catalog/watch_systemd.go +++ b/internal/catalog/watch_systemd.go @@ -4,10 +4,12 @@ package catalog import ( "bytes" + "errors" "fmt" "os" "os/exec" "path/filepath" + "strings" "text/template" "time" @@ -83,24 +85,32 @@ func WatchEnable(cfg *model.Config) error { servicePath := filepath.Join(unitDir, "nugs-watch.service") timerPath := filepath.Join(unitDir, "nugs-watch.timer") - backup := snapshotUnitFiles(servicePath, timerPath) - rollback := func() { - _ = systemctlUser("disable", "--now", "nugs-watch.timer") - restoreUnitFiles(backup) - _ = systemctlUser("daemon-reload") + backup, err := snapshotUnitFiles(servicePath, timerPath) + if err != nil { + return fmt.Errorf("snapshot existing watch units: %w", err) + } + activation, err := snapshotUnitActivation("nugs-watch.timer") + if err != nil { + return fmt.Errorf("snapshot watch timer state: %w", err) + } + rollback := func(cause error) error { + if rollbackErr := restoreWatchInstallation(backup, activation); rollbackErr != nil { + return fmt.Errorf("%w; rollback failed: %w", cause, rollbackErr) + } + return cause } if err := writeWatchUnitFiles(cfg, unitDir, binPath); err != nil { - restoreUnitFiles(backup) + if restoreErr := restoreUnitFiles(backup); restoreErr != nil { + return fmt.Errorf("%w; restore unit files: %w", err, restoreErr) + } return err } if err := systemctlUser("daemon-reload"); err != nil { - rollback() - return fmt.Errorf("daemon-reload failed: %w", err) + return rollback(fmt.Errorf("daemon-reload failed: %w", err)) } if err := systemctlUser("enable", "--now", "nugs-watch.timer"); err != nil { - rollback() - return fmt.Errorf("failed to enable timer: %w", err) + return rollback(fmt.Errorf("failed to enable timer: %w", err)) } systemdInterval, _ := toSystemdDuration(watchIntervalOrDefault(cfg)) @@ -181,13 +191,6 @@ func toSystemdDuration(s string) (string, error) { // WatchDisable stops and disables the nugs-watch systemd timer and removes unit files. func WatchDisable() error { - // A failed disable leaves the unit files intact so the operation can be - // retried without losing the installed configuration. - if err := systemctlUser("disable", "--now", "nugs-watch.timer"); err != nil { - return fmt.Errorf("failed to disable timer: %w", err) - } - _ = systemctlUser("stop", "nugs-watch.service") - unitDir, err := systemdUserDir() if err != nil { return err @@ -195,22 +198,44 @@ func WatchDisable() error { servicePath := filepath.Join(unitDir, "nugs-watch.service") timerPath := filepath.Join(unitDir, "nugs-watch.timer") - backup := snapshotUnitFiles(servicePath, timerPath) + backup, err := snapshotUnitFiles(servicePath, timerPath) + if err != nil { + return fmt.Errorf("snapshot watch units: %w", err) + } + activation, err := snapshotUnitActivation("nugs-watch.timer") + if err != nil { + return fmt.Errorf("snapshot watch timer state: %w", err) + } + if err := systemctlUser("disable", "--now", "nugs-watch.timer"); err != nil && !isUnitAbsentError(err) { + return fmt.Errorf("failed to disable timer: %w", err) + } + if err := systemctlUser("stop", "nugs-watch.service"); err != nil && !isUnitAbsentError(err) { + cause := fmt.Errorf("failed to stop watch service: %w", err) + if rollbackErr := restoreWatchInstallation(backup, activation); rollbackErr != nil { + return fmt.Errorf("%w; rollback failed: %w", cause, rollbackErr) + } + return cause + } removed := 0 for _, p := range []string{servicePath, timerPath} { if err := os.Remove(p); err != nil && !os.IsNotExist(err) { - restoreUnitFiles(backup) - return fmt.Errorf("failed to remove %s: %w", p, err) + cause := fmt.Errorf("failed to remove %s: %w", p, err) + if rollbackErr := restoreWatchInstallation(backup, activation); rollbackErr != nil { + return fmt.Errorf("%w; rollback failed: %w", cause, rollbackErr) + } + return cause } else if err == nil { removed++ } } if err := systemctlUser("daemon-reload"); err != nil { - restoreUnitFiles(backup) - _ = systemctlUser("daemon-reload") - return fmt.Errorf("daemon-reload failed after removing unit files: %w", err) + cause := fmt.Errorf("daemon-reload failed after removing unit files: %w", err) + if rollbackErr := restoreWatchInstallation(backup, activation); rollbackErr != nil { + return fmt.Errorf("%w; rollback failed: %w", cause, rollbackErr) + } + return cause } if removed == 0 { @@ -221,29 +246,111 @@ func WatchDisable() error { return nil } -type unitSnapshot map[string][]byte +type unitFileSnapshot struct { + exists bool + data []byte +} + +type unitSnapshot map[string]unitFileSnapshot -func snapshotUnitFiles(paths ...string) unitSnapshot { +func snapshotUnitFiles(paths ...string) (unitSnapshot, error) { snapshot := make(unitSnapshot, len(paths)) for _, path := range paths { data, err := os.ReadFile(path) - if err == nil { - snapshot[path] = data - } else { - snapshot[path] = nil + if err != nil { + if os.IsNotExist(err) { + snapshot[path] = unitFileSnapshot{} + continue + } + return nil, fmt.Errorf("read %s: %w", path, err) } + snapshot[path] = unitFileSnapshot{exists: true, data: data} } - return snapshot + return snapshot, nil } -func restoreUnitFiles(snapshot unitSnapshot) { - for path, data := range snapshot { - if data == nil { - _ = os.Remove(path) +func restoreUnitFiles(snapshot unitSnapshot) error { + var restoreErrs []error + for path, file := range snapshot { + if !file.exists { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + restoreErrs = append(restoreErrs, fmt.Errorf("remove %s: %w", path, err)) + } continue } - _ = cache.WriteFileAtomic(path, data, 0644) + if err := cache.WriteFileAtomic(path, file.data, 0644); err != nil { + restoreErrs = append(restoreErrs, fmt.Errorf("write %s: %w", path, err)) + } + } + return errors.Join(restoreErrs...) +} + +type unitActivation struct { + enabled bool + active bool +} + +func snapshotUnitActivation(unit string) (unitActivation, error) { + enabled, err := systemctlUnitState("is-enabled", unit) + if err != nil { + return unitActivation{}, err + } + active, err := systemctlUnitState("is-active", unit) + if err != nil { + return unitActivation{}, err + } + return unitActivation{enabled: enabled, active: active}, nil +} + +func systemctlUnitState(action, unit string) (bool, error) { + output, err := runSystemctlUser(action, unit) + state := strings.ToLower(strings.TrimSpace(output)) + if err == nil { + return true, nil + } + for _, benign := range []string{"disabled", "inactive", "failed", "unknown", "not-found", "static", "masked", "no such file or directory", "does not exist", "not loaded"} { + if strings.Contains(state, benign) { + return false, nil + } } + return false, fmt.Errorf("systemctl --user %s %s: %w: %s", action, unit, err, strings.TrimSpace(output)) +} + +func restoreWatchInstallation(files unitSnapshot, activation unitActivation) error { + var rollbackErrs []error + if err := systemctlUser("disable", "--now", "nugs-watch.timer"); err != nil && !isUnitAbsentError(err) { + rollbackErrs = append(rollbackErrs, err) + } + if err := restoreUnitFiles(files); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + if err := systemctlUser("daemon-reload"); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + if activation.enabled { + if err := systemctlUser("enable", "nugs-watch.timer"); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + } + if activation.active { + if err := systemctlUser("start", "nugs-watch.timer"); err != nil { + rollbackErrs = append(rollbackErrs, err) + } + } + return errors.Join(rollbackErrs...) +} + +func isUnitAbsentError(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + for _, marker := range []string{"not found", "not-found", "not loaded", "does not exist", "no such file or directory"} { + if strings.Contains(message, marker) { + return true + } + } + return false } // systemdUserDir returns ~/.config/systemd/user, creating it if needed. @@ -280,9 +387,16 @@ func writeUnitFile(path, tmplStr string, data any) error { // systemctlUser runs systemctl --user with the given arguments. func systemctlUser(args ...string) error { + output, err := runSystemctlUser(args...) + if err != nil { + return fmt.Errorf("systemctl --user %s: %w: %s", strings.Join(args, " "), err, strings.TrimSpace(output)) + } + return nil +} + +var runSystemctlUser = func(args ...string) (string, error) { cmdArgs := append([]string{"--user"}, args...) cmd := exec.Command("systemctl", cmdArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - return cmd.Run() + output, err := cmd.CombinedOutput() + return string(output), err } diff --git a/internal/catalog/watch_systemd_test.go b/internal/catalog/watch_systemd_test.go new file mode 100644 index 0000000..503e9c8 --- /dev/null +++ b/internal/catalog/watch_systemd_test.go @@ -0,0 +1,157 @@ +//go:build linux + +package catalog + +import ( + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/jmagar/nugs-cli/internal/model" +) + +func stubSystemctl(t *testing.T, fn func(args ...string) (string, error)) { + t.Helper() + old := runSystemctlUser + runSystemctlUser = fn + t.Cleanup(func() { runSystemctlUser = old }) +} + +func TestRestoreUnitFilesRollsBackCreatedAndReplacedFiles(t *testing.T) { + dir := t.TempDir() + existing := filepath.Join(dir, "existing.service") + created := filepath.Join(dir, "created.timer") + if err := os.WriteFile(existing, []byte("old"), 0644); err != nil { + t.Fatal(err) + } + snapshot, err := snapshotUnitFiles(existing, created) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(existing, []byte("new"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(created, []byte("new"), 0644); err != nil { + t.Fatal(err) + } + if err := restoreUnitFiles(snapshot); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(existing) + if err != nil || string(data) != "old" { + t.Fatalf("existing unit not restored: %q, %v", data, err) + } + if _, err := os.Stat(created); !os.IsNotExist(err) { + t.Fatalf("new unit not removed: %v", err) + } +} + +func TestWatchEnableRollbackRestoresFilesAndActivation(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + unitDir, err := systemdUserDir() + if err != nil { + t.Fatal(err) + } + servicePath := filepath.Join(unitDir, "nugs-watch.service") + timerPath := filepath.Join(unitDir, "nugs-watch.timer") + if err := os.WriteFile(servicePath, []byte("old service"), 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(timerPath, []byte("old timer"), 0644); err != nil { + t.Fatal(err) + } + + var calls []string + stubSystemctl(t, func(args ...string) (string, error) { + call := strings.Join(args, " ") + calls = append(calls, call) + switch call { + case "is-enabled nugs-watch.timer": + return "enabled\n", nil + case "is-active nugs-watch.timer": + return "active\n", nil + case "daemon-reload": + return "bus unavailable", errors.New("exit status 1") + default: + return "", nil + } + }) + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + err = WatchEnable(&model.Config{WatchedArtists: []string{"1125"}, FfmpegNameStr: executable}) + if err == nil || !strings.Contains(err.Error(), "rollback failed") { + t.Fatalf("WatchEnable() error = %v, want surfaced rollback failure", err) + } + for path, want := range map[string]string{servicePath: "old service", timerPath: "old timer"} { + got, readErr := os.ReadFile(path) + if readErr != nil || string(got) != want { + t.Fatalf("restored %s = %q, err %v; want %q", path, got, readErr, want) + } + } + joined := strings.Join(calls, "\n") + for _, want := range []string{"disable --now nugs-watch.timer", "enable nugs-watch.timer", "start nugs-watch.timer"} { + if !strings.Contains(joined, want) { + t.Fatalf("systemctl calls missing %q:\n%s", want, joined) + } + } +} + +func TestWatchDisableTreatsAbsentUnitsAsAlreadyDisabled(t *testing.T) { + t.Setenv("HOME", t.TempDir()) + stubSystemctl(t, func(args ...string) (string, error) { + switch args[0] { + case "is-enabled": + return "not-found\n", errors.New("exit status 4") + case "is-active": + return "unknown\n", errors.New("exit status 3") + case "disable", "stop": + return "Unit nugs-watch.timer does not exist", errors.New("exit status 1") + default: + return "", nil + } + }) + if err := WatchDisable(); err != nil { + t.Fatalf("WatchDisable() error = %v, want idempotent success", err) + } +} + +func TestSnapshotUnitFilesReportsReadFailure(t *testing.T) { + path := filepath.Join(t.TempDir(), "unit.service") + if err := os.Mkdir(path, 0755); err != nil { + t.Fatal(err) + } + if _, err := snapshotUnitFiles(path); err == nil { + t.Fatal("snapshotUnitFiles() error = nil for unreadable unit path") + } +} + +func TestRestoreUnitFilesReportsRemoveAndWriteFailures(t *testing.T) { + t.Run("remove", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "unit.service") + if err := os.Mkdir(path, 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "child"), []byte("x"), 0644); err != nil { + t.Fatal(err) + } + if err := restoreUnitFiles(unitSnapshot{path: {}}); err == nil { + t.Fatal("restoreUnitFiles() error = nil for failed removal") + } + }) + + t.Run("write", func(t *testing.T) { + parent := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(parent, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + path := filepath.Join(parent, "unit.service") + snapshot := unitSnapshot{path: {exists: true, data: []byte("unit")}} + if err := restoreUnitFiles(snapshot); err == nil { + t.Fatal("restoreUnitFiles() error = nil for failed write") + } + }) +} diff --git a/internal/catalog/watch_test.go b/internal/catalog/watch_test.go index 04a2329..dca7a79 100644 --- a/internal/catalog/watch_test.go +++ b/internal/catalog/watch_test.go @@ -226,6 +226,7 @@ func TestWatchCheck_CatalogUpdateFailureIsNonFatal(t *testing.T) { fetchErr := errors.New("network timeout") artistMetaCalled := false + notify, called, title, message, priority := notifyCapture() deps := &Deps{ FetchCatalog: func(_ context.Context) (*model.LatestCatalogResp, error) { @@ -239,6 +240,7 @@ func TestWatchCheck_CatalogUpdateFailureIsNonFatal(t *testing.T) { return []*model.ArtistMeta{emptyArtistMeta("Billy Strings")}, false, false, nil }, GetShowMediaType: func(_ *model.AlbArtResp) model.MediaType { return model.MediaTypeAudio }, + Notify: notify, } cfg := &model.Config{ @@ -253,6 +255,9 @@ func TestWatchCheck_CatalogUpdateFailureIsNonFatal(t *testing.T) { if !artistMetaCalled { t.Error("GetArtistMetaCached should be called even when catalog update fails") } + if !*called || *title != "Nugs Watch Error" || *priority != 7 || !strings.Contains(*message, fetchErr.Error()) { + t.Fatalf("degraded notification = called:%t title:%q message:%q priority:%d", *called, *title, *message, *priority) + } } // TestWatchCheck_PerArtistFailureContinues verifies WatchCheck continues to the next artist @@ -503,7 +508,7 @@ func TestSendWatchSummary(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { fn, gotCalled, gotTitle, gotMsg, gotPriority := notifyCapture() - sendWatchSummary(context.Background(), fn, tc.downloaded, tc.failed, tc.errs) + sendWatchSummary(context.Background(), fn, tc.downloaded, tc.failed, tc.errs, nil) if *gotCalled != tc.wantCalled { t.Fatalf("notify called = %v, want %v", *gotCalled, tc.wantCalled) @@ -705,27 +710,3 @@ func TestWatchEnableUnitContent_MinutesConversion(t *testing.T) { t.Errorf("timer file should use '30min' for minutes:\n%s", content) } } - -func TestRestoreUnitFilesRollsBackCreatedAndReplacedFiles(t *testing.T) { - dir := t.TempDir() - existing := filepath.Join(dir, "existing.service") - created := filepath.Join(dir, "created.timer") - if err := os.WriteFile(existing, []byte("old"), 0644); err != nil { - t.Fatal(err) - } - snapshot := snapshotUnitFiles(existing, created) - if err := os.WriteFile(existing, []byte("new"), 0644); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(created, []byte("new"), 0644); err != nil { - t.Fatal(err) - } - restoreUnitFiles(snapshot) - data, err := os.ReadFile(existing) - if err != nil || string(data) != "old" { - t.Fatalf("existing unit not restored: %q, %v", data, err) - } - if _, err := os.Stat(created); !os.IsNotExist(err) { - t.Fatalf("new unit not removed: %v", err) - } -} diff --git a/internal/config/atomic_test.go b/internal/config/atomic_test.go index df79ac1..254f5e5 100644 --- a/internal/config/atomic_test.go +++ b/internal/config/atomic_test.go @@ -50,3 +50,9 @@ func TestAtomicWriteFileConcurrentWriters(t *testing.T) { t.Fatalf("orphan temp files: %v", matches) } } + +func TestSyncDirectoryPlatformHook(t *testing.T) { + if err := syncDirectory(t.TempDir()); err != nil { + t.Fatalf("syncDirectory() error = %v", err) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index a5facd7..978e547 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -270,13 +270,7 @@ func atomicWriteFile(targetPath string, data []byte, perm os.FileMode) error { return fmt.Errorf("rename temp file: %w", err) } committed = true - dirFile, err := os.Open(dir) - if err != nil { - return fmt.Errorf("open config directory for sync: %w", err) - } - syncErr := dirFile.Sync() - closeErr := dirFile.Close() - if err := errors.Join(syncErr, closeErr); err != nil { + if err := syncDirectory(dir); err != nil { return fmt.Errorf("sync config directory: %w", err) } return nil diff --git a/internal/config/sync_dir_unix.go b/internal/config/sync_dir_unix.go new file mode 100644 index 0000000..863a0a6 --- /dev/null +++ b/internal/config/sync_dir_unix.go @@ -0,0 +1,16 @@ +//go:build !windows + +package config + +import ( + "errors" + "os" +) + +func syncDirectory(path string) error { + dir, err := os.Open(path) + if err != nil { + return err + } + return errors.Join(dir.Sync(), dir.Close()) +} diff --git a/internal/config/sync_dir_windows.go b/internal/config/sync_dir_windows.go new file mode 100644 index 0000000..c2b8035 --- /dev/null +++ b/internal/config/sync_dir_windows.go @@ -0,0 +1,9 @@ +//go:build windows + +package config + +// Windows does not support opening a directory and calling Sync on it. The +// renamed file itself has already been flushed before this platform hook. +func syncDirectory(_ string) error { + return nil +} diff --git a/internal/download/audio.go b/internal/download/audio.go index f241c4a..5ecf99f 100644 --- a/internal/download/audio.go +++ b/internal/download/audio.go @@ -148,11 +148,6 @@ func ExtractBitrate(manUrl string) string { return "" } -// ParseHlsMaster parses an HLS master playlist and selects the best variant. -func ParseHlsMaster(qual *model.Quality) error { - return ParseHlsMasterContext(context.Background(), qual) -} - // ParseHlsMasterContext fetches and parses a master playlist with cancellation. func ParseHlsMasterContext(ctx context.Context, qual *model.Quality) error { httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, qual.URL, nil) @@ -196,11 +191,6 @@ func ParseHlsMasterContext(ctx context.Context, qual *model.Quality) error { return nil } -// GetKey retrieves an AES encryption key from the given URL. -func GetKey(keyUrl string) ([]byte, error) { - return GetKeyContext(context.Background(), keyUrl) -} - // GetKeyContext retrieves an AES key with cancellation. func GetKeyContext(ctx context.Context, keyUrl string) ([]byte, error) { httpReq, err := http.NewRequestWithContext(ctx, http.MethodGet, keyUrl, nil) @@ -256,11 +246,6 @@ func DecryptTrack(tempPath string, key, iv []byte) ([]byte, error) { return decrypted, nil } -// TsToAac converts a decrypted TS stream to AAC using ffmpeg. -func TsToAac(decData []byte, outPath, ffmpegNameStr string) error { - return TsToAacContext(context.Background(), decData, outPath, ffmpegNameStr) -} - // TsToAacContext converts a decrypted TS stream and cancels ffmpeg with ctx. func TsToAacContext(ctx context.Context, decData []byte, outPath, ffmpegNameStr string) error { var errBuffer bytes.Buffer @@ -348,11 +333,6 @@ func DecryptTrackToFile(tempPath, decryptedPath string, key, iv []byte) error { return errors.New("no encrypted data to decrypt") } -// TsFileToAac converts a decrypted TS file to AAC using ffmpeg. -func TsFileToAac(decPath, outPath, ffmpegNameStr string) error { - return TsFileToAacContext(context.Background(), decPath, outPath, ffmpegNameStr) -} - // TsFileToAacContext converts a file and cancels ffmpeg with ctx. func TsFileToAacContext(ctx context.Context, decPath, outPath, ffmpegNameStr string) error { var errBuffer bytes.Buffer @@ -917,7 +897,7 @@ func downloadAlbumAudio(ctx context.Context, tracks []model.Track, albumPath, ar progressBox.TotalDuration = time.Since(progressBox.StartTime) progressBox.Mu.Unlock() } - if cfg.RcloneEnabled { + if cfg.RcloneEnabled && len(failures) == 0 { if err := deps.UploadPath(ctx, albumPath, artistFolder, cfg, progressBox, false); err != nil { helpers.ReportErr("Upload failed.", err) failures = append(failures, fmt.Errorf("upload album: %w", err)) diff --git a/internal/download/audio_album_test.go b/internal/download/audio_album_test.go index a7717a8..798ec23 100644 --- a/internal/download/audio_album_test.go +++ b/internal/download/audio_album_test.go @@ -2,8 +2,12 @@ package download import ( "context" + "errors" + "net/http" + "strings" "testing" + "github.com/jmagar/nugs-cli/internal/api" "github.com/jmagar/nugs-cli/internal/model" ) @@ -75,6 +79,64 @@ func TestResolveAlbumDownloadModes(t *testing.T) { } } +func TestDownloadAlbumAudioDoesNotUploadPartialAlbum(t *testing.T) { + errDownload := errors.New("download failed") + client := &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return nil, errDownload + })} + ctx := api.WithHTTPClient(context.Background(), client) + uploads := 0 + deps := &Deps{UploadToRclone: func(context.Context, string, string, *model.Config, *model.ProgressBoxState, bool) error { + uploads++ + return nil + }} + cfg := &model.Config{Format: 2, RcloneEnabled: true} + tracks := []model.Track{{ + TrackID: 1, + SongTitle: "failed track", + TrackURL: "https://cdn.example.test/audio.flac16/failed.flac", + }} + + err := downloadAlbumAudio(ctx, tracks, t.TempDir(), "artist", cfg, &model.StreamParams{}, nil, false, deps) + if !errors.Is(err, errDownload) { + t.Fatalf("downloadAlbumAudio error = %v, want wrapped download failure", err) + } + if uploads != 0 { + t.Fatalf("upload count = %d, want 0 for a partial album", uploads) + } +} + +func TestProcessArtistAlbumsReturnsJoinedFailures(t *testing.T) { + artist := &model.ArtistMeta{} + artist.Response.Containers = []*model.AlbArtResp{ + {ContainerID: 101, ContainerInfo: "First failed album"}, + {ContainerID: 202, ContainerInfo: "Second failed album"}, + } + batchState := &model.BatchProgressState{TotalAlbums: 2} + progressBox := &model.ProgressBoxState{BatchState: batchState} + + err := processArtistAlbums( + context.Background(), + []*model.ArtistMeta{artist}, + &model.Config{SkipVideos: true}, + &model.StreamParams{}, + batchState, + progressBox, + &Deps{}, + ) + if !errors.Is(err, model.ErrReleaseHasNoContent) { + t.Fatalf("processArtistAlbums error = %v, want joined ErrReleaseHasNoContent", err) + } + for _, want := range []string{"First failed album", "Second failed album"} { + if !strings.Contains(err.Error(), want) { + t.Fatalf("processArtistAlbums error %q does not include %q", err, want) + } + } + if batchState.Failed != 2 || batchState.Complete != 0 { + t.Fatalf("batch counts = failed %d, complete %d; want 2, 0", batchState.Failed, batchState.Complete) + } +} + func TestBuildAlbumShowNumber(t *testing.T) { meta := &model.AlbArtResp{PerformanceDateShort: "2026-01-01"} diff --git a/internal/download/batch.go b/internal/download/batch.go index 0039cca..59b95ee 100644 --- a/internal/download/batch.go +++ b/internal/download/batch.go @@ -63,6 +63,7 @@ func prepareBatchProgress(meta []*model.ArtistMeta, cfg *model.Config, deps *Dep // processArtistAlbums iterates over all containers in the artist metadata, downloading each album. func processArtistAlbums(ctx context.Context, meta []*model.ArtistMeta, cfg *model.Config, streamParams *model.StreamParams, batchState *model.BatchProgressState, progressBox *model.ProgressBoxState, deps *Deps) error { albumCount := 0 + var failures []error for _, m := range meta { for _, container := range m.Response.Containers { if deps.WaitIfPausedOrCancelled != nil { @@ -93,6 +94,8 @@ func processArtistAlbums(ctx context.Context, meta []*model.ArtistMeta, cfg *mod } ui.PrintError(fmt.Sprintf("Album %d/%d failed (ID %d, %s): %v", albumCount, batchState.TotalAlbums, container.ContainerID, container.ContainerInfo, err)) + failures = append(failures, fmt.Errorf("album %d/%d (ID %d, %s): %w", + albumCount, batchState.TotalAlbums, container.ContainerID, container.ContainerInfo, err)) } else { progressBox.Mu.Lock() batchState.Complete++ @@ -100,7 +103,7 @@ func processArtistAlbums(ctx context.Context, meta []*model.ArtistMeta, cfg *mod } } } - return nil + return errors.Join(failures...) } // Playlist downloads all tracks in a playlist. diff --git a/internal/download/regression_test.go b/internal/download/regression_test.go index 9faa42e..9c211f4 100644 --- a/internal/download/regression_test.go +++ b/internal/download/regression_test.go @@ -58,7 +58,6 @@ func TestProcessTrack_QualityFallback_NoHang(t *testing.T) { for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { - t.Parallel() client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { switch { case strings.Contains(req.URL.Path, "/bigriver/subPlayer.aspx"): @@ -69,25 +68,16 @@ func TestProcessTrack_QualityFallback_NoHang(t *testing.T) { return nil, fmt.Errorf("unexpected request: %s", req.URL.String()) } })} - ctx := api.WithHTTPClient(context.Background(), client) + ctx, cancel := context.WithTimeout(api.WithHTTPClient(context.Background(), client), 5*time.Second) + defer cancel() dir := t.TempDir() cfg := &model.Config{Format: tc.wantFmt} track := &model.Track{TrackID: 1234, SongTitle: "Track Name"} streamParams := &model.StreamParams{} - errCh := make(chan error, 1) - go func() { - errCh <- ProcessTrack(ctx, dir, 1, 1, cfg, track, streamParams, nil, &Deps{}) - }() - - select { - case err := <-errCh: - if err != nil { - t.Fatalf("ProcessTrack returned error: %v", err) - } - case <-time.After(2 * time.Second): - t.Fatal("ProcessTrack timed out (possible fallback loop regression)") + if err := ProcessTrack(ctx, dir, 1, 1, cfg, track, streamParams, nil, &Deps{}); err != nil { + t.Fatalf("ProcessTrack returned error: %v", err) } matches, err := filepath.Glob(filepath.Join(dir, "*"+tc.wantExt)) diff --git a/internal/download/video.go b/internal/download/video.go index decb088..0bc53a4 100644 --- a/internal/download/video.go +++ b/internal/download/video.go @@ -222,11 +222,6 @@ func GetManifestBase(manifestUrl string) (string, string, error) { return base, "?" + u.RawQuery, nil } -// GetSegUrls retrieves segment URLs from an HLS media playlist. -func GetSegUrls(manifestUrl, query string) ([]string, error) { - return GetSegUrlsContext(context.Background(), manifestUrl, query) -} - // GetSegUrlsContext retrieves media segments with cancellation. func GetSegUrlsContext(ctx context.Context, manifestUrl, query string) ([]string, error) { var segUrls []string @@ -400,11 +395,6 @@ func ParseDuration(dur string) (int, error) { return int(rounded), nil } -// GetDuration gets the duration of a TS file using ffmpeg. -func GetDuration(tsPath, ffmpegNameStr string) (int, error) { - return GetDurationContext(context.Background(), tsPath, ffmpegNameStr) -} - // GetDurationContext reads duration while allowing ffmpeg cancellation. func GetDurationContext(ctx context.Context, tsPath, ffmpegNameStr string) (int, error) { var errBuffer bytes.Buffer @@ -512,12 +502,6 @@ func WriteChapsFile(chapters []any, dur int) (string, error) { return chapsPath, nil } -// TsToMp4 converts a TS file to MP4 using ffmpeg, optionally including chapters. -// chapsFilePath is the path to the chapters metadata file (empty if no chapters). -func TsToMp4(vidPathTs, vidPath, ffmpegNameStr, chapsFilePath string) error { - return TsToMp4Context(context.Background(), vidPathTs, vidPath, ffmpegNameStr, chapsFilePath) -} - // TsToMp4Context converts a TS file and cancels ffmpeg with ctx. func TsToMp4Context(ctx context.Context, vidPathTs, vidPath, ffmpegNameStr, chapsFilePath string) error { var ( diff --git a/internal/helpers/paths.go b/internal/helpers/paths.go index f5a7700..b744eb2 100644 --- a/internal/helpers/paths.go +++ b/internal/helpers/paths.go @@ -13,7 +13,7 @@ import ( "github.com/jmagar/nugs-cli/internal/model" ) -var sanRegex = regexp.MustCompile(`[\/:*?"><|]`) +var sanRegex = regexp.MustCompile(`[\\/:*?"><|]`) var ( // ErrInvalidPathCharacters indicates control characters were found in a path. @@ -224,7 +224,12 @@ func (r *ConfigPathResolver) LocalShowPath(show *model.AlbArtResp, mediaType mod return "" } albumFolder := BuildAlbumFolderName(show.ArtistName, show.ContainerInfo) - return filepath.Join(r.LocalBaseForMedia(mediaType), Sanitise(show.ArtistName), albumFolder) + root := r.LocalBaseForMedia(mediaType) + showPath, err := JoinWithinRoot(root, Sanitise(show.ArtistName), albumFolder) + if err != nil { + return "" + } + return showPath } // RemoteShowPath returns the artist/show remote-relative path. diff --git a/internal/helpers/paths_security_test.go b/internal/helpers/paths_security_test.go index 2136b7c..78ab05f 100644 --- a/internal/helpers/paths_security_test.go +++ b/internal/helpers/paths_security_test.go @@ -16,6 +16,13 @@ func TestSanitiseReservedPathComponents(t *testing.T) { } } +func TestSanitiseWindowsSeparators(t *testing.T) { + got := Sanitise(`..\..\outside`) + if strings.ContainsAny(got, `/\`) { + t.Fatalf("Sanitise retained Windows traversal components: %q", got) + } +} + func TestLocalShowPathConfinedToConfiguredRoot(t *testing.T) { root := t.TempDir() resolver := NewConfigPathResolver(&model.Config{OutPath: root}) @@ -26,6 +33,22 @@ func TestLocalShowPathConfinedToConfiguredRoot(t *testing.T) { } } +func TestLocalShowPathConfinesWindowsStyleMetadata(t *testing.T) { + root := t.TempDir() + resolver := NewConfigPathResolver(&model.Config{OutPath: root}) + got := resolver.LocalShowPath(&model.AlbArtResp{ + ArtistName: `..\..\outside`, + ContainerInfo: `..\..\show`, + }, model.MediaTypeAudio) + if strings.Contains(got, `\`) { + t.Fatalf("path retained Windows separators: %q", got) + } + rel, err := filepath.Rel(root, got) + if err != nil || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + t.Fatalf("path escaped root: %q (rel %q, err %v)", got, rel, err) + } +} + func TestJoinWithinRootRejectsTraversal(t *testing.T) { if _, err := JoinWithinRoot(t.TempDir(), "..", "escape"); err == nil { t.Fatal("JoinWithinRoot accepted traversal") diff --git a/internal/notify/gotify.go b/internal/notify/gotify.go index b09a418..bc3e805 100644 --- a/internal/notify/gotify.go +++ b/internal/notify/gotify.go @@ -30,6 +30,9 @@ func validateServerURL(raw string) (*url.URL, error) { if u.User != nil { return nil, fmt.Errorf("gotify: server URL must not contain userinfo") } + if u.RawQuery != "" || u.Fragment != "" { + return nil, fmt.Errorf("gotify: server URL must not contain query parameters or fragments") + } if u.Scheme == "https" { return u, nil } @@ -51,7 +54,9 @@ func Send(ctx context.Context, serverURL, token, title, message string, priority if err != nil { return err } - requestURL := strings.TrimRight(base.String(), "/") + "/message" + base.Path = strings.TrimRight(base.Path, "/") + "/message" + base.RawPath = "" + requestURL := base.String() body, err := json.Marshal(map[string]any{ "title": title, diff --git a/internal/notify/gotify_security_test.go b/internal/notify/gotify_security_test.go index 4be0358..16f161b 100644 --- a/internal/notify/gotify_security_test.go +++ b/internal/notify/gotify_security_test.go @@ -39,3 +39,29 @@ func TestSendDoesNotForwardTokenAcrossRedirect(t *testing.T) { t.Fatal("redirect target was contacted") } } + +func TestSendBuildsMessageURLStructurally(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/gotify/message" { + t.Errorf("request path = %q, want /gotify/message", r.URL.Path) + } + if r.URL.RawQuery != "" || r.URL.Fragment != "" { + t.Errorf("request URL retained query or fragment: %s", r.URL.String()) + } + w.WriteHeader(http.StatusOK) + })) + defer server.Close() + + if err := Send(context.Background(), server.URL+"/gotify/", "secret", "title", "body", 1); err != nil { + t.Fatalf("Send() error = %v", err) + } +} + +func TestSendRejectsServerURLQueryAndFragment(t *testing.T) { + for _, suffix := range []string{"?redirect=evil", "#fragment"} { + err := Send(context.Background(), "http://localhost/"+suffix, "secret", "title", "body", 1) + if err == nil { + t.Fatalf("Send accepted server URL suffix %q", suffix) + } + } +} diff --git a/internal/rclone/concurrency_test.go b/internal/rclone/concurrency_test.go index 00141b2..c319481 100644 --- a/internal/rclone/concurrency_test.go +++ b/internal/rclone/concurrency_test.go @@ -15,7 +15,7 @@ import ( func TestListArtistFoldersPreservesLastGoodBulkIndex(t *testing.T) { adapter := NewStorageAdapter() adapter.validatePath = func(string) error { return nil } - adapter.commandContext = func(context.Context, string, ...string) *exec.Cmd { return exec.Command("true") } + adapter.commandContext = func(ctx context.Context, _ string, _ ...string) *exec.Cmd { return exec.CommandContext(ctx, "true") } calls := 0 adapter.outputCommand = func(*exec.Cmd) ([]byte, error) { calls++ @@ -31,15 +31,16 @@ func TestListArtistFoldersPreservesLastGoodBulkIndex(t *testing.T) { t.Fatalf("first listing = %v, %v", first, err) } stale, err := adapter.ListArtistFolders(context.Background(), cfg, "artist", false) - if err != nil || len(stale) != 2 { - t.Fatalf("stale listing = %v, %v", stale, err) + var degraded *StaleFolderListingError + if !errors.As(err, °raded) || len(stale) != 2 { + t.Fatalf("stale listing = %v, %v; want degraded error", stale, err) } } func TestRcloneProcessesAreGloballyBounded(t *testing.T) { adapter := NewStorageAdapter() adapter.validatePath = func(string) error { return nil } - adapter.commandContext = func(context.Context, string, ...string) *exec.Cmd { return exec.Command("true") } + adapter.commandContext = func(ctx context.Context, _ string, _ ...string) *exec.Cmd { return exec.CommandContext(ctx, "true") } var active atomic.Int32 var peak atomic.Int32 release := make(chan struct{}) @@ -71,3 +72,29 @@ func TestRcloneProcessesAreGloballyBounded(t *testing.T) { close(release) wg.Wait() } + +func TestWithProcessSlotHonorsCanceledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + called := false + err := WithProcessSlot(ctx, func() error { + called = true + return nil + }) + if !errors.Is(err, context.Canceled) { + t.Fatalf("WithProcessSlot error = %v, want context.Canceled", err) + } + if called { + t.Fatal("process callback ran after cancellation") + } +} + +func TestRcloneChecksRequireCallerContext(t *testing.T) { + var missingContext context.Context + if err := CheckRcloneAvailable(missingContext, true); err == nil { + t.Fatal("CheckRcloneAvailable accepted nil context") + } + if got := CheckRclonePathOnline(missingContext, &model.Config{RcloneEnabled: true}); got != "Offline (context required)" { + t.Fatalf("CheckRclonePathOnline(nil) = %q", got) + } +} diff --git a/internal/rclone/rclone.go b/internal/rclone/rclone.go index 60a0ab0..3f2883e 100644 --- a/internal/rclone/rclone.go +++ b/internal/rclone/rclone.go @@ -35,10 +35,13 @@ var ( var defaultStorageProvider model.StorageProvider = NewStorageAdapter() // CheckRcloneAvailable verifies rclone is installed and available in PATH. -func CheckRcloneAvailable(quiet bool) error { - cmd := exec.Command("rclone", "version") +func CheckRcloneAvailable(ctx context.Context, quiet bool) error { + if ctx == nil { + return fmt.Errorf("rclone availability context is required") + } + cmd := exec.CommandContext(ctx, "rclone", "version") var output []byte - err := WithProcessSlot(context.Background(), func() error { + err := WithProcessSlot(ctx, func() error { var runErr error output, runErr = cmd.Output() return runErr @@ -59,7 +62,10 @@ func CheckRcloneAvailable(quiet bool) error { } // CheckRclonePathOnline checks if the configured rclone remote is reachable. -func CheckRclonePathOnline(cfg *model.Config) string { +func CheckRclonePathOnline(ctx context.Context, cfg *model.Config) string { + if ctx == nil { + return "Offline (context required)" + } if !cfg.RcloneEnabled { return "Disabled" } @@ -67,7 +73,7 @@ func CheckRclonePathOnline(cfg *model.Config) string { return "Offline (remote not configured)" } target := cfg.RcloneRemote + ":" + helpers.GetRcloneBasePath(cfg, false) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) defer cancel() cmd := exec.CommandContext(ctx, "rclone", "lsf", target) err := WithProcessSlot(ctx, cmd.Run) @@ -108,11 +114,6 @@ func UploadToRclone(ctx context.Context, localPath string, artistFolder string, }) } -// BuildRcloneUploadCommand constructs the rclone copy/copyto command. -func BuildRcloneUploadCommand(localPath, artistFolder string, cfg *model.Config, transfers int, isVideo bool) (*exec.Cmd, string, error) { - return BuildRcloneUploadCommandContext(context.Background(), localPath, artistFolder, cfg, transfers, isVideo) -} - // BuildRcloneUploadCommandContext constructs the rclone copy/copyto command with context support. func BuildRcloneUploadCommandContext(ctx context.Context, localPath, artistFolder string, cfg *model.Config, transfers int, isVideo bool) (*exec.Cmd, string, error) { remoteDest := cfg.RcloneRemote + ":" + helpers.GetRcloneBasePath(cfg, isVideo) @@ -333,11 +334,6 @@ func RunRcloneWithProgress(cmd *exec.Cmd, onProgress func(percent int, speed, up return waitErr } -// BuildRcloneVerifyCommand constructs the rclone check command for upload verification. -func BuildRcloneVerifyCommand(localPath, remoteFullPath string) (*exec.Cmd, error) { - return BuildRcloneVerifyCommandContext(context.Background(), localPath, remoteFullPath) -} - // BuildRcloneVerifyCommandContext constructs a cancellable rclone check command. func BuildRcloneVerifyCommandContext(ctx context.Context, localPath, remoteFullPath string) (*exec.Cmd, error) { localInfo, err := os.Stat(localPath) diff --git a/internal/rclone/storage_adapter.go b/internal/rclone/storage_adapter.go index 29704a6..400b181 100644 --- a/internal/rclone/storage_adapter.go +++ b/internal/rclone/storage_adapter.go @@ -25,6 +25,12 @@ var ( ) func acquireRcloneSlot(ctx context.Context) (func(), error) { + if ctx == nil { + return nil, fmt.Errorf("rclone process context is required") + } + if err := ctx.Err(); err != nil { + return nil, err + } select { case rcloneProcessSlots <- struct{}{}: return func() { <-rcloneProcessSlots }, nil @@ -37,7 +43,7 @@ func acquireRcloneSlot(ctx context.Context) (func(), error) { // slot. Catalog bulk scans use this too, so every rclone spawn shares one cap. func WithProcessSlot(ctx context.Context, fn func() error) error { if ctx == nil { - ctx = context.Background() + return fmt.Errorf("rclone process context is required") } release, err := acquireRcloneSlot(ctx) if err != nil { @@ -113,7 +119,7 @@ func (a *StorageAdapter) Upload(ctx context.Context, cfg *model.Config, req mode } if ctx == nil { - ctx = context.Background() + return fmt.Errorf("upload context is required") } release, err := acquireRcloneSlot(ctx) if err != nil { @@ -209,7 +215,7 @@ func (a *StorageAdapter) PathExists(ctx context.Context, cfg *model.Config, remo fullPath := remoteDest + "/" + remotePath if ctx == nil { - ctx = context.Background() + return false, fmt.Errorf("path check context is required") } timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() @@ -246,7 +252,7 @@ func (a *StorageAdapter) ListArtistFolders(ctx context.Context, cfg *model.Confi remoteDest := cfg.RcloneRemote + ":" + a.getRcloneBasePath(cfg, isVideo) fullPath := remoteDest + "/" + artistFolder if ctx == nil { - ctx = context.Background() + return nil, fmt.Errorf("folder listing context is required") } timeoutCtx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() @@ -265,7 +271,7 @@ func (a *StorageAdapter) ListArtistFolders(ctx context.Context, cfg *model.Confi stale, ok := folderCache[fullPath] folderCacheMu.RUnlock() if ok { - return cloneFolderSet(stale), nil + return cloneFolderSet(stale), &StaleFolderListingError{Path: fullPath, Cause: err} } return nil, fmt.Errorf("failed to list remote artist folders: %w", err) } @@ -283,6 +289,20 @@ func (a *StorageAdapter) ListArtistFolders(ctx context.Context, cfg *model.Confi return folders, nil } +// StaleFolderListingError indicates that Folders contains the last successful +// listing, but callers must treat absent entries as unknown and fall back to +// individual path checks. +type StaleFolderListingError struct { + Path string + Cause error +} + +func (e *StaleFolderListingError) Error() string { + return fmt.Sprintf("using stale folder listing for %s after refresh failed: %v", e.Path, e.Cause) +} + +func (e *StaleFolderListingError) Unwrap() error { return e.Cause } + func cloneFolderSet(source map[string]struct{}) map[string]struct{} { clone := make(map[string]struct{}, len(source)) for value := range source { diff --git a/internal/rclone/storage_adapter_test.go b/internal/rclone/storage_adapter_test.go index d98f312..23448ec 100644 --- a/internal/rclone/storage_adapter_test.go +++ b/internal/rclone/storage_adapter_test.go @@ -26,7 +26,7 @@ func TestStorageAdapterUploadWithHooksAndDelete(t *testing.T) { if !isVideo { t.Fatal("expected video upload request") } - return exec.Command("echo"), "remote:path/artist/local", nil + return exec.CommandContext(ctx, "echo"), "remote:path/artist/local", nil } adapter.runWithProgress = func(cmd *exec.Cmd, onProgress UploadProgressFunc) error { if onProgress == nil { @@ -35,11 +35,11 @@ func TestStorageAdapterUploadWithHooksAndDelete(t *testing.T) { onProgress(55, "8 MiB/s", "440 MiB", "800 MiB") return nil } - adapter.buildVerifyCommand = func(_ context.Context, localPath, remoteFullPath string) (*exec.Cmd, error) { + adapter.buildVerifyCommand = func(ctx context.Context, localPath, remoteFullPath string) (*exec.Cmd, error) { if remoteFullPath != "remote:path/artist/local" { t.Fatalf("verify remote path = %q", remoteFullPath) } - return exec.Command("echo"), nil + return exec.CommandContext(ctx, "echo"), nil } adapter.runCommand = func(*exec.Cmd) error { return nil } deletedPath := "" @@ -97,7 +97,7 @@ func TestStorageAdapterPathExistsHandlesExitCodeThree(t *testing.T) { adapter := NewStorageAdapter() adapter.validatePath = func(string) error { return nil } adapter.commandContext = func(ctx context.Context, name string, args ...string) *exec.Cmd { - return exec.Command("echo") + return exec.CommandContext(ctx, "echo") } notFoundErr := errors.New("missing") adapter.runCommand = func(*exec.Cmd) error { return notFoundErr } @@ -125,7 +125,7 @@ func TestStorageAdapterListArtistFoldersParsesOutput(t *testing.T) { adapter := NewStorageAdapter() adapter.validatePath = func(string) error { return nil } adapter.commandContext = func(ctx context.Context, name string, args ...string) *exec.Cmd { - return exec.Command("echo") + return exec.CommandContext(ctx, "echo") } adapter.outputCommand = func(*exec.Cmd) ([]byte, error) { return []byte("2026-01-01/\n2026-01-02/\n\n"), nil diff --git a/internal/runtime/detach_unix.go b/internal/runtime/detach_unix.go index 7c117bc..d00370b 100644 --- a/internal/runtime/detach_unix.go +++ b/internal/runtime/detach_unix.go @@ -25,7 +25,7 @@ func SpawnDetached(args []string) (int, string, error) { return 0, "", err } logPath := filepath.Join(cacheDir, "runtime.log") - logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + logFile, err := openRuntimeLog(logPath) if err != nil { return 0, "", err } @@ -43,6 +43,18 @@ func SpawnDetached(args []string) (int, string, error) { return cmd.Process.Pid, logPath, nil } +func openRuntimeLog(logPath string) (*os.File, error) { + logFile, err := os.OpenFile(logPath, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600) + if err != nil { + return nil, err + } + if err := logFile.Chmod(0600); err != nil { + _ = logFile.Close() + return nil, fmt.Errorf("secure runtime log: %w", err) + } + return logFile, nil +} + // MaybeDetachAndExit checks if the process should detach and, if so, spawns // a background process and returns true. The caller should exit if true. func MaybeDetachAndExit(args []string, urls []string) bool { diff --git a/internal/runtime/detach_unix_test.go b/internal/runtime/detach_unix_test.go new file mode 100644 index 0000000..60b59e0 --- /dev/null +++ b/internal/runtime/detach_unix_test.go @@ -0,0 +1,30 @@ +//go:build !windows + +package runtime + +import ( + "os" + "path/filepath" + "testing" +) + +func TestOpenRuntimeLogSecuresExistingFile(t *testing.T) { + logPath := filepath.Join(t.TempDir(), "runtime.log") + if err := os.WriteFile(logPath, []byte("old log\n"), 0644); err != nil { + t.Fatal(err) + } + logFile, err := openRuntimeLog(logPath) + if err != nil { + t.Fatal(err) + } + if err := logFile.Close(); err != nil { + t.Fatal(err) + } + info, err := os.Stat(logPath) + if err != nil { + t.Fatal(err) + } + if got := info.Mode().Perm(); got != 0600 { + t.Fatalf("runtime.log mode = %o, want 600", got) + } +} diff --git a/scripts/check-docs.py b/scripts/check-docs.py index 8570d53..e4e0e5c 100644 --- a/scripts/check-docs.py +++ b/scripts/check-docs.py @@ -3,6 +3,7 @@ from __future__ import annotations +import html import re import sys from pathlib import Path @@ -16,16 +17,37 @@ LINK_RE = re.compile(r"(? set[str]: + anchors: set[str] = set() + occurrences: dict[str, int] = {} + for line in text.splitlines(): + match = HEADING_RE.match(line) + if not match: + continue + heading = html.unescape(match.group(1)) + heading = re.sub(r"<[^>]+>", "", heading) + heading = re.sub(r"[`*_~]", "", heading).strip().lower() + slug = re.sub(r"[^\w\- ]", "", heading, flags=re.UNICODE) + slug = re.sub(r"\s+", "-", slug) + duplicate = occurrences.get(slug, 0) + occurrences[slug] = duplicate + 1 + anchors.add(slug if duplicate == 0 else f"{slug}-{duplicate}") + return anchors def local_link_errors(path: Path, text: str) -> list[str]: errors: list[str] = [] for raw_target in LINK_RE.findall(text): target = raw_target.strip().split(maxsplit=1)[0].strip("<>") - if not target or target.startswith(("#", "http://", "https://", "mailto:")): + if not target or target.startswith(("http://", "https://", "mailto:")): continue - file_part = unquote(target.split("#", 1)[0]) - resolved = (path.parent / file_part).resolve() + file_part, _, fragment = target.partition("#") + file_part = unquote(file_part) + fragment = unquote(fragment) + resolved = path.resolve() if not file_part else (path.parent / file_part).resolve() try: resolved.relative_to(ROOT) except ValueError: @@ -33,6 +55,13 @@ def local_link_errors(path: Path, text: str) -> list[str]: continue if not resolved.exists(): errors.append(f"{path.relative_to(ROOT)}: missing local link target: {target}") + continue + if fragment and resolved.suffix.lower() == ".md": + anchors = heading_anchors(resolved.read_text(encoding="utf-8")) + if fragment not in anchors: + errors.append( + f"{path.relative_to(ROOT)}: missing Markdown anchor: {target}" + ) return errors diff --git a/scripts/test_check_docs.py b/scripts/test_check_docs.py new file mode 100644 index 0000000..871ebad --- /dev/null +++ b/scripts/test_check_docs.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import importlib.util +import tempfile +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).with_name("check-docs.py") +SPEC = importlib.util.spec_from_file_location("check_docs", SCRIPT) +assert SPEC and SPEC.loader +check_docs = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(check_docs) + + +class CheckDocsTests(unittest.TestCase): + def test_heading_anchors_match_github_duplicates(self) -> None: + anchors = check_docs.heading_anchors("# Hello, World!\n## Hello, World!\n") + self.assertEqual(anchors, {"hello-world", "hello-world-1"}) + + def test_local_link_errors_reports_missing_anchor(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "README.md" + target = root / "guide.md" + source.write_text("[broken](guide.md#missing)\n", encoding="utf-8") + target.write_text("# Present\n", encoding="utf-8") + old_root = check_docs.ROOT + check_docs.ROOT = root + try: + errors = check_docs.local_link_errors(source, source.read_text()) + finally: + check_docs.ROOT = old_root + self.assertEqual(len(errors), 1) + self.assertIn("missing Markdown anchor", errors[0]) + + def test_local_link_errors_accepts_same_page_anchor(self) -> None: + with tempfile.TemporaryDirectory() as temp: + root = Path(temp) + source = root / "README.md" + source.write_text("# Present\n[ok](#present)\n", encoding="utf-8") + old_root = check_docs.ROOT + check_docs.ROOT = root + try: + errors = check_docs.local_link_errors(source, source.read_text()) + finally: + check_docs.ROOT = old_root + self.assertEqual(errors, []) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/test_repository_contracts.py b/scripts/test_repository_contracts.py new file mode 100644 index 0000000..bebffeb --- /dev/null +++ b/scripts/test_repository_contracts.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import re +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent + + +class RepositoryContractTests(unittest.TestCase): + def test_mise_matches_go_directive(self) -> None: + go_mod = (ROOT / "go.mod").read_text(encoding="utf-8") + mise = (ROOT / ".mise.toml").read_text(encoding="utf-8") + go_version = re.search(r"^go (\S+)$", go_mod, re.MULTILINE) + mise_version = re.search(r'^go = "([^"]+)"$', mise, re.MULTILINE) + self.assertIsNotNone(go_version) + self.assertIsNotNone(mise_version) + self.assertEqual(go_version.group(1), mise_version.group(1)) + + def test_linux_arm_release_is_natively_smoke_tested(self) -> None: + workflow = (ROOT / ".github/workflows/release.yml").read_text(encoding="utf-8") + arm_entry = workflow.split("- target: linux-arm64", 1)[1].split("- target:", 1)[0] + self.assertIn("runner: ubuntu-24.04-arm", arm_entry) + self.assertIn("smoke: true", arm_entry) + + def test_openwiki_publish_mirrors_generated_tree(self) -> None: + workflow = (ROOT / ".github/workflows/openwiki-update.yml").read_text(encoding="utf-8") + clear = workflow.index("rm -rf -- openwiki") + download = workflow.index("name: Download generated documentation") + self.assertLess(clear, download) + self.assertIn("git add -A -- openwiki", workflow) + self.assertIn("title=Missing OpenWiki API key", workflow) + + +if __name__ == "__main__": + unittest.main() From 84c15068c6c8c5f6765ea43ecbf6a91b6eb2db88 Mon Sep 17 00:00:00 2001 From: Jacob Magar Date: Sat, 18 Jul 2026 02:54:25 -0400 Subject: [PATCH 3/3] fix: close final PR review gaps --- cmd/nugs/main.go | 20 +++++++------- cmd/nugs/main_flow_test.go | 11 ++++++++ internal/download/audio.go | 3 ++- internal/download/precalculate_test.go | 37 ++++++++++++++++++++++++++ 4 files changed, 61 insertions(+), 10 deletions(-) diff --git a/cmd/nugs/main.go b/cmd/nugs/main.go index 891e5fa..cad4b23 100644 --- a/cmd/nugs/main.go +++ b/cmd/nugs/main.go @@ -170,15 +170,7 @@ func run(cfg *Config, jsonLevel string) (runErr error) { if !trackRuntime { return } - if runCancelled { - finalizeRuntimeStatus("cancelled") - return - } - if runErr != nil { - finalizeRuntimeStatus("failed") - return - } - finalizeRuntimeStatus("completed") + finalizeRuntimeStatus(runtimeFinalState(runCancelled, runErr)) }() stopHotkeys := startCrawlHotkeysIfNeeded(cfg.Urls) defer stopHotkeys() @@ -247,6 +239,16 @@ func run(cfg *Config, jsonLevel string) (runErr error) { return runErr } +func runtimeFinalState(cancelled bool, err error) string { + if cancelled || isCrawlCancelledErr(err) { + return "cancelled" + } + if err != nil { + return "failed" + } + return "completed" +} + func authenticateForDownloads(ctx context.Context, cfg *Config) (*StreamParams, string, string, error) { if err := makeDirs(cfg.OutPath); err != nil { return nil, "", "", fmt.Errorf("failed to make output folder: %w", err) diff --git a/cmd/nugs/main_flow_test.go b/cmd/nugs/main_flow_test.go index ba08ab1..29a158d 100644 --- a/cmd/nugs/main_flow_test.go +++ b/cmd/nugs/main_flow_test.go @@ -2,6 +2,8 @@ package main import ( "context" + "errors" + "fmt" "os" "strings" "testing" @@ -88,3 +90,12 @@ func TestCommandValidationReturnsErrors(t *testing.T) { }) } } + +func TestRuntimeFinalStateRecognizesWrappedCancellation(t *testing.T) { + if got := runtimeFinalState(false, fmt.Errorf("watch check: %w", ErrCrawlCancelled)); got != "cancelled" { + t.Fatalf("runtimeFinalState() = %q, want cancelled", got) + } + if got := runtimeFinalState(false, errors.New("ordinary failure")); got != "failed" { + t.Fatalf("runtimeFinalState() = %q, want failed", got) + } +} diff --git a/internal/download/audio.go b/internal/download/audio.go index 5ecf99f..1a50c93 100644 --- a/internal/download/audio.go +++ b/internal/download/audio.go @@ -701,8 +701,8 @@ func PreCalculateShowSize(ctx context.Context, tracks []model.Track, streamParam continue } resp, err := api.Do(req) - reqCancel() if err != nil { + reqCancel() continue } if resp.ContentLength > 0 { @@ -712,6 +712,7 @@ func PreCalculateShowSize(ctx context.Context, tracks []model.Track, streamParam } _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 64<<10)) _ = resp.Body.Close() + reqCancel() } } }() diff --git a/internal/download/precalculate_test.go b/internal/download/precalculate_test.go index 14e4de1..292eba0 100644 --- a/internal/download/precalculate_test.go +++ b/internal/download/precalculate_test.go @@ -2,6 +2,7 @@ package download import ( "context" + "io" "net/http" "net/http/httptest" "sync/atomic" @@ -11,6 +12,20 @@ import ( "github.com/jmagar/nugs-cli/internal/model" ) +type contextCheckingBody struct { + ctx context.Context + canceledBeforeClose atomic.Bool +} + +func (b *contextCheckingBody) Read(_ []byte) (int, error) { return 0, io.EOF } + +func (b *contextCheckingBody) Close() error { + if b.ctx.Err() != nil { + b.canceledBeforeClose.Store(true) + } + return nil +} + func TestPreCalculateShowSizeReusesTrackURL(t *testing.T) { var requests atomic.Int32 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { @@ -35,3 +50,25 @@ func TestPreCalculateShowSizeReusesTrackURL(t *testing.T) { t.Fatalf("requests = %d, want one metadata URL HEAD", requests.Load()) } } + +func TestPreCalculateShowSizeClosesResponseBeforeCancel(t *testing.T) { + body := &contextCheckingBody{} + client := &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + body.ctx = req.Context() + return &http.Response{ + StatusCode: http.StatusOK, + Body: body, + ContentLength: 123, + Header: make(http.Header), + Request: req, + }, nil + })} + ctx := api.WithHTTPClient(context.Background(), client) + + if _, err := PreCalculateShowSize(ctx, []model.Track{{TrackID: 42, TrackURL: "https://example.test/track"}}, &model.StreamParams{}, &model.Config{}); err != nil { + t.Fatal(err) + } + if body.canceledBeforeClose.Load() { + t.Fatal("request context was canceled before response body close") + } +}