Harden recovery, delivery safety, and owner-scoped associations

.dockerignore

@@ -4,3 +4,6 @@
 artist-tracker
 artist-trackarr
 coverage.out
+*.tgz
+*.sha256
+.artist-trackarr-last-*

.github/workflows/docker.yml

@@ -61,8 +61,9 @@ jobs:
         with:
           # Keep CI on the patched toolchain used by the Docker build. The
           # go.mod language version remains 1.25 for source compatibility;
-          # reading it verbatim would install the vulnerable 1.25.0 release.
-          go-version: '1.26.x'
+          # pin the patch version so a newly published standard-library
+          # vulnerability cannot silently select an older toolchain.
+          go-version: '1.26.6'
           cache: true
 
       - name: Run race detector

.gitignore

@@ -5,3 +5,6 @@
 /artist-tracker
 /artist-trackarr
 /coverage.out
+*.tgz
+*.sha256
+.artist-trackarr-last-*

Dockerfile

@@ -1,5 +1,5 @@
 # syntax=docker/dockerfile:1.26@sha256:ecfaec9ed6d810b56388c508f4121597bfbba70d41a6dfeee4d8cad5f295fc32
-FROM golang:1.26-alpine@sha256:0178a641fbb4858c5f1b48e34bdaabe0350a330a1b1149aabd498d0699ff5fb2 AS dependencies
+FROM golang:1.26.6-alpine@sha256:af8d6740070b8906d12eae1c3e3ea0957fb63f492051ea05e354c38ef9fe88df AS dependencies
 WORKDIR /src
 RUN apk add --no-cache ca-certificates tzdata
 COPY go.mod go.sum* ./

Makefile

@@ -9,17 +9,18 @@ build:
 	docker build -t artist-trackarr:local .
 
 run:
-	docker compose up --build
+	$(MAKE) build
+	ARTIST_TRACKARR_IMAGE=artist-trackarr:local docker compose up
 
 fmt-check:
 	@test -z "$$(gofmt -l internal cmd)"
 
 tooling-check:
-	@marker=$$(printf '@%s' latest); if rg -n "$$marker" .github Dockerfile scripts; then \
+	@marker=$$(printf '@%s' latest); if git grep -n -F "$$marker" -- .github Dockerfile scripts; then \
 		echo "unpinned tool reference found" >&2; \
 		exit 1; \
 	fi
-	@if rg -n --pcre2 '\b(?:golang|alpine):[^@[:space:]]+(?:[[:space:]]|$$)' Dockerfile scripts; then \
+	@if git grep -n -E '(^|[^[:alnum:]_-])(golang|alpine):[^@[:space:]]+([[:space:]]|$$)' -- Dockerfile scripts; then \
 		echo "unpinned container helper reference found" >&2; \
 		exit 1; \
 	fi

README.md

@@ -45,7 +45,7 @@ releases without creating releases or notifications.
 Use the moon/sun button in the header to switch between light and dark mode;
 your choice is remembered in the browser.
 The running application version and project repository are available in the
-footer. The current release is `v0.41.0`; release images display the injected
+footer. The current release is `v0.42.0`; release images display the injected
 semantic version while local builds identify themselves as `dev`. Operational
 timestamps are stored
 in UTC and rendered in the configured system timezone; existing databases are
@@ -57,7 +57,7 @@ error when a data lookup fails, while the detailed cause remains in structured
 logs. Static assets use immutable, version-stamped URLs and continue to serve
 their unversioned paths for compatibility.
 
-The v0.41.0 persistence-hardening release routes production SQLite writes
+The v0.42.0 reliability-guardrails release routes production SQLite writes
 through a bounded busy/locked retry path, rejects malformed operational
 timestamps instead of silently showing zero values, and preflights database
 paths and MusicBrainz contact input before startup. The v0.40.0 operations
@@ -212,7 +212,7 @@ GitHub Actions builds and publishes the Docker image to
 
 - `latest` and `main` follow the current `main` branch.
 - `sha-<commit>` identifies an exact source revision.
-- Pushing a tag such as `v0.41.0` publishes `0.41.0`, `0.41`, and `latest`.
+- Pushing a tag such as `v0.42.0` publishes `0.42.0`, `0.42`, and `latest`.
 
 Release images receive their version through the Docker build's `APP_VERSION`
 argument. Tag builds inject the semantic tag (without the leading `v`), while
@@ -222,7 +222,7 @@ not confused with a release.
 Pin a deployment to a release by setting the Compose image before starting:
 
 ```console
-ARTIST_TRACKARR_IMAGE=ghcr.io/crypt0rr/artist-trackarr:0.41.0 docker compose up -d
+ARTIST_TRACKARR_IMAGE=ghcr.io/crypt0rr/artist-trackarr:0.42.0 docker compose up -d
 ```
 
 ## Configuration
@@ -348,10 +348,22 @@ filters. Genres come from MusicBrainz tags and are normalized locally.
 
 ## Notification destinations
 
-Users can add guided Email, Discord, Telegram, ntfy, Gotify, and generic
-webhook destinations. Any service supported by Shoutrrr can be added with its
-raw service URL. Credentials are encrypted in SQLite and redacted from the UI
-and logs. Use the **Send test** action after adding a destination.
+Users can add Discord, Telegram, ntfy, and generic HTTP(S) webhook
+destinations. Advanced Shoutrrr URLs are limited to those same audited
+transports; SMTP, Gotify, and unknown schemes are rejected because the
+application cannot apply its connection-time SSRF policy to them. Existing
+legacy destinations remain visible as **Unsupported**, are never contacted,
+and must be replaced explicitly. Credentials are encrypted in SQLite and
+redacted from the UI and logs. Use the **Send test** action after adding a
+destination.
+
+Delivery is at-least-once. A process crash after an external provider accepts a
+message can result in a duplicate, but durable claims and recovery avoid
+silently losing queued work. Paused or unsupported destinations receive a
+blocked queue row instead of disappearing from an event; an administrator or
+owner can retry after replacing/recovering the destination. A newly added
+destination receives future events only and is not backfilled with historical
+notifications.
 
 Users can choose whether albums, EPs, singles, announcements, and release-day
 reminders should be delivered. Followed artists show their last and next
@@ -397,29 +409,42 @@ persistent data directory and is accompanied by a restrictive-permission
 ./scripts/backup.sh artist-trackarr-backup.tgz
 ```
 
+Successful backups write a non-sensitive timestamp marker into the persistent
+volume so administrator diagnostics can show an approximate backup age. The
+marker is archived with the next backup and is not a substitute for an
+off-host backup inventory.
+
 Restore into an empty Compose volume while the app is stopped, keep the
 original `APP_ENCRYPTION_KEY` available, and run the temporary restore
 rehearsal before replacing production data. The key is required to decrypt
 existing notification destinations. Embedded migrations run automatically
 during upgrades; the rehearsal must pass SQLite foreign-key checks and
 `/readyz` before the restored instance is considered usable.
 
-The rehearsal verifies the checksum sidecar when present, fingerprints the
-durable database state, and compares that fingerprint after a clean restart. A
-mismatch fails the rehearsal rather than declaring the restore usable. Legacy
-archives without a sidecar are accepted with a warning; new backups should
-always retain both files.
+The rehearsal requires an immutable image digest (`@sha256:`), verifies the
+checksum sidecar, runs SQLite `integrity_check` and `foreign_key_check`,
+validates that encrypted destinations can be opened with the original key,
+fingerprints the durable logical database state, and compares that fingerprint
+after a clean restart. A mismatch fails the rehearsal rather than declaring the
+restore usable. Legacy archives without a sidecar or mutable images are
+accepted only when explicitly opted in with `RESTORE_ALLOW_LEGACY_ARCHIVE=true`
+or `RESTORE_ALLOW_MUTABLE_IMAGE=true`; new backups should always use the
+immutable path. Backup archives and encryption keys are confidential operator
+artifacts.
 
 The rehearsal uses an isolated Docker volume, starts the selected image,
 stops it with the configured grace period, and starts it again to verify that
 the restored data remains usable:
 
 ```console
 APP_ENCRYPTION_KEY="$APP_ENCRYPTION_KEY" \
-  ARTIST_TRACKARR_IMAGE=ghcr.io/crypt0rr/artist-trackarr:0.41.0 \
+  ARTIST_TRACKARR_IMAGE=ghcr.io/crypt0rr/artist-trackarr@sha256:<release-digest> \
   ./scripts/restore-smoke.sh artist-trackarr-backup.tgz
 ```
 
+The rehearsal records a non-sensitive restore result marker in its temporary
+volume; that volume is removed when the rehearsal exits.
+
 ## Development
 
 The test suite runs in the pinned Go toolchain from the build image:

internal/jobs/jobs.go

@@ -54,6 +54,7 @@ type Runner struct {
 	musicBrainzFailureStreak  int
 	running                   atomic.Bool
 	lastActivity              atomic.Int64
+	workerID                  string
 }
 
 // RunnerStatus is a process-local scheduler snapshot for the admin assurance
@@ -153,6 +154,11 @@ func New(s *store.Store, provider catalog.CatalogProvider, normalizer catalog.Re
 		cipher: cipher, interval: interval, spotifyInterval: 24 * time.Hour, logger: logger,
 		metrics: metrics.New(),
 	}
+	if token, err := security.Token(12); err == nil {
+		runner.workerID = "runner-" + token
+	} else {
+		runner.workerID = fmt.Sprintf("runner-%d", time.Now().UnixNano())
+	}
 	runner.initLifecycle()
 	for _, option := range options {
 		option(runner)
@@ -340,6 +346,16 @@ func (r *Runner) Run(ctx context.Context) {
 		r.tasks.Wait()
 		close(r.done)
 	}()
+	if recovered, err := r.store.RecoverExpiredWork(ctx, time.Now().UTC()); err != nil {
+		r.logger.Warn("durable work recovery failed", "error", err)
+	} else if recovered > 0 {
+		r.logger.Info("durable work recovered", "rows", recovered)
+	}
+	if reconciled, err := r.store.ReconcileStaleDeliveryAttempts(ctx, time.Now().UTC(), 10*time.Minute); err != nil {
+		r.logger.Warn("stale delivery attempt reconciliation failed", "error", err)
+	} else if reconciled > 0 {
+		r.logger.Info("stale delivery attempts reconciled", "attempts", reconciled)
+	}
 	if ctx.Err() == nil {
 		r.launchSync(ctx)
 		r.launchReleaseDayQueue(ctx)
@@ -545,6 +561,16 @@ func (r *Runner) runSyncCadence(ctx context.Context) {
 
 func (r *Runner) runMaintenance(ctx context.Context) {
 	r.metrics.RecordMaintenance()
+	if recovered, err := r.store.RecoverExpiredWork(ctx, time.Now().UTC()); err != nil {
+		r.logger.Warn("durable work recovery failed", "error", err)
+	} else if recovered > 0 {
+		r.logger.Info("durable work recovered", "rows", recovered)
+	}
+	if reconciled, err := r.store.ReconcileStaleDeliveryAttempts(ctx, time.Now().UTC(), 10*time.Minute); err != nil {
+		r.logger.Warn("stale delivery attempt reconciliation failed", "error", err)
+	} else if reconciled > 0 {
+		r.logger.Info("stale delivery attempts reconciled", "attempts", reconciled)
+	}
 	if err := r.store.PruneApplicationLogs(ctx, time.Now().UTC().Add(-7*24*time.Hour)); err != nil {
 		r.logger.Debug("application log pruning failed", "error", err)
 	}
@@ -702,7 +728,7 @@ func (r *Runner) refreshListenBrainz(ctx context.Context, now time.Time) (int, e
 }
 
 func (r *Runner) processManualSyncRequests(ctx context.Context, now time.Time) int {
-	requests, err := r.store.ClaimManualSyncRequests(ctx, 3)
+	requests, err := r.store.ClaimManualSyncRequestsWithLease(ctx, 3, r.workerID, 5*time.Minute)
 	if err != nil {
 		r.logger.Warn("manual synchronization queue failed", "error", err)
 		return 0
@@ -722,7 +748,7 @@ func (r *Runner) processManualSyncRequests(ctx context.Context, now time.Time) i
 				_, syncErr = r.syncArtists(ctx, now)
 			}
 		}
-		if err := r.store.CompleteManualSyncRequest(ctx, req.ID, syncErr); err != nil {
+		if err := r.store.CompleteManualSyncRequestOwned(ctx, req.ID, r.workerID, syncErr); err != nil {
 			r.logger.Warn("manual synchronization completion failed", "request_id", req.ID, "error", err)
 		}
 	}
@@ -953,6 +979,33 @@ func (r *Runner) syncOne(ctx context.Context, artist store.Artist, now time.Time
 		return outcome, nil
 	}
 	if len(strategy.batches) == 0 {
+		// Empty provider catalogs are successful health checks but are not
+		// actionable release observations. Once every fallback provider has
+		// answered successfully, advance the normal artist cadence instead of
+		// treating the empty response as a failure.
+		// A successful empty catalog is safe to treat as a normal cadence only
+		// when every provider that was attempted completed cleanly.  If a
+		// fallback provider failed or is cooling down, keep the artist on a
+		// bounded retry cadence instead of letting an empty response mask the
+		// outage.
+		if len(strategy.providerErrors) == 0 && (strategy.spotifyHealthy || strategy.itunesHealthy) {
+			if err := r.store.MarkArtistChecked(ctx, artist.ID, now, r.interval); err != nil {
+				return outcome, err
+			}
+			if spotifyWasDue && strategy.spotifyAttempted {
+				// Empty Spotify results are a healthy request but not an
+				// actionable catalog. They must not enter adaptive backoff;
+				// retry on the bounded failure cadence instead.
+				retryAt := now.Add(providerFailureRetryDelay(strategy.spotifyRateLimit, r.interval))
+				if strategy.spotifyRateLimit != nil {
+					retryAt = now.Add(syncRetryDelay(strategy.spotifyRateLimit, r.spotifyInterval))
+				}
+				if err := r.store.ScheduleSpotifyCheck(ctx, artist.ID, retryAt); err != nil {
+					return outcome, err
+				}
+			}
+			return outcome, nil
+		}
 		retryAt := now.Add(providerFailureRetryDelay(strategy.spotifyRateLimit, r.interval))
 		if strategy.spotifyRateLimit != nil {
 			r.logger.Debug("Spotify check retry scheduled", "artist_id", artist.ID,
@@ -979,7 +1032,7 @@ func (r *Runner) syncOne(ctx context.Context, artist store.Artist, now time.Time
 		}
 		return outcome, nil
 	}
-	if spotifyWasDue {
+	if spotifyWasDue && strategy.spotifySucceeded {
 		if strategy.spotifyRateLimit != nil {
 			r.logger.Debug("Spotify check retry scheduled", "artist_id", artist.ID,
 				"retry_after", syncRetryDelay(strategy.spotifyRateLimit, r.spotifyInterval).String(),
@@ -1002,6 +1055,18 @@ func (r *Runner) syncOne(ctx context.Context, artist store.Artist, now time.Time
 			return outcome, err
 		}
 		outcome.SpotifyBackoff = outcome.SpotifyUnchanged && !upcoming
+	} else if spotifyWasDue && strategy.spotifyAttempted && !strategy.spotifySucceeded {
+		// A failed or empty Spotify response must not be counted as an
+		// unchanged catalog merely because a fallback provider produced data.
+		// Keep Spotify on the bounded retry cadence and leave adaptive streaks
+		// untouched.
+		retryAt := now.Add(providerFailureRetryDelay(strategy.spotifyRateLimit, r.interval))
+		if strategy.spotifyRateLimit != nil {
+			retryAt = now.Add(syncRetryDelay(strategy.spotifyRateLimit, r.spotifyInterval))
+		}
+		if err := r.store.ScheduleSpotifyCheck(ctx, artist.ID, retryAt); err != nil {
+			return outcome, err
+		}
 	}
 	return outcome, nil
 }
@@ -1072,15 +1137,15 @@ func providerFailureRetryDelay(rateLimit *catalog.SpotifyRateLimitError, interva
 
 func (r *Runner) deliver(ctx context.Context, now time.Time) (deliveryStats, error) {
 	var summary deliveryStats
-	deliveries, err := r.store.DueDeliveries(ctx, now, 25)
+	deliveries, err := r.store.ClaimDueDeliveries(ctx, now, 25, r.workerID, 5*time.Minute)
 	if err != nil {
 		return summary, err
 	}
 	digestLimit := 25 - len(deliveries)
 	if digestLimit < 0 {
 		digestLimit = 0
 	}
-	digestDeliveries, err := r.store.DueDigestDeliveries(ctx, now, digestLimit)
+	digestDeliveries, err := r.store.ClaimDueDigestDeliveries(ctx, now, digestLimit, r.workerID, 5*time.Minute)
 	if err != nil {
 		return summary, err
 	}
@@ -1167,7 +1232,7 @@ func (r *Runner) deliverDigestOne(ctx context.Context, now time.Time, delivery s
 		}
 	}
 	if err == nil {
-		if markErr := r.store.MarkDigestDeliverySent(ctx, delivery.ID, now); markErr != nil {
+		if markErr := r.store.MarkDigestDeliverySentOwned(ctx, delivery.ID, delivery.ClaimOwner, now); markErr != nil {
 			if attemptID > 0 {
 				_ = r.store.FinishDeliveryAttempt(ctx, attemptID, delivery.Destination.ID, false, markErr.Error(), nil, time.Now().UTC())
 			}
@@ -1186,7 +1251,7 @@ func (r *Runner) deliverDigestOne(ctx context.Context, now time.Time, delivery s
 	redactedError := notify.RedactError(err)
 	r.logger.Warn("release digest delivery attempt failed",
 		"digest_delivery_id", delivery.ID, "destination_id", delivery.Destination.ID, "error", redactedError)
-	if markErr := r.store.MarkDigestDeliveryFailed(ctx, delivery.ID, delivery.Attempts+1, redactedError, now); markErr != nil {
+	if markErr := r.store.MarkDigestDeliveryFailedOwned(ctx, delivery.ID, delivery.Attempts+1, redactedError, delivery.ClaimOwner, now); markErr != nil {
 		result.err = markErr
 	}
 	if attemptID > 0 {
@@ -1222,7 +1287,7 @@ func (r *Runner) deliverOne(ctx context.Context, now time.Time, delivery store.D
 		}
 	}
 	if err == nil {
-		if markErr := r.store.MarkDeliverySent(ctx, delivery.ID, now); markErr != nil {
+		if markErr := r.store.MarkDeliverySentOwned(ctx, delivery.ID, delivery.ClaimOwner, now); markErr != nil {
 			if attemptID > 0 {
 				_ = r.store.FinishDeliveryAttempt(ctx, attemptID, delivery.Destination.ID, false, markErr.Error(), nil, time.Now().UTC())
 			}
@@ -1241,7 +1306,7 @@ func (r *Runner) deliverOne(ctx context.Context, now time.Time, delivery store.D
 	redactedError := notify.RedactError(err)
 	r.logger.Warn("notification attempt failed",
 		"delivery_id", delivery.ID, "destination_id", delivery.Destination.ID, "error", redactedError)
-	if markErr := r.store.MarkDeliveryFailed(ctx, delivery.ID, delivery.Attempts+1, redactedError, now); markErr != nil {
+	if markErr := r.store.MarkDeliveryFailedOwned(ctx, delivery.ID, delivery.Attempts+1, redactedError, delivery.ClaimOwner, now); markErr != nil {
 		result.err = markErr
 	}
 	if attemptID > 0 {