From 2dc2fc53b2bf96b7f278d6e0bf6e594a95ac3908 Mon Sep 17 00:00:00 2001 From: botre Date: Sun, 9 Aug 2026 09:35:13 +0200 Subject: [PATCH] Sweep at boot, and stop rendering swept captures The 4 hour retention window is stated on the landing page and on every endpoint page. Two things let reality drift from it. Cron's first tick is a full interval away, so a process restarted more often than every 5 minutes never swept at all and captures outlived the window for as long as the database file did. The sweep now also runs once at startup. An open endpoint page never expired anything from its own list, so a page left open past the window kept rendering captures the server had already deleted, directly under the line promising they were gone. The page now drops them on the interval it already runs for relative timestamps, then refetches so the total and the windowed list come from the server again. The window reaches the page as data-retention-seconds, rendered from the same constant the sweep uses, rather than as a fourth hardcoded copy of the figure. Claude-Session: https://claude.ai/code/session_01XkbFE6pgcxRfsMAvnwyuqS --- PRODUCT.md | 3 ++- public/endpoint.js | 29 +++++++++++++++++++++- src/application.go | 21 +++++++++++----- src/application_test.go | 54 +++++++++++++++++++++++++++++++++++++++++ src/pages.go | 3 +++ src/routes_test.go | 9 +++++++ src/views/endpoint.html | 1 + 7 files changed, 112 insertions(+), 8 deletions(-) create mode 100644 src/application_test.go diff --git a/PRODUCT.md b/PRODUCT.md index 20cf20f..1755223 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -92,7 +92,8 @@ Confirmed functionality: Technical constraints: -- Retention is 4 hours; a cron sweep runs every 5 minutes. +- Retention is 4 hours; a sweep runs at startup and every 5 minutes after. An + open page drops captures from its own list as they age out. - Storage is SQLite on the container's writable layer. Capture history is lost on restart, by design. No durable store, no migration path. - Request body limit is 1 MiB. diff --git a/public/endpoint.js b/public/endpoint.js index 97593ee..6ef21c4 100644 --- a/public/endpoint.js +++ b/public/endpoint.js @@ -117,6 +117,23 @@ }) .catch((err) => console.error(err)); }, + + // Nothing tells the page that the server swept a capture out from under + // it, so a list left open long enough renders requests that no longer + // exist, next to the promise that they were deleted. Dropping them + // locally is only the immediate half: the list is windowed, so the + // refetch is what resyncs `total` and pulls any older-but-live capture + // into the window. + pruneExpired(retentionMs) { + if (!retentionMs) return; + const cutoff = Date.now() - retentionMs; + const live = this.requests.filter( + (r) => r.createdAt.getTime() > cutoff, + ); + if (live.length === this.requests.length) return; + this.requests = live; + return this.fetchRequests(); + }, }); }); @@ -154,9 +171,19 @@ if (!endpointId) return; const scheme = location.protocol === "https:" ? "wss:" : "ws:"; this._wsUrl = `${scheme}//${location.host}/ws/${endpointId}`; + // A page served without the window keeps every capture it was given + // rather than expiring them against a guess. + const retentionSeconds = Number(this.$el.dataset.retentionSeconds); + this._retentionMs = + Number.isFinite(retentionSeconds) && retentionSeconds > 0 + ? retentionSeconds * 1000 + : 0; Alpine.store("main").setEndpoint(endpointId); this._connectWebSocket(); - setInterval(() => this.tick++, TICK_MS); + setInterval(() => { + this.tick++; + Alpine.store("main").pruneExpired(this._retentionMs); + }, TICK_MS); document.addEventListener("visibilitychange", () => { if (!document.hidden) this._clearUnread(); }); diff --git a/src/application.go b/src/application.go index 2feea8c..3aa810e 100644 --- a/src/application.go +++ b/src/application.go @@ -122,14 +122,23 @@ func newApplication(config applicationConfig) *fiber.App { return application } -// startRetentionSweep drops captures older than the retention window on a -// schedule. It returns the stopped-on-exit scheduler so the caller owns its -// lifetime. +// sweepRetention drops every capture older than the retention window. +func sweepRetention() { + database.DeleteOldRequests(context.Background(), time.Now().Add(-retentionWindow)) +} + +// startRetentionSweep sweeps once and then on a schedule. The returned +// scheduler runs for the life of the process; there is no shutdown path that +// stops it. +// +// The sweep at startup is what makes the window hold: cron's first tick is a +// full interval away, so a process that restarts more often than the interval +// would otherwise never sweep at all and captures would outlive the window for +// as long as the database file does. func startRetentionSweep() *cron.Cron { + sweepRetention() scheduler := cron.New() - if _, err := scheduler.AddFunc(retentionSweep, func() { - database.DeleteOldRequests(context.Background(), time.Now().Add(-retentionWindow)) - }); err != nil { + if _, err := scheduler.AddFunc(retentionSweep, sweepRetention); err != nil { slog.Error("cron job registration failed", "err", err) os.Exit(1) } diff --git a/src/application_test.go b/src/application_test.go new file mode 100644 index 0000000..438a6ff --- /dev/null +++ b/src/application_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "httphq/src/database" +) + +// storeCapture writes a capture for an endpoint at a chosen age. GORM only +// stamps CreatedAt when it is zero, so passing one keeps it. +func storeCapture(t *testing.T, endpointID, uuid string, createdAt time.Time) { + t.Helper() + database.CreateRequest(t.Context(), &database.Request{ + UUID: uuid, + EndpointID: endpointID, + Method: "POST", + Path: "/", + CreatedAt: createdAt, + }) +} + +func uuidsFor(ctx context.Context, endpointID string) []string { + var uuids []string + for _, request := range database.GetRequestsForEndpointID(ctx, endpointID, "", 10) { + uuids = append(uuids, request.UUID) + } + return uuids +} + +func TestSweepRetention(t *testing.T) { + t.Run("drops captures older than the retention window", func(t *testing.T) { + endpointID := "sweep-old" + storeCapture(t, endpointID, "sweep-old-expired", + time.Now().Add(-retentionWindow).Add(-time.Minute)) + + sweepRetention() + + assert.Empty(t, uuidsFor(t.Context(), endpointID)) + }) + + t.Run("keeps captures inside the retention window", func(t *testing.T) { + endpointID := "sweep-recent" + storeCapture(t, endpointID, "sweep-recent-live", + time.Now().Add(-retentionWindow).Add(time.Minute)) + + sweepRetention() + + assert.Equal(t, []string{"sweep-recent-live"}, uuidsFor(t.Context(), endpointID)) + }) +} diff --git a/src/pages.go b/src/pages.go index f1c1238..68e9158 100644 --- a/src/pages.go +++ b/src/pages.go @@ -59,6 +59,9 @@ func renderEndpoint(c fiber.Ctx) error { "EndpointID": endpointID, "EndpointURL": endpointURL, "EndpointWebSocketURL": websocketURL, + // The page drops captures from its own list once they age out, so it + // needs the window as a number rather than as the prose it renders. + "RetentionSeconds": int(retentionWindow.Seconds()), }) } diff --git a/src/routes_test.go b/src/routes_test.go index a59f1bd..89b84fe 100644 --- a/src/routes_test.go +++ b/src/routes_test.go @@ -178,6 +178,15 @@ func TestPageRoutes(t *testing.T) { assert.Contains(t, body, "endpoint.js?v=") }) + // The page expires captures out of its own list, so it needs the window as + // a number. Rendering it from the same constant the sweep uses is what keeps + // the two from drifting. + t.Run("an endpoint page carries the retention window", func(t *testing.T) { + body := bodyOf(t, get(t, "/"+endpointID(t))) + + assert.Contains(t, body, `data-retention-seconds="14400"`) + }) + // robots.txt excludes endpoint pages, so a canonical URL pointing them at a // shared address would be a claim nothing else in the site makes. t.Run("an endpoint page carries no canonical URL", func(t *testing.T) { diff --git a/src/views/endpoint.html b/src/views/endpoint.html index 37a54a1..cb24b8b 100644 --- a/src/views/endpoint.html +++ b/src/views/endpoint.html @@ -4,6 +4,7 @@ class="flex-1 pb-12" x-data="endpointPage" data-endpoint-id="{{.EndpointID}}" + data-retention-seconds="{{.RetentionSeconds}}" >

Captured requests for {{.EndpointID}}