From a852bc8be4ca36e42fb9c376c997fb9ee7f36803 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 4 Aug 2026 17:39:30 +0200 Subject: [PATCH 1/2] fix(memory): show a write in the wiki and the vault mount right away A committed page only became visible once the curator's next poll tick projected it, so an agent could tick an item off, read the page back to check itself, and find its own work missing. Every write now asks the curator to project it and waits briefly. The curator stays the projection's only writer: writers ask, they never mirror. A projection that has not caught up in time is reported as a delay ("the wiki and the vault mount catch up shortly"), never as a failed write. --- stacklets/memory/cli/sync.py | 138 +++++++---------------- stacklets/memory/cli/write.py | 10 +- stacklets/memory/lib.py | 123 +++++++++++++++++++- tests/stacklets/test_memory_sync.py | 111 +++++++++++++----- tests/stacklets/test_memory_write_cli.py | 32 +++++- 5 files changed, 283 insertions(+), 131 deletions(-) diff --git a/stacklets/memory/cli/sync.py b/stacklets/memory/cli/sync.py index 56f6f835..271654a0 100644 --- a/stacklets/memory/cli/sync.py +++ b/stacklets/memory/cli/sync.py @@ -1,121 +1,57 @@ """stack memory sync - mirror memory source into the brain projection now. +Every write already asks for this on its own way out (`propagate_write`, +called from the write seam) and waits about five seconds. This command is +the same two steps with an operator's patience instead of an agent's: it +waits long enough to sit through a nightly sweep, and it says out loud +how far the projection got. + The curator is the ONLY writer of the brain projection (one-writer invariant, ADR-011). This command therefore mirrors nothing itself: it -drops a `mirror-now` trigger file the curator's tick loop watches, -then waits until the curator records the current memory HEAD as -mirrored. The explicit fast path for tests and operators who need -read-your-writes in the rendered wiki without waiting for the next -curator tick - without becoming a second git writer on brain. - -Failure shape: if the curator is down (memory stacklet stopped) or -busy past the wait cap (a nightly LLM sweep can hold a cycle for -minutes), this times out with the lag printed and a nonzero exit. -The trigger file survives either way; the curator consumes it on its -next free tick, so the request is never lost. +drops a `mirror-now` trigger file the curator's tick loop watches, then +waits until the curator records the current memory HEAD as mirrored. + +Failure shape: if the curator is down (memory stacklet stopped) or busy +past the wait cap (a nightly LLM sweep can hold a cycle for minutes), +this times out with the lag printed and a nonzero exit. The trigger file +survives either way; the curator consumes it on its next free tick, so +the request is never lost. """ from __future__ import annotations -import subprocess import sys -import time from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) -from lib import vault_path_for # noqa: E402 +from lib import ( # noqa: E402 + curator_state_dir_for, + request_mirror, + vault_local_head, + vault_path_for, + vault_remote_head, + wait_for_mirror, +) HELP = "Mirror memory source into the brain projection now" -TRIGGER_NAME = "mirror-now" -MIRROR_SHA_NAME = "last-mirrored-sha" - -# Wait cap per the test-loop rule: polls are cheap, caps are hard. +# Wait cap per the test-loop rule: polls are cheap, caps are hard. Far +# longer than a write's own wait, because a person at a terminal asked +# for this by hand and would rather wait than re-run it. WAIT_SECS = 40.0 POLL_INTERVAL = 0.75 -def request_mirror(state_dir: Path) -> Path: - """Drop the trigger file the curator's tick loop watches. +def target_head(memory: Path) -> str: + """The commit the mirror has to reach. - Content is a timestamp purely for debuggability; the curator only - cares that the file exists and consumes it by deletion. + Prefers the remote's HEAD: a hand edit pushed from Obsidian is + exactly the change an operator runs this to bring through, and it is + not in the local clone yet. Falls back to the local HEAD when Forgejo + is unreachable - the curator pulls before mirroring, so a local-only + target is still a valid best-effort floor. """ - state_dir.mkdir(parents=True, exist_ok=True) - trigger = state_dir / TRIGGER_NAME - trigger.write_text(str(time.time()), encoding="utf-8") - return trigger - - -def remote_head(memory: Path) -> str: - """Memory's remote HEAD - the commit the mirror must reach. - - The origin URL of the working copy already embeds credentials (set - at clone time), so `ls-remote` needs no token plumbing here. Falls - back to the local HEAD when Forgejo is unreachable: the curator - pulls before mirroring, so a local-only target is still a valid - best-effort floor. - """ - out = _git(memory, "ls-remote", "origin", "HEAD") - if out and out.split(): - return out.split()[0] - return (_git(memory, "rev-parse", "HEAD") or "").strip() - - -def mirrored_contains(memory: Path, target: str, mirrored: str) -> bool: - """True when the curator's recorded mirror sha includes `target`. - - Checked against the memory clone's history: the curator pulls that - same working copy before mirroring, so once `target` is mirrored - both commits exist there. A sha git does not know yet is simply - "not yet" - the caller keeps polling. - """ - if not mirrored: - return False - if mirrored == target: - return True - rc = subprocess.run( - ["git", "-C", str(memory), "merge-base", "--is-ancestor", target, mirrored], - capture_output=True, - ).returncode - return rc == 0 - - -def wait_for_mirror( - state_dir: Path, - memory: Path, - target: str, - *, - timeout: float = WAIT_SECS, - interval: float = POLL_INTERVAL, -) -> str | None: - """Poll `last-mirrored-sha` until it contains `target`. - - Returns the mirrored sha on success, None on timeout. Checks the - condition once before looking at the clock, so an already-current - mirror succeeds even with a zero timeout. - """ - sha_file = state_dir / MIRROR_SHA_NAME - deadline = time.monotonic() + timeout - while True: - mirrored = "" - if sha_file.exists(): - mirrored = sha_file.read_text(encoding="utf-8").strip() - if mirrored_contains(memory, target, mirrored): - return mirrored - if time.monotonic() >= deadline: - return None - time.sleep(interval) - - -def _git(repo: Path, *args: str) -> str | None: - result = subprocess.run( - ["git", "-C", str(repo), *args], - capture_output=True, text=True, - ) - if result.returncode != 0: - return None - return result.stdout + return vault_remote_head(memory) or vault_local_head(memory) or "" def run(args, stacklet, config): @@ -124,17 +60,19 @@ def run(args, stacklet, config): return {"error": "stack data_dir not configured"} memory = vault_path_for(Path(data_dir)) - state_dir = Path(data_dir) / "memory" / "curator" + state_dir = curator_state_dir_for(Path(data_dir)) if not (memory / ".git").exists(): return {"error": f"memory vault not cloned at {memory}"} - target = remote_head(memory) + target = target_head(memory) if not target: return {"error": "cannot resolve memory HEAD"} request_mirror(state_dir) - mirrored = wait_for_mirror(state_dir, memory, target) + mirrored = wait_for_mirror( + state_dir, memory, target, timeout=WAIT_SECS, interval=POLL_INTERVAL, + ) if mirrored is None: return {"error": ( f"curator did not mirror {target[:10]} within {int(WAIT_SECS)}s " diff --git a/stacklets/memory/cli/write.py b/stacklets/memory/cli/write.py index 52e79ff2..dfe6016e 100644 --- a/stacklets/memory/cli/write.py +++ b/stacklets/memory/cli/write.py @@ -159,10 +159,16 @@ def _replace(prior: str) -> str: print(f"{repo_path} was already exactly this; nothing to commit") return {"ok": True, "committed": False, "path": repo_path} - print(f"Wrote {repo_path} (by {actor})\n {told}") + # Forgejo has the commit; the agent's mount and the wiki are fed by the + # curator and may be a beat behind. Say that as a delay, never as a + # doubt: a model that reads anything short of "done" writes the page + # again, and the second write is a duplicate nobody asked for. + mirrored = bool(result.get("mirrored")) + lag = "" if mirrored else "\n The wiki and the vault mount catch up shortly." + print(f"Wrote {repo_path} (by {actor})\n {told}{lag}") return { "ok": True, "committed": True, "path": repo_path, "by": actor, - "summary": told, + "summary": told, "mirrored": mirrored, "destructive": bool(change and change.destructive()), "removed": list(change.removed) if change else [], } diff --git a/stacklets/memory/lib.py b/stacklets/memory/lib.py index 773bff78..e5076bf1 100644 --- a/stacklets/memory/lib.py +++ b/stacklets/memory/lib.py @@ -648,6 +648,117 @@ def _preserve_and_reset(git, local_head: str, remote_head: str) -> SyncResult: return SyncResult("preserved_and_reset" if rc == 0 else "failed", name if rc == 0 else err) +# ─── Write propagation (Forgejo -> the trees people read) ──────────────── +# +# A commit in Forgejo is the truth, and it is also invisible. The agent +# reads the brain projection through a read-only mount; the family reads +# Quartz's render of that same tree. Both are fed by the curator, which +# pulls memory and mirrors it into brain on its poll tick. Left to the +# tick alone a write is unseeable by its own author for most of a poll +# interval -- which is how an agent comes to tick an item off, read the +# page back to check itself, and find its own work missing. +# +# The curator stays brain's only writer (ADR-011), so a writer here asks +# instead of mirroring: drop a trigger file the curator's tick loop +# watches, then watch the sha it records until the commit shows up in it. +# Both ends are plain files in the shared state dir, which is the one +# thing a host process and a container process can both reach with no +# transport between them. + +# The curator consumes the trigger by deleting it, so its content is +# only ever a breadcrumb for whoever is debugging the loop. +MIRROR_TRIGGER_NAME = "mirror-now" +# The file the curator writes after each successful projection. +MIRROR_SHA_NAME = "last-mirrored-sha" + +# How long a write waits for its own change to become visible. Short on +# purpose: the curator slices its sleep by the second and a mirror is +# file copies plus one commit, so this covers the ordinary case -- and +# the extraordinary one must never hold up a commit that already landed. +MIRROR_WAIT_SECS = 5.0 +MIRROR_POLL_INTERVAL = 0.25 + + +def curator_state_dir_for(data_dir: Path) -> Path: + """Where the curator keeps its progress, reachable from both planes.""" + return Path(data_dir) / "memory" / "curator" + + +def request_mirror(state_dir: Path) -> Path: + """Drop the trigger file the curator's tick loop watches.""" + state_dir.mkdir(parents=True, exist_ok=True) + trigger = state_dir / MIRROR_TRIGGER_NAME + trigger.write_text(str(time.time()), encoding="utf-8") + return trigger + + +def mirrored_contains(memory: Path, target: str, mirrored: str) -> bool: + """True when the curator's recorded sha already includes `target`. + + Checked against the memory clone's history: the curator pulls that + same working copy before mirroring, so once `target` is projected + both commits exist there. A sha git does not know yet simply reads + as "not yet" and the caller keeps waiting. + """ + if not mirrored: + return False + if mirrored == target: + return True + rc, _, _ = run_git( + Path(memory), "merge-base", "--is-ancestor", target, mirrored, timeout=10, + ) + return rc == 0 + + +def wait_for_mirror(state_dir: Path, memory: Path, target: str, *, + timeout: float = MIRROR_WAIT_SECS, + interval: float = MIRROR_POLL_INTERVAL) -> Optional[str]: + """Poll `last-mirrored-sha` until it contains `target`. + + Returns the mirrored sha, or None on timeout. The condition is + checked once before the clock is consulted, so an already-current + mirror succeeds even with a zero timeout. + """ + sha_file = Path(state_dir) / MIRROR_SHA_NAME + deadline = time.monotonic() + timeout + while True: + mirrored = "" + if sha_file.exists(): + mirrored = sha_file.read_text(encoding="utf-8").strip() + if mirrored_contains(memory, target, mirrored): + return mirrored + if time.monotonic() >= deadline: + return None + time.sleep(interval) + + +def propagate_write(data_dir: Path, *, + timeout: float = MIRROR_WAIT_SECS, + interval: float = MIRROR_POLL_INTERVAL) -> bool: + """Ask the curator to project a just-committed write, and wait briefly. + + Returns whether the change reached the read surfaces in time. False + is a delay, never a failure: the commit is in Forgejo either way and + the curator's own tick picks the trigger up regardless, so a caller + reports "saved, the view will catch up" rather than an error. That + distinction is the whole contract -- an agent told anything short of + "done" runs the edit again, and a re-run edit is a duplicate. + """ + data_dir = Path(data_dir) + memory = vault_path_for(data_dir) + target = vault_local_head(memory) + if not target: + return False # nothing to name as the target, so nothing to await + state_dir = curator_state_dir_for(data_dir) + try: + request_mirror(state_dir) + except OSError: + return False + return wait_for_mirror( + state_dir, memory, target, timeout=timeout, interval=interval, + ) is not None + + # ─── Vault writers ─────────────────────────────────────────────────────── def _code_url_from_config(config: dict | None) -> str: @@ -697,6 +808,11 @@ def update_memory(config: dict, repo_path: str, success (committed=False when the transform was a no-op, so nothing was written), or `{"error": ...}` when credentials are missing, the transform rejects the input (e.g. no matching todo), or Forgejo is unreachable. + + A committed write also carries `mirrored`: whether the change had + reached the trees that are actually read (the agent's mount, the + wiki) by the time this returned. See `propagate_write` -- False + there means "not yet", never "not written". """ secrets = config.get("secrets", {}) if config else {} token = secrets.get("memory__MEMORY_BOT_TOKEN", "") @@ -723,9 +839,14 @@ def update_memory(config: dict, repo_path: str, return {"ok": True, "committed": False} data_dir = config.get("data_dir") if config else None + mirrored = False if data_dir: pull_vault(vault_path_for(Path(data_dir))) # write-through so reads agree - return {"ok": True, "committed": True, "path": repo_path} + # ...and on to the trees that are actually read. `pull_vault` + # only makes the *source clone* agree; the agent's mount and the + # wiki both render the projection, which only the curator writes. + mirrored = propagate_write(Path(data_dir)) + return {"ok": True, "committed": True, "path": repo_path, "mirrored": mirrored} # ─── Vault readers ─────────────────────────────────────────────────────── diff --git a/tests/stacklets/test_memory_sync.py b/tests/stacklets/test_memory_sync.py index 3e937d62..27b04703 100644 --- a/tests/stacklets/test_memory_sync.py +++ b/tests/stacklets/test_memory_sync.py @@ -1,11 +1,21 @@ -"""`stack memory sync` — trigger + wait, curator stays brain's only writer. - -The command owns no mirror logic: it drops the trigger file and polls -`last-mirrored-sha` against the memory clone's history. What is worth -pinning here is the wait machinery: the ancestry check that defines -"mirrored", the immediate success on an already-current mirror, and a -fast, capped timeout. The end-to-end path (trigger picked up by a live -curator) rides the demo rig. +"""Write propagation — a committed page reaching the surfaces that show it. + +Forgejo is the source of truth and nobody reads it. The agent reads the +brain projection through a read-only mount; the family reads Quartz's +render of that same tree. So a write that stops at Forgejo is invisible +to both until the curator's next poll, which is exactly how "I ticked it +off and nothing changed" happens. + +The curator stays brain's only writer (ADR-011), so propagation is a +request and a wait, never a second writer. What is pinned here: the +ancestry check that defines "mirrored", the immediate success when the +mirror is already current, the capped wait, and — the property the write +path leans on — that propagation which never lands still leaves the +write committed. The end-to-end path (a live curator picking the trigger +up) rides the demo rig. + +`stack memory sync` is the same two steps with an operator's patience +instead of an agent's, so it is tested here as the thin wrapper it is. """ from __future__ import annotations @@ -20,6 +30,17 @@ _MEMORY_DIR = _REPO_ROOT / "stacklets" / "memory" sys.path.insert(0, str(_MEMORY_DIR)) +from lib import ( # noqa: E402 + MIRROR_SHA_NAME, + MIRROR_TRIGGER_NAME, + curator_state_dir_for, + mirrored_contains, + propagate_write, + request_mirror, + vault_path_for, + wait_for_mirror, +) + _SPEC = importlib.util.spec_from_file_location( "memory_cli_sync", _MEMORY_DIR / "cli" / "sync.py", ) @@ -27,12 +48,6 @@ memory_sync = importlib.util.module_from_spec(_SPEC) _SPEC.loader.exec_module(memory_sync) -MIRROR_SHA_NAME = memory_sync.MIRROR_SHA_NAME -TRIGGER_NAME = memory_sync.TRIGGER_NAME -mirrored_contains = memory_sync.mirrored_contains -request_mirror = memory_sync.request_mirror -wait_for_mirror = memory_sync.wait_for_mirror - def _git(repo: Path, *args: str) -> str: result = subprocess.run( @@ -42,28 +57,39 @@ def _git(repo: Path, *args: str) -> str: return result.stdout.strip() -def _repo_with_two_commits(tmp_path: Path) -> tuple[Path, str, str]: - repo = tmp_path / "memory" - repo.mkdir() +def _commit(repo: Path, name: str, body: str) -> str: + (repo / name).write_text(body, encoding="utf-8") + _git(repo, "add", ".") + _git(repo, "commit", "-qm", f"write {name}") + return _git(repo, "rev-parse", "HEAD") + + +def _init(repo: Path) -> None: + repo.mkdir(parents=True) subprocess.run(["git", "init", "-q", str(repo)], check=True) _git(repo, "config", "user.email", "t@local") _git(repo, "config", "user.name", "t") - (repo / "a.md").write_text("a", encoding="utf-8") - _git(repo, "add", ".") - _git(repo, "commit", "-qm", "one") - first = _git(repo, "rev-parse", "HEAD") - (repo / "b.md").write_text("b", encoding="utf-8") - _git(repo, "add", ".") - _git(repo, "commit", "-qm", "two") - second = _git(repo, "rev-parse", "HEAD") - return repo, first, second + + +def _repo_with_two_commits(tmp_path: Path) -> tuple[Path, str, str]: + repo = tmp_path / "memory" + _init(repo) + return repo, _commit(repo, "a.md", "a"), _commit(repo, "b.md", "b") + + +def _data_dir_with_vault(tmp_path: Path) -> tuple[Path, Path, str]: + """A stack data dir whose memory clone holds one committed page.""" + data_dir = tmp_path / "data" + vault = vault_path_for(data_dir) + _init(vault) + return data_dir, vault, _commit(vault, "a.md", "a") class TestRequestMirror: def test_writes_trigger_and_creates_state_dir(self, tmp_path): state = tmp_path / "curator" trigger = request_mirror(state) - assert trigger == state / TRIGGER_NAME + assert trigger == state / MIRROR_TRIGGER_NAME assert trigger.exists() @@ -119,6 +145,37 @@ def test_times_out_when_mirror_never_lands(self, tmp_path): assert wait_for_mirror(state, repo, second, timeout=0.05, interval=0.02) is None +class TestPropagateWrite: + """What a just-committed write does to make itself visible.""" + + def test_a_mirror_already_carrying_the_write_reports_propagated(self, tmp_path): + data_dir, vault, head = _data_dir_with_vault(tmp_path) + state = curator_state_dir_for(data_dir) + state.mkdir(parents=True) + (state / MIRROR_SHA_NAME).write_text(head, encoding="utf-8") + assert propagate_write(data_dir, timeout=0) is True + + def test_it_asks_the_curator_rather_than_writing_brain_itself(self, tmp_path): + # ADR-011: the curator is brain's only writer. Propagation drops a + # request and waits; it must never touch the projection directly. + data_dir, _, _ = _data_dir_with_vault(tmp_path) + propagate_write(data_dir, timeout=0.05, interval=0.02) + assert (curator_state_dir_for(data_dir) / MIRROR_TRIGGER_NAME).exists() + assert not (data_dir / "memory" / "brain").exists() + + def test_a_stalled_curator_costs_the_write_nothing(self, tmp_path): + # The write is already committed to Forgejo by the time we get + # here. A curator that is down, or busy in a nightly sweep, means + # the wiki catches up later — never that the write failed. + data_dir, _, _ = _data_dir_with_vault(tmp_path) + assert propagate_write(data_dir, timeout=0.05, interval=0.02) is False + + def test_an_unclonable_vault_is_simply_not_propagated(self, tmp_path): + # Nothing to name as the target, so there is nothing to wait for. + # Still no exception: propagation never speaks for the write. + assert propagate_write(tmp_path / "empty", timeout=0.05) is False + + class TestRunGuards: def test_missing_data_dir_errors(self): assert "error" in memory_sync.run(None, {}, {}) diff --git a/tests/stacklets/test_memory_write_cli.py b/tests/stacklets/test_memory_write_cli.py index d7201929..8022df6c 100644 --- a/tests/stacklets/test_memory_write_cli.py +++ b/tests/stacklets/test_memory_write_cli.py @@ -71,6 +71,9 @@ class _Store: def __init__(self): self.page = PAGE self.commits = [] + # Whether the curator projected the commit into the trees the + # agent and the wiki read before the write returned. + self.mirrored = True def update_memory(self, config, repo_path, transform, *, actor, message): try: @@ -84,7 +87,8 @@ def update_memory(self, config, repo_path, transform, *, actor, message): subject = message(self.page, after) if callable(message) else message self.page = after self.commits.append((actor, subject)) - return {"ok": True, "committed": True, "path": repo_path} + return {"ok": True, "committed": True, "path": repo_path, + "mirrored": self.mirrored} @property def last_subject(self): @@ -294,3 +298,29 @@ def test_writing_the_page_it_already_holds_commits_nothing(store, tmp_path): assert result["committed"] is False assert store.commits == [] + + +# ── saying whether the change is visible yet ───────────────────────────── +# +# Forgejo has the commit the instant `update_memory` returns, but the +# agent reads a mount and the family reads Quartz, and both are fed by +# the curator. When the curator has not caught up yet the caller has to +# hear that as a delay, never as a failure — a model told anything less +# than "done" re-runs the edit, and a re-run edit is a duplicate. + +def test_a_write_the_readers_already_show_reports_plainly(store, tmp_path, capsys): + store.mirrored = True + _run(store, PAGE.replace("- [ ] Wetter", "- [x] Wetter"), tmp=tmp_path) + + assert "catch up" not in capsys.readouterr().out + + +def test_a_write_the_readers_have_not_caught_up_with_is_still_done(store, tmp_path, + capsys): + store.mirrored = False + result = _run(store, PAGE.replace("- [ ] Wetter", "- [x] Wetter"), tmp=tmp_path) + + assert result["committed"] is True, "the commit landed; only the view lags" + assert result["mirrored"] is False + out = capsys.readouterr().out + assert "Wrote" in out and "catch up" in out From 8386f09bf675437a8169c96819385b4b0df575c2 Mon Sep 17 00:00:00 2001 From: Arthur Date: Tue, 4 Aug 2026 17:39:39 +0200 Subject: [PATCH 2/2] fix(memory): stop retrying a failed wiki rebuild every three minutes With the AI endpoint down, the curator retried the same rebuild on every quiet window, indefinitely, printing a line each time that looked like a healthy heartbeat. Observed running for hours. The gap now doubles with each consecutive failure up to an hour, and the log says how many attempts failed and when the next one is due. Source mirroring is untouched: it stays undebounced, so the wiki and the vault mount keep receiving new pages while generation is down. --- stacklets/memory/bot/curator.py | 47 ++++++++++++++++---- tests/stacklets/test_memory_curator.py | 60 ++++++++++++++++++++++++-- 2 files changed, 95 insertions(+), 12 deletions(-) diff --git a/stacklets/memory/bot/curator.py b/stacklets/memory/bot/curator.py index 3fbf16f7..59f26ca6 100644 --- a/stacklets/memory/bot/curator.py +++ b/stacklets/memory/bot/curator.py @@ -59,6 +59,8 @@ from loguru import logger # noqa: E402 from memory.lib import ( # noqa: E402 + MIRROR_SHA_NAME, + MIRROR_TRIGGER_NAME, PRESERVE_LOCAL, RESET_LOCAL, SyncResult, @@ -103,10 +105,11 @@ # removal operations. _GENERATED_NAMES = {"about.md", "index.md"} -# Trigger file `stack memory sync` drops into the state dir to request -# an immediate tick. The curator stays brain's only writer; the CLI -# only asks and waits (one-writer invariant, ADR-011). -TRIGGER_NAME = "mirror-now" +# Both ends of the trigger protocol live in `memory.lib`: every writer +# (the write seam on its way out, `stack memory sync` by hand) asks +# there, and this loop is what answers. The curator stays brain's only +# writer; callers only ask and wait (one-writer invariant, ADR-011). +TRIGGER_NAME = MIRROR_TRIGGER_NAME # The curator's own git remote. `origin` belongs to the host plane # (hooks and the host CLI reach Forgejo on a localhost/LAN port); this @@ -132,24 +135,42 @@ def only_own_commits(subjects: list[str]) -> bool: class Debounce: """Quiet-window tracker: `observe(head, now)` returns True once the - same head has been seen unchanged for `quiet_secs`.""" + same head has been seen unchanged for the current window. + + The window is `quiet_secs` while rebuilds are succeeding, and doubles + with each consecutive failure up to `max_backoff_secs`. That second + part exists because the usual reason a rebuild fails is the AI + endpoint being down, and being down is a condition that outlives one + quiet window: retrying on the flat window spends a subprocess and an + LLM call every three minutes, all day, and prints a line each time + that is indistinguishable from a healthy heartbeat. + """ - def __init__(self, quiet_secs: float): + def __init__(self, quiet_secs: float, max_backoff_secs: float = 3600.0): self.quiet_secs = quiet_secs + self.max_backoff_secs = max_backoff_secs + self.failures = 0 self._head: str | None = None self._since = 0.0 + def window(self) -> float: + """The gap currently being waited out. Reported in the log.""" + return min(self.quiet_secs * (2 ** self.failures), self.max_backoff_secs) + def observe(self, head: str, now: float) -> bool: if head != self._head: self._head, self._since = head, now return False - return (now - self._since) >= self.quiet_secs + return (now - self._since) >= self.window() def reset(self) -> None: + """Nothing pending, or a rebuild worked: back to the flat window.""" self._head = None + self.failures = 0 def retry_later(self, now: float) -> None: - """Failed rebuild: keep the head, restart the quiet window.""" + """Failed rebuild: keep the head, restart the window, wider.""" + self.failures += 1 self._since = now @@ -790,7 +811,7 @@ async def main() -> None: shared_bucket = os.environ.get("SHARED_BUCKET", "family") state_dir = Path(os.environ.get("CURATOR_STATE_DIR", "/data/memory/curator")) sha_file = state_dir / "last-rebuilt-sha" - mirror_file = state_dir / "last-mirrored-sha" + mirror_file = state_dir / MIRROR_SHA_NAME nightly_file = state_dir / "last-nightly-date" while not (vault_dir / ".git").exists(): @@ -977,6 +998,14 @@ async def mirror_reconcile() -> bool: debounce.reset() else: debounce.retry_later(time.monotonic()) + # Say the shape of the failure, not just that there was one. A + # per-cycle line at the same interval reads as a heartbeat; a + # widening gap with a count on it reads as an outage. + logger.warning( + "[curator] rebuild failed {}x in a row; next attempt in {}m " + "(source is still mirrored, only generated pages are stale)", + debounce.failures, round(debounce.window() / 60), + ) if __name__ == "__main__": diff --git a/tests/stacklets/test_memory_curator.py b/tests/stacklets/test_memory_curator.py index 84139960..70f00335 100644 --- a/tests/stacklets/test_memory_curator.py +++ b/tests/stacklets/test_memory_curator.py @@ -102,13 +102,67 @@ def test_reset_forgets_the_head(self): assert d.observe("aaa", now=2000.0) is False assert d.observe("aaa", now=2180.0) is True - def test_retry_later_defers_a_failed_rebuild(self): + def test_a_failed_rebuild_waits_longer_than_the_quiet_window(self): d = Debounce(quiet_secs=180) d.observe("aaa", now=1000.0) assert d.observe("aaa", now=1180.0) is True d.retry_later(now=1200.0) - assert d.observe("aaa", now=1300.0) is False - assert d.observe("aaa", now=1380.0) is True + # One quiet window on its own would have fired again at 1380. + assert d.observe("aaa", now=1380.0) is False + assert d.observe("aaa", now=1560.0) is True + + def test_each_consecutive_failure_widens_the_window(self): + # A rebuild fails for a reason that usually outlives one window — + # the AI endpoint is down. Retrying on the quiet window forever + # spends a subprocess and an LLM call every three minutes, all + # day, and reads in the log exactly like a healthy heartbeat. + d = Debounce(quiet_secs=180) + d.observe("aaa", now=0.0) + d.retry_later(now=0.0) + assert d.observe("aaa", now=359.0) is False + assert d.observe("aaa", now=360.0) is True + d.retry_later(now=360.0) + assert d.observe("aaa", now=1079.0) is False + assert d.observe("aaa", now=1080.0) is True + + def test_backoff_stops_widening_at_the_cap(self): + # A ceiling keeps a days-long outage retrying on a human + # timescale rather than drifting out to never. + d = Debounce(quiet_secs=180, max_backoff_secs=900) + d.observe("aaa", now=0.0) + for _ in range(10): + d.retry_later(now=0.0) + assert d.observe("aaa", now=899.0) is False + assert d.observe("aaa", now=900.0) is True + + def test_a_new_head_does_not_clear_the_backoff(self): + # Fresh commits keep arriving while the endpoint is down. They + # are not evidence that generation works again, so they must not + # reset the gap back to one quiet window. + d = Debounce(quiet_secs=180) + d.observe("aaa", now=0.0) + d.retry_later(now=0.0) + assert d.observe("bbb", now=100.0) is False + assert d.observe("bbb", now=459.0) is False + assert d.observe("bbb", now=460.0) is True + + def test_a_successful_rebuild_clears_the_backoff(self): + d = Debounce(quiet_secs=180) + d.observe("aaa", now=0.0) + d.retry_later(now=0.0) + d.retry_later(now=360.0) + d.reset() # what the loop does on success + assert d.observe("bbb", now=1000.0) is False + assert d.observe("bbb", now=1180.0) is True + + def test_the_window_it_is_waiting_on_is_reportable(self): + # The loop logs this, so an operator reading `docker logs` sees + # "retrying in 12m after 3 failures" instead of a silent cycle. + d = Debounce(quiet_secs=180) + assert d.window() == 180 + d.retry_later(now=0.0) + assert d.window() == 360 + assert d.failures == 1 # ── member_selection ─────────────────────────────────────────────────────