Harden config, capture, privacy gates, and store I/O

.gitignore

@@ -11,7 +11,6 @@ build/
 venv/
 .pytest_cache/
 .ruff_cache/
-uv.lock
 
 # OS
 .DS_Store

README.md

@@ -38,12 +38,18 @@ Coding agents forget everything between sessions. Workarounds exist, but each is
 
 ## Quickstart
 
-```bash
-uv tool install engram          # or: pipx install engram
+Not yet on PyPI — install from source:
 
-engram remember "I prefer pnpm over npm"
-engram recall                   # list what engram knows
-engram serve                    # start the MCP server for your agents
+```bash
+uv tool install git+https://github.com/xantorres/engram
+# or: pipx install git+https://github.com/xantorres/engram
+# or from a clone: uv tool install .
+
+engram remember "I prefer pnpm over npm"    # stage a fact (pending review)
+engram list --status pending                # see what's staged
+ENGRAM_AUTOPROMOTE=true engram sync --apply  # promote the low-risk ones
+engram recall                               # recall promoted memories
+engram serve                                # start the MCP server for your agents
 ```
 
 Wire it into an agent (Codex shown):
@@ -80,7 +86,7 @@ It federates capture across your agents, gates sensitive writes behind your appr
 - [Quickstart](examples/quickstart.md)
 - [Architecture](docs/ARCHITECTURE.md)
 - [Security and privacy](docs/SECURITY.md)
-- [Adapters](adapters/README.md)
+- [Adapters](https://github.com/xantorres/engram/tree/main/adapters)
 
 ## License
 

examples/quickstart.md

@@ -2,8 +2,12 @@
 
 ## Install
 
+Not yet on PyPI — install from source:
+
 ```bash
-uv tool install engram      # or: pipx install engram
+uv tool install git+https://github.com/xantorres/engram
+# or: pipx install git+https://github.com/xantorres/engram
+# or from a clone: uv tool install .
 ```
 
 ## Capture a few facts

pyproject.toml

@@ -34,6 +34,9 @@ dev = ["pytest>=8", "ruff>=0.6"]
 [tool.hatch.build.targets.wheel]
 packages = ["src/engram"]
 
+[tool.hatch.build.targets.sdist]
+only-include = ["src", "tests", "docs", "examples"]
+
 [tool.ruff]
 line-length = 100
 src = ["src", "tests"]

src/engram/bridge/promote.py

@@ -7,12 +7,14 @@
 
 from __future__ import annotations
 
+import contextlib
 import datetime as dt
 from dataclasses import dataclass, field
 
-from engram.core import dedup, tiers
+from engram.core import atomic, dedup, tiers
+from engram.core.locking import store_lock
 from engram.core.schema import Memory, Status
-from engram.core.store import Store
+from engram.core.store import MarkdownStore, Store
 
 
 @dataclass
@@ -43,9 +45,10 @@ def skipped(self) -> list[Route]:
 def plan(store: Store, *, kind_allowlist: list[str] | None = None) -> PromotionResult:
     """Route pending candidates to append, queue, or skip.
 
-    kind_allowlist: when provided, any candidate whose kind is in the list is
-    appended directly (bypassing tier classification) unless a conflict exists.
-    None falls back to the standard AUTO_KINDS tier logic.
+    kind_allowlist: when provided, a candidate whose non-curated kind is in the
+    list is appended directly (bypassing tier classification) unless a conflict
+    exists. Curated kinds always fall through to classification and queue for
+    review, regardless of the allowlist. None falls back to standard tier logic.
     """
     promoted = store.list(status=Status.promoted)
     result = PromotionResult()
@@ -55,7 +58,15 @@ def plan(store: Store, *, kind_allowlist: list[str] | None = None) -> PromotionR
             result.routes.append(Route(candidate, "skip", f"already known ({against})"))
             continue
         conflict = verdict == "conflict"
-        if kind_allowlist is not None and candidate.kind.value in kind_allowlist and not conflict:
+        if candidate.risk_tier >= tiers.TIER_CURATED:
+            result.routes.append(Route(candidate, "queue", "flagged for review at capture"))
+        elif (
+            kind_allowlist is not None
+            and candidate.kind.value in kind_allowlist
+            and candidate.kind not in tiers.CURATED_KINDS
+            and candidate.risk_tier < tiers.TIER_CURATED
+            and not conflict
+        ):
             result.routes.append(Route(candidate, "append", "kind in allowlist"))
         else:
             tier = tiers.classify(candidate.kind, conflict=conflict)
@@ -81,25 +92,48 @@ def apply(
     if not autopromote:
         return result  # dry-run: report routes, change nothing
     today = today or dt.date.today()
-    for route in result.routes:
-        candidate = route.memory
-        if route.action == "append":
-            store.append_log(candidate)
-            store.update(
-                candidate.model_copy(
-                    update={
-                        "status": Status.promoted,
-                        "last_verified": today,
-                        "dest": "memory-log.md",
-                    }
-                )
-            )
-        elif route.action == "queue":
-            escalated = candidate.model_copy(update={"risk_tier": tiers.TIER_CURATED})
-            store.update(escalated)
-            store.enqueue(escalated, dest="memory.md", reason=route.reason)
-        elif route.action == "skip":
-            store.update(candidate.model_copy(update={"status": Status.rejected}))
+    root = getattr(store, "root", None)
+    lock = store_lock(root) if root is not None else contextlib.nullcontext()
+    with lock:
+        for route in result.routes:
+            candidate = route.memory
+            if route.action == "append":
+                # Log first, then flip the registry. If the registry write fails,
+                # undo the log append so recall and the log can't diverge.
+                log_result = store.append_log(candidate)
+                try:
+                    store.update(
+                        candidate.model_copy(
+                            update={
+                                "status": Status.promoted,
+                                "last_verified": today,
+                                "dest": "memory-log.md",
+                            }
+                        )
+                    )
+                except Exception:
+                    if root is not None:
+                        atomic.restore_from_bak(log_result["undo_token"], root=root)
+                    raise
+            elif route.action == "queue":
+                # Escalate the registry, then file the review item. If the queue
+                # write fails, undo the escalation so a sensitive fact is never
+                # left pending-but-invisible to review.
+                escalated = candidate.model_copy(update={"risk_tier": tiers.TIER_CURATED})
+                if isinstance(store, MarkdownStore):
+                    _, write_result = store.update_with_token(escalated)
+                    undo_token = write_result["undo_token"]
+                else:
+                    store.update(escalated)
+                    undo_token = None
+                try:
+                    store.enqueue(escalated, dest="memory.md", reason=route.reason)
+                except Exception:
+                    if undo_token is not None and root is not None:
+                        atomic.restore_from_bak(undo_token, root=root)
+                    raise
+            elif route.action == "skip":
+                store.update(candidate.model_copy(update={"status": Status.rejected}))
     result.applied = True
     return result
 

src/engram/bridge/review.py

@@ -6,9 +6,11 @@
 
 from __future__ import annotations
 
+import contextlib
 import datetime as dt
 
 from engram.core import atomic
+from engram.core.locking import store_lock
 from engram.core.schema import Memory, Status
 from engram.core.store import MarkdownStore, Store
 
@@ -20,21 +22,55 @@ def pending_reviews(store: Store) -> list[dict]:
 def approve(store: Store, memory_id: str, *, confirm: bool, today: dt.date | None = None) -> dict:
     if not confirm:
         return {"ok": False, "error": "tier-3 write requires confirmation (pass --confirm)"}
-    item = store.queue_get(memory_id)
-    if item is None:
-        return {"ok": False, "error": f"no queued memory {memory_id}"}
-    today = today or dt.date.today()
-    memory = Memory.from_item(item["memory"]).model_copy(
-        update={
-            "status": Status.promoted,
-            "last_verified": today,
-            "dest": item.get("dest") or "memory.md",
-        }
-    )
-    store.update(memory)
-    store.append_log(memory)
-    store.resolve_queue(memory_id)
-    return {"ok": True, "id": memory.id}
+    root = getattr(store, "root", None)
+    lock = store_lock(root) if root is not None else contextlib.nullcontext()
+    with lock:
+        item = store.queue_get(memory_id)
+        if item is None:
+            return {"ok": False, "error": f"no queued memory {memory_id}"}
+        today = today or dt.date.today()
+        memory = Memory.from_item(item["memory"]).model_copy(
+            update={
+                "status": Status.promoted,
+                "last_verified": today,
+                "dest": item.get("dest") or "memory.md",
+            }
+        )
+        try:
+            if isinstance(store, MarkdownStore):
+                _, write_result = store.update_with_token(memory)
+                undo_token = write_result["undo_token"]
+            else:
+                store.update(memory)
+                undo_token = ""
+        except KeyError:
+            return {"ok": False, "error": f"unknown memory {memory_id}"}
+        try:
+            store.resolve_queue(memory_id)
+        except Exception:
+            # Promotion and queue resolution must land together. If resolving the
+            # queue item fails, undo the registry promotion so the fact stays
+            # pending in the queue rather than promoted-yet-still-queued.
+            if undo_token and root is not None:
+                atomic.restore_from_bak(undo_token, root=root)
+            raise
+
+        # A curated approval writes only the registry; appending to the low-risk
+        # log would duplicate the sensitive fact into an auto-captured surface it
+        # never belongs in. The audit entry stays traceable without the text.
+        if root is not None:
+            atomic._append_audit(
+                root,
+                {
+                    "ts": dt.datetime.now(dt.UTC).isoformat(),
+                    "endpoint": "review/approve",
+                    "entity_id": memory.id,
+                    "path": str(store.registry) if hasattr(store, "registry") else "",
+                    "undo_token": undo_token,
+                    "created": False,
+                },
+            )
+        return {"ok": True, "id": memory.id}
 
 
 def reject(store: Store, memory_id: str, *, reason: str = "") -> dict:
@@ -56,39 +92,41 @@ def forget(store: Store, memory_id: str) -> dict:
     A separate audit entry tagged endpoint=fact/forget is appended so the action
     is traceable by endpoint name without conflating it with routine store/save writes.
     """
-    memory = store.get(memory_id)
-    if memory is None:
-        return {"ok": False, "error": f"no memory {memory_id}"}
-    if memory.status != Status.promoted:
-        return {
-            "ok": False,
-            "error": f"memory {memory_id} is not promoted (status={memory.status.value})",
-        }
-
-    updated = memory.model_copy(update={"status": Status.rejected})
-    try:
-        if isinstance(store, MarkdownStore):
-            _, write_result = store.update_with_token(updated)
-            undo_token = write_result["undo_token"]
-        else:
-            store.update(updated)
-            undo_token = ""
-    except KeyError:
-        return {"ok": False, "error": f"concurrent write conflict for {memory_id}"}
-
-    # Append a dedicated audit entry so the forget action is traceable by endpoint.
     root = getattr(store, "root", None)
-    if root is not None:
-        atomic._append_audit(
-            root,
-            {
-                "ts": dt.datetime.now(dt.UTC).isoformat(),
-                "endpoint": "fact/forget",
-                "entity_id": memory_id,
-                "path": str(store.registry) if hasattr(store, "registry") else "",
-                "undo_token": undo_token,
-                "created": False,
-            },
-        )
-
-    return {"ok": True, "id": memory_id, "undo_token": undo_token}
+    lock = store_lock(root) if root is not None else contextlib.nullcontext()
+    with lock:
+        memory = store.get(memory_id)
+        if memory is None:
+            return {"ok": False, "error": f"no memory {memory_id}"}
+        if memory.status != Status.promoted:
+            return {
+                "ok": False,
+                "error": f"memory {memory_id} is not promoted (status={memory.status.value})",
+            }
+
+        updated = memory.model_copy(update={"status": Status.rejected})
+        try:
+            if isinstance(store, MarkdownStore):
+                _, write_result = store.update_with_token(updated)
+                undo_token = write_result["undo_token"]
+            else:
+                store.update(updated)
+                undo_token = ""
+        except KeyError:
+            return {"ok": False, "error": f"concurrent write conflict for {memory_id}"}
+
+        # A dedicated audit entry keeps the forget action traceable by endpoint.
+        if root is not None:
+            atomic._append_audit(
+                root,
+                {
+                    "ts": dt.datetime.now(dt.UTC).isoformat(),
+                    "endpoint": "fact/forget",
+                    "entity_id": memory_id,
+                    "path": str(store.registry) if hasattr(store, "registry") else "",
+                    "undo_token": undo_token,
+                    "created": False,
+                },
+            )
+
+        return {"ok": True, "id": memory_id, "undo_token": undo_token}

src/engram/capture/active.py

@@ -8,6 +8,7 @@
 
 from engram.core.schema import Kind, LearnedBy, Memory
 from engram.core.store import Store
+from engram.core.text import clean_fact
 
 
 def remember(
@@ -20,7 +21,7 @@ def remember(
 ) -> Memory:
     return store.add(
         Memory(
-            fact=fact.strip(),
+            fact=clean_fact(fact),
             kind=kind,
             confidence=confidence,
             learned_by=LearnedBy.remember,

src/engram/capture/importer.py

@@ -14,6 +14,7 @@
 
 from engram.core.schema import Kind, LearnedBy, Memory
 from engram.core.store import Store
+from engram.core.text import clean_fact
 
 _FRONTMATTER = re.compile(r"^---\n(.*?)\n---\n?(.*)$", re.DOTALL)
 _SKIP = {"memory.md", "index.md", "readme.md"}
@@ -62,7 +63,7 @@ def import_markdown_dir(
         if path.name.lower() in _SKIP:
             continue
         front, body = _split(path.read_text(encoding="utf-8"))
-        fact = (front.get("description") or _first_line(body)).strip()
+        fact = clean_fact(front.get("description") or _first_line(body))
         if not fact:
             continue
         hint = front.get("kind") or front.get("type")

src/engram/capture/readers/base.py

@@ -43,7 +43,9 @@ def generic_jsonl_turns(path: str | Path) -> list[Turn]:
             record = json.loads(line)
         except json.JSONDecodeError:
             continue
-        message = record.get("message") if isinstance(record, dict) else None
+        if not isinstance(record, dict):
+            continue
+        message = record.get("message")
         message = message if isinstance(message, dict) else record
         role = message.get("role") or record.get("role")
         if role not in ("user", "assistant"):

src/engram/capture/readers/claude_code.py

@@ -22,7 +22,12 @@ def read_session(path: str | Path) -> list[Turn]:
             record = json.loads(line)
         except json.JSONDecodeError:
             continue
-        message = record.get("message") or {}
+        if not isinstance(record, dict):
+            continue
+        message = record.get("message")
+        if message is not None and not isinstance(message, dict):
+            continue
+        message = message or {}
         role = message.get("role") or record.get("role")
         if role not in ("user", "assistant"):
             continue