From a93add41da6bc620b6270d63dc9dedabc5484c8d Mon Sep 17 00:00:00 2001 From: Adarsh Prashar Date: Sat, 30 May 2026 23:58:26 +0530 Subject: [PATCH] feat: git-native memory layer (md/yaml reader + episodic facts) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A directory of files the user owns and versions in git; RiskKernel only reads them. Deterministic retrieval — no embedding index / vector DB in v0.1. - internal/memory: path-traversal-safe Reader over a configured root — List/Read/keyword-Search of .md/.yaml/.txt by namespace; markdown frontmatter (title/description, simple key:value, no YAML dep) and first-heading fallback. - internal/storage: episodic facts (migration 00004 memory_facts) — Put/Get/ ListFact, upsert by (namespace, key); run_id optional attribution. - API: GET /v1/memory (list/search), GET /v1/memory/entry, GET + PUT /v1/memory/facts. Path traversal -> 400; missing entry -> 404. - CLI: riskkernel memory list/show (reads the dir directly). - Python SDK: list_memory / read_memory / list_facts / put_fact. - config: RISKKERNEL_MEMORY_DIR (default ./memory); RISKKERNEL_MEMORY_EMBEDDINGS off by default and explicitly not implemented (logs a warning if set). - examples/memory/README.md documents the format. - Tests: reader (list/read/search/traversal/missing), facts storage, HTTP endpoints, SDK stub. Verified live (CLI + endpoints + traversal block). --- .env.example | 5 + CHANGELOG.md | 8 + cmd/riskkernel/main.go | 6 +- cmd/riskkernel/memory.go | 67 +++++ examples/memory/README.md | 59 ++++ internal/app/bootstrap.go | 9 + internal/config/config.go | 20 ++ internal/httpapi/memory.go | 131 +++++++++ internal/httpapi/memory_test.go | 99 +++++++ internal/httpapi/server.go | 15 +- internal/httpapi/server_test.go | 3 +- internal/memory/reader.go | 252 ++++++++++++++++++ internal/memory/reader_test.go | 117 ++++++++ internal/storage/facts.go | 82 ++++++ .../storage/migrations/00004_memory_facts.sql | 19 ++ internal/storage/sqlite_test.go | 27 ++ internal/storage/store.go | 7 + sdks/python/riskkernel/client.py | 29 ++ sdks/python/tests/test_sdk.py | 23 ++ 19 files changed, 974 insertions(+), 4 deletions(-) create mode 100644 cmd/riskkernel/memory.go create mode 100644 examples/memory/README.md create mode 100644 internal/httpapi/memory.go create mode 100644 internal/httpapi/memory_test.go create mode 100644 internal/memory/reader.go create mode 100644 internal/memory/reader_test.go create mode 100644 internal/storage/facts.go create mode 100644 internal/storage/migrations/00004_memory_facts.sql diff --git a/.env.example b/.env.example index bb1143a..b5fdab7 100644 --- a/.env.example +++ b/.env.example @@ -43,3 +43,8 @@ RISKKERNEL_MCP_ALLOWLIST= RISKKERNEL_MCP_READONLY= # Seconds a gated tools/call waits for a human decision (default 110). RISKKERNEL_MCP_APPROVAL_TIMEOUT=110 + +# Git-native memory: a directory of markdown/YAML the agent reads (you own it). +RISKKERNEL_MEMORY_DIR=./memory +# Embeddings/semantic index — OFF and not implemented in v0.1 (keyword search only). +RISKKERNEL_MEMORY_EMBEDDINGS=false diff --git a/CHANGELOG.md b/CHANGELOG.md index d66f0b3..cd7758c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -79,5 +79,13 @@ surface is governed by [`COMPATIBILITY.md`](COMPATIBILITY.md). `tool_calls` row. Enabled by `RISKKERNEL_MCP_UPSTREAM`; allowlist/read-only via `RISKKERNEL_MCP_ALLOWLIST` / `RISKKERNEL_MCP_READONLY`. Point your MCP client at the gateway and governance is invisible to allowed, approved calls. +- **Git-native memory layer** — a user-owned directory of markdown/YAML/text the + agent reads (`RISKKERNEL_MEMORY_DIR`, default `./memory`). Deterministic + retrieval: list, read, keyword search; markdown frontmatter (`title`/ + `description`) surfaced; reads are path-traversal-safe. **No embedding index / + vector DB** (off by default, not implemented in v0.1). Episodic facts (small + key/value) persist in SQLite (migration `00004`). `GET /v1/memory`, + `GET /v1/memory/entry`, `GET`/`PUT /v1/memory/facts`; `riskkernel memory + list/show`; Python SDK `list_memory`/`read_memory`/`list_facts`/`put_fact`. [Unreleased]: https://github.com/prashar32/riskkernel/commits/main diff --git a/cmd/riskkernel/main.go b/cmd/riskkernel/main.go index 3b4bc42..cf359ac 100644 --- a/cmd/riskkernel/main.go +++ b/cmd/riskkernel/main.go @@ -46,6 +46,8 @@ func main() { err = runAudit(args) case "approvals": err = runApprovals(args) + case "memory": + err = runMemory(args) case "version", "--version", "-v": fmt.Println("riskkernel", version.String()) case "help", "--help", "-h": @@ -74,6 +76,8 @@ Usage: riskkernel approvals list List pending human-in-the-loop approvals riskkernel approvals approve [--reason ...] Approve a pending request riskkernel approvals deny [--reason ...] Deny a pending request + riskkernel memory list [namespace] List git-native memory entries + riskkernel memory show [namespace] Print a memory file riskkernel version Print build identity riskkernel help Show this help @@ -117,7 +121,7 @@ func runServe(_ []string) error { ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() - srv := httpapi.New(cfg, deps.Gateway, deps.Runs, deps.Approvals, deps.MCP, deps.Log) + srv := httpapi.New(cfg, deps.Gateway, deps.Runs, deps.Approvals, deps.MCP, deps.Memory, deps.Log) addr := fmt.Sprintf(":%d", cfg.Port) return srv.Serve(ctx, addr) } diff --git a/cmd/riskkernel/memory.go b/cmd/riskkernel/memory.go new file mode 100644 index 0000000..a4c8337 --- /dev/null +++ b/cmd/riskkernel/memory.go @@ -0,0 +1,67 @@ +package main + +import ( + "fmt" + "os" + "text/tabwriter" + + "github.com/prashar32/riskkernel/internal/config" + "github.com/prashar32/riskkernel/internal/memory" +) + +// runMemory implements `riskkernel memory ` — read-only inspection of +// the git-native memory directory (the files you own on disk). +func runMemory(args []string) error { + if len(args) == 0 { + return fmt.Errorf("usage: riskkernel memory [namespace]>") + } + cfg, err := config.Load() + if err != nil { + return err + } + reader := memory.NewReader(cfg.Memory.Dir) + + switch args[0] { + case "list": + ns := "" + if len(args) > 1 { + ns = args[1] + } + entries, err := reader.List(ns) + if err != nil { + return err + } + if len(entries) == 0 { + fmt.Printf("no memory entries under %s\n", reader.Root()) + return nil + } + tw := tabwriter.NewWriter(os.Stdout, 0, 2, 2, ' ', 0) + fmt.Fprintln(tw, "NAME\tFORMAT\tTITLE") + for _, e := range entries { + fmt.Fprintf(tw, "%s\t%s\t%s\n", e.Name, e.Format, e.Title) + } + return tw.Flush() + + case "show": + if len(args) < 2 { + return fmt.Errorf("usage: riskkernel memory show [namespace]") + } + name := args[1] + ns := "" + if len(args) > 2 { + ns = args[2] + } + content, _, err := reader.Read(ns, name) + if err != nil { + return err + } + fmt.Print(content) + if len(content) > 0 && content[len(content)-1] != '\n' { + fmt.Println() + } + return nil + + default: + return fmt.Errorf("unknown memory subcommand %q (want list|show)", args[0]) + } +} diff --git a/examples/memory/README.md b/examples/memory/README.md new file mode 100644 index 0000000..198fd21 --- /dev/null +++ b/examples/memory/README.md @@ -0,0 +1,59 @@ +# Git-native memory + +RiskKernel's memory layer is **a directory of files you own** — version them in +git, edit them in your editor. RiskKernel only reads them (it never rewrites your +markdown). Retrieval is deterministic: list, read, and keyword search. There is +**no embedding index / vector DB** in v0.1 (it's a future opt-in). + +Point the daemon at a directory: + +```bash +RISKKERNEL_MEMORY_DIR=./memory riskkernel serve +``` + +Organize by **namespace** (a subdirectory). Files are `.md` / `.markdown`, +`.yaml` / `.yml`, or `.txt`. Markdown files may carry simple `key: value` +frontmatter — `title` and `description` are surfaced in listings: + +``` +memory/ + notes/ + architecture.md + decisions.md +``` + +```markdown +--- +title: Architecture decisions +description: why we chose SQLite + a pure-Go driver +--- + +# Architecture + +We use a single WAL-mode SQLite file as the default store ... +``` + +Read it from the daemon (agents) or the CLI (you): + +```bash +# files +curl "http://localhost:7070/v1/memory?namespace=notes" +curl "http://localhost:7070/v1/memory/entry?namespace=notes&name=architecture.md" +riskkernel memory list notes +riskkernel memory show architecture.md notes + +# episodic facts (small key/value an agent accumulates during runs) +curl -X PUT http://localhost:7070/v1/memory/facts \ + -d '{"namespace":"notes","key":"db","value":"sqlite"}' +curl "http://localhost:7070/v1/memory/facts?namespace=notes" +``` + +From the Python SDK: + +```python +import riskkernel as rk +c = rk.RiskKernel() +c.list_memory(namespace="notes") +c.read_memory("architecture.md", namespace="notes") +c.put_fact("notes", "db", "sqlite") +``` diff --git a/internal/app/bootstrap.go b/internal/app/bootstrap.go index 658f6a8..87df20e 100644 --- a/internal/app/bootstrap.go +++ b/internal/app/bootstrap.go @@ -17,6 +17,7 @@ import ( "github.com/prashar32/riskkernel/internal/gateway" "github.com/prashar32/riskkernel/internal/governor" "github.com/prashar32/riskkernel/internal/mcp" + "github.com/prashar32/riskkernel/internal/memory" "github.com/prashar32/riskkernel/internal/otel" "github.com/prashar32/riskkernel/internal/pricing" "github.com/prashar32/riskkernel/internal/provider" @@ -36,6 +37,7 @@ type Deps struct { Tracer *otel.Tracer Approvals *approval.Gate MCP *mcp.Gateway // nil when no upstream is configured + Memory *memory.Reader } // Close releases dependencies that hold resources (the tracer's buffered spans, @@ -96,6 +98,12 @@ func Build(cfg *config.Config) (*Deps, error) { "allowlist", len(cfg.MCP.Allowlist), "readonly", len(cfg.MCP.ReadOnly)) } + memReader := memory.NewReader(cfg.Memory.Dir) + log.Info("memory layer ready", "dir", memReader.Root()) + if cfg.Memory.Embeddings { + log.Warn("RISKKERNEL_MEMORY_EMBEDDINGS is set but embeddings are not implemented in v0.1; using deterministic keyword search") + } + return &Deps{ Config: cfg, Log: log, @@ -107,6 +115,7 @@ func Build(cfg *config.Config) (*Deps, error) { Tracer: tracer, Approvals: gate, MCP: mcpGW, + Memory: memReader, }, nil } diff --git a/internal/config/config.go b/internal/config/config.go index ca30a5a..407bfca 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -50,6 +50,22 @@ type Config struct { // MCP configures the MCP gateway (tool governance). Disabled unless an upstream // MCP server URL is set. MCP MCPConfig + + // Memory configures the git-native memory layer. + Memory MemoryConfig +} + +// MemoryConfig configures the git-native memory layer: a user-owned directory of +// markdown/YAML the agent reads, plus episodic facts in SQLite. +type MemoryConfig struct { + // Dir is the root memory directory (user-owned, git-native). Read from + // RISKKERNEL_MEMORY_DIR (default "./memory"). + Dir string + // Embeddings enables a semantic index. OFF by default and NOT implemented in + // v0.1 — retrieval is deterministic keyword/path search (no vector DB). The + // flag exists so the default posture is explicit. Read from + // RISKKERNEL_MEMORY_EMBEDDINGS (default false). + Embeddings bool } // MCPConfig configures the MCP gateway: a JSON-RPC reverse proxy in front of an @@ -149,6 +165,10 @@ func Load() (*Config, error) { ReadOnly: splitList(os.Getenv("RISKKERNEL_MCP_READONLY")), ApprovalTimeoutSeconds: envIntDefault("RISKKERNEL_MCP_APPROVAL_TIMEOUT", 110), }, + Memory: MemoryConfig{ + Dir: getenvDefault("RISKKERNEL_MEMORY_DIR", "./memory"), + Embeddings: envBoolDefault("RISKKERNEL_MEMORY_EMBEDDINGS", false), + }, } return cfg, nil } diff --git a/internal/httpapi/memory.go b/internal/httpapi/memory.go new file mode 100644 index 0000000..7c1cf33 --- /dev/null +++ b/internal/httpapi/memory.go @@ -0,0 +1,131 @@ +package httpapi + +import ( + "errors" + "net/http" + "time" + + "github.com/prashar32/riskkernel/internal/httpx" + "github.com/prashar32/riskkernel/internal/memory" + "github.com/prashar32/riskkernel/internal/storage" +) + +// handleListMemory implements GET /v1/memory?namespace=&q= — list (or keyword +// search) the user-owned markdown/YAML memory entries. +func (s *Server) handleListMemory(w http.ResponseWriter, r *http.Request) { + ns := r.URL.Query().Get("namespace") + q := r.URL.Query().Get("q") + var ( + entries []memory.Entry + err error + ) + if q != "" { + entries, err = s.memory.Search(ns, q) + } else { + entries, err = s.memory.List(ns) + } + if err != nil { + s.writeMemoryErr(w, err) + return + } + if entries == nil { + entries = []memory.Entry{} + } + httpx.WriteJSON(w, http.StatusOK, entries) +} + +// handleReadMemory implements GET /v1/memory/entry?namespace=&name= — read one +// memory file's content + metadata. +func (s *Server) handleReadMemory(w http.ResponseWriter, r *http.Request) { + ns := r.URL.Query().Get("namespace") + name := r.URL.Query().Get("name") + if name == "" { + httpx.WriteError(w, http.StatusBadRequest, "bad_request", "name is required") + return + } + content, e, err := s.memory.Read(ns, name) + if err != nil { + s.writeMemoryErr(w, err) + return + } + httpx.WriteJSON(w, http.StatusOK, map[string]any{ + "namespace": e.Namespace, "name": e.Name, "title": e.Title, + "description": e.Description, "format": e.Format, + "size": e.Size, "modTime": e.ModTime, "content": content, + }) +} + +// handleListFacts implements GET /v1/memory/facts?namespace= — episodic facts. +func (s *Server) handleListFacts(w http.ResponseWriter, r *http.Request) { + store := s.runs.Store() + if store == nil { + httpx.WriteJSON(w, http.StatusOK, []any{}) + return + } + facts, err := store.ListFacts(r.Context(), r.URL.Query().Get("namespace")) + if err != nil { + httpx.WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + out := make([]map[string]any, 0, len(facts)) + for _, f := range facts { + out = append(out, factView(f)) + } + httpx.WriteJSON(w, http.StatusOK, out) +} + +type putFactBody struct { + Namespace string `json:"namespace"` + Key string `json:"key"` + Value string `json:"value"` + RunID string `json:"runId"` +} + +// handlePutFact implements PUT /v1/memory/facts — write an episodic fact. +func (s *Server) handlePutFact(w http.ResponseWriter, r *http.Request) { + store := s.runs.Store() + if store == nil { + httpx.WriteError(w, http.StatusServiceUnavailable, "no_store", "no durable store configured") + return + } + var body putFactBody + if err := decodeJSON(w, r, &body); err != nil { + httpx.WriteError(w, http.StatusBadRequest, "bad_request", err.Error()) + return + } + if body.Key == "" { + httpx.WriteError(w, http.StatusBadRequest, "bad_request", "key is required") + return + } + f := storage.Fact{ + Namespace: body.Namespace, Key: body.Key, Value: body.Value, + RunID: body.RunID, UpdatedAt: time.Now(), + } + if err := store.PutFact(r.Context(), f); err != nil { + httpx.WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + return + } + httpx.WriteJSON(w, http.StatusCreated, factView(f)) +} + +func (s *Server) writeMemoryErr(w http.ResponseWriter, err error) { + switch { + case errors.Is(err, memory.ErrNotFound): + httpx.WriteError(w, http.StatusNotFound, "not_found", "memory entry not found") + case errors.Is(err, memory.ErrUnsafePath): + httpx.WriteError(w, http.StatusBadRequest, "bad_request", "unsafe memory path") + default: + httpx.WriteError(w, http.StatusInternalServerError, "internal_error", err.Error()) + } +} + +func factView(f storage.Fact) map[string]any { + v := map[string]any{ + "namespace": f.Namespace, "key": f.Key, "value": f.Value, + "updatedAt": f.UpdatedAt.Format(time.RFC3339), + } + if f.RunID != "" { + v["runId"] = f.RunID + } + return v +} diff --git a/internal/httpapi/memory_test.go b/internal/httpapi/memory_test.go new file mode 100644 index 0000000..370a12c --- /dev/null +++ b/internal/httpapi/memory_test.go @@ -0,0 +1,99 @@ +package httpapi + +import ( + "encoding/json" + "io" + "log/slog" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/prashar32/riskkernel/internal/config" + "github.com/prashar32/riskkernel/internal/governor" + "github.com/prashar32/riskkernel/internal/memory" + "github.com/prashar32/riskkernel/internal/runs" + "github.com/prashar32/riskkernel/internal/storage" +) + +// newMemoryServer builds a server with a populated memory dir and a store. +func newMemoryServer(t *testing.T) http.Handler { + t.Helper() + memDir := t.TempDir() + if err := os.MkdirAll(filepath.Join(memDir, "developer"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(memDir, "developer", "style.md"), + []byte("---\ntitle: Style\n---\nuse tabs"), 0o644); err != nil { + t.Fatal(err) + } + store, err := storage.OpenSQLite(filepath.Join(t.TempDir(), "mem.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = store.Close() }) + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + mgr := runs.NewManager(governor.Budget{}).WithStore(store, log) + srv := New(&config.Config{}, nil, mgr, nil, nil, memory.NewReader(memDir), log) + return srv.Handler() +} + +func TestMemoryEndpoints(t *testing.T) { + h := newMemoryServer(t) + + // List a namespace. + w := do(t, h, http.MethodGet, "/v1/memory?namespace=developer", "") + if w.Code != http.StatusOK { + t.Fatalf("list status = %d", w.Code) + } + var entries []map[string]any + _ = json.Unmarshal(w.Body.Bytes(), &entries) + if len(entries) != 1 || entries[0]["title"] != "Style" { + t.Fatalf("entries = %v", entries) + } + + // Read an entry. + w = do(t, h, http.MethodGet, "/v1/memory/entry?namespace=developer&name=style.md", "") + if w.Code != http.StatusOK { + t.Fatalf("read status = %d", w.Code) + } + var entry map[string]any + _ = json.Unmarshal(w.Body.Bytes(), &entry) + if entry["content"] != "---\ntitle: Style\n---\nuse tabs" { + t.Fatalf("content = %v", entry["content"]) + } + + // Path traversal rejected. + w = do(t, h, http.MethodGet, "/v1/memory/entry?namespace=developer&name=../../etc/passwd", "") + if w.Code != http.StatusBadRequest { + t.Fatalf("traversal status = %d, want 400", w.Code) + } + + // Missing entry → 404. + w = do(t, h, http.MethodGet, "/v1/memory/entry?namespace=developer&name=nope.md", "") + if w.Code != http.StatusNotFound { + t.Fatalf("missing status = %d, want 404", w.Code) + } +} + +func TestFactsEndpoints(t *testing.T) { + h := newMemoryServer(t) + + // Write a fact. + w := do(t, h, http.MethodPut, "/v1/memory/facts", + `{"namespace":"developer","key":"db","value":"postgres"}`) + if w.Code != http.StatusCreated { + t.Fatalf("put fact status = %d, body=%s", w.Code, w.Body.String()) + } + + // Read it back. + w = do(t, h, http.MethodGet, "/v1/memory/facts?namespace=developer", "") + if w.Code != http.StatusOK { + t.Fatalf("list facts status = %d", w.Code) + } + var facts []map[string]any + _ = json.Unmarshal(w.Body.Bytes(), &facts) + if len(facts) != 1 || facts[0]["key"] != "db" || facts[0]["value"] != "postgres" { + t.Fatalf("facts = %v", facts) + } +} diff --git a/internal/httpapi/server.go b/internal/httpapi/server.go index d5b1628..8f095eb 100644 --- a/internal/httpapi/server.go +++ b/internal/httpapi/server.go @@ -18,6 +18,7 @@ import ( "github.com/prashar32/riskkernel/internal/gateway" "github.com/prashar32/riskkernel/internal/httpx" "github.com/prashar32/riskkernel/internal/mcp" + "github.com/prashar32/riskkernel/internal/memory" "github.com/prashar32/riskkernel/internal/runs" "github.com/prashar32/riskkernel/internal/storage" "github.com/prashar32/riskkernel/internal/version" @@ -30,13 +31,14 @@ type Server struct { runs *runs.Manager approvals *approval.Gate mcp *mcp.Gateway + memory *memory.Reader log *slog.Logger } // New constructs a Server. func New(cfg *config.Config, gw *gateway.Gateway, mgr *runs.Manager, gate *approval.Gate, - mcpGW *mcp.Gateway, log *slog.Logger) *Server { - return &Server{cfg: cfg, gateway: gw, runs: mgr, approvals: gate, mcp: mcpGW, log: log} + mcpGW *mcp.Gateway, mem *memory.Reader, log *slog.Logger) *Server { + return &Server{cfg: cfg, gateway: gw, runs: mgr, approvals: gate, mcp: mcpGW, memory: mem, log: log} } // Handler returns the root HTTP handler with all routes mounted. @@ -77,6 +79,15 @@ func (s *Server) Handler() http.Handler { // Local admin web page (Surface: human-in-the-loop, pull channel). mux.HandleFunc("GET /admin/approvals", s.requireAuth(s.handleAdminApprovalsPage)) } + // Git-native memory layer. + if s.memory != nil { + mux.HandleFunc("GET /v1/memory", s.requireAuth(s.handleListMemory)) + mux.HandleFunc("GET /v1/memory/entry", s.requireAuth(s.handleReadMemory)) + } + if s.runs != nil { + mux.HandleFunc("GET /v1/memory/facts", s.requireAuth(s.handleListFacts)) + mux.HandleFunc("PUT /v1/memory/facts", s.requireAuth(s.handlePutFact)) + } return s.recoverer(mux) } diff --git a/internal/httpapi/server_test.go b/internal/httpapi/server_test.go index f71f44c..cfb7c1a 100644 --- a/internal/httpapi/server_test.go +++ b/internal/httpapi/server_test.go @@ -15,6 +15,7 @@ import ( "github.com/prashar32/riskkernel/internal/approval" "github.com/prashar32/riskkernel/internal/config" "github.com/prashar32/riskkernel/internal/governor" + "github.com/prashar32/riskkernel/internal/memory" "github.com/prashar32/riskkernel/internal/runs" "github.com/prashar32/riskkernel/internal/storage" ) @@ -29,7 +30,7 @@ func newTestServer(t *testing.T, token string) (*Server, *runs.Manager, *approva log := slog.New(slog.NewTextHandler(io.Discard, nil)) mgr := runs.NewManager(governor.Budget{Tokens: 100000}).WithStore(store, log) gate := approval.NewGate(store, approval.Policy{DefaultSafe: true}, nil, log) - srv := New(&config.Config{APIToken: token}, nil, mgr, gate, nil, log) + srv := New(&config.Config{APIToken: token}, nil, mgr, gate, nil, memory.NewReader(t.TempDir()), log) return srv, mgr, gate } diff --git a/internal/memory/reader.go b/internal/memory/reader.go new file mode 100644 index 0000000..47d7340 --- /dev/null +++ b/internal/memory/reader.go @@ -0,0 +1,252 @@ +// Package memory implements RiskKernel's git-native memory layer: a user-owned +// directory of markdown/YAML/text files the agent reads. The files are yours — +// version them in git, edit them in your editor; RiskKernel only reads them. +// +// Retrieval is deterministic: list, read, and keyword search. There is NO +// embedding index / vector DB in v0.1 (CLAUDE.md §9) — semantic search is a future +// opt-in. Reads are path-traversal-safe: a request can never escape the root. +package memory + +import ( + "errors" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "time" +) + +// ErrNotFound is returned when a memory entry does not exist. +var ErrNotFound = errors.New("memory: not found") + +// ErrUnsafePath is returned when a name would escape the memory root. +var ErrUnsafePath = errors.New("memory: unsafe path") + +// Entry is a memory file's metadata. +type Entry struct { + Namespace string `json:"namespace"` + Name string `json:"name"` // path relative to the namespace + Title string `json:"title"` + Description string `json:"description,omitempty"` + Format string `json:"format"` // markdown | yaml | text + Size int64 `json:"size"` + ModTime time.Time `json:"modTime"` +} + +// Reader reads a configured memory root directory. +type Reader struct { + root string +} + +// NewReader returns a Reader rooted at dir. +func NewReader(dir string) *Reader { + abs, err := filepath.Abs(dir) + if err != nil { + abs = dir + } + return &Reader{root: filepath.Clean(abs)} +} + +// Root returns the configured memory root. +func (r *Reader) Root() string { return r.root } + +var memoryExts = map[string]string{ + ".md": "markdown", + ".markdown": "markdown", + ".yaml": "yaml", + ".yml": "yaml", + ".txt": "text", +} + +// List returns the memory entries under namespace (recursively). A missing +// directory yields an empty list, not an error. +func (r *Reader) List(namespace string) ([]Entry, error) { + base, err := r.resolveDir(namespace) + if err != nil { + return nil, err + } + var out []Entry + walkErr := filepath.WalkDir(base, func(path string, d fs.DirEntry, err error) error { + if err != nil { + if os.IsNotExist(err) { + return fs.SkipAll + } + return err + } + if d.IsDir() { + return nil + } + format, ok := memoryExts[strings.ToLower(filepath.Ext(d.Name()))] + if !ok { + return nil + } + rel, _ := filepath.Rel(base, path) + info, _ := d.Info() + e := Entry{ + Namespace: namespace, + Name: filepath.ToSlash(rel), + Format: format, + Size: sizeOf(info), + ModTime: modTime(info), + } + e.Title, e.Description = metadata(path, e.Name, format) + out = append(out, e) + return nil + }) + if walkErr != nil && !os.IsNotExist(walkErr) { + return nil, fmt.Errorf("memory: list %q: %w", namespace, walkErr) + } + return out, nil +} + +// Read returns the content and metadata of a memory entry. +func (r *Reader) Read(namespace, name string) (string, Entry, error) { + p, err := r.resolveFile(namespace, name) + if err != nil { + return "", Entry{}, err + } + data, err := os.ReadFile(p) + if err != nil { + if os.IsNotExist(err) { + return "", Entry{}, ErrNotFound + } + return "", Entry{}, fmt.Errorf("memory: read: %w", err) + } + format := memoryExts[strings.ToLower(filepath.Ext(p))] + if format == "" { + format = "text" + } + info, _ := os.Stat(p) + e := Entry{ + Namespace: namespace, Name: filepath.ToSlash(name), Format: format, + Size: sizeOf(info), ModTime: modTime(info), + } + e.Title, e.Description = titleFrom(string(data), name, format) + return string(data), e, nil +} + +// Search returns entries in a namespace whose title, name, or content contains +// the (case-insensitive) query. Deterministic keyword search — no embeddings. +func (r *Reader) Search(namespace, query string) ([]Entry, error) { + entries, err := r.List(namespace) + if err != nil { + return nil, err + } + if query == "" { + return entries, nil + } + q := strings.ToLower(query) + var out []Entry + for _, e := range entries { + if strings.Contains(strings.ToLower(e.Name), q) || + strings.Contains(strings.ToLower(e.Title), q) || + strings.Contains(strings.ToLower(e.Description), q) { + out = append(out, e) + continue + } + if content, _, err := r.Read(namespace, e.Name); err == nil && + strings.Contains(strings.ToLower(content), q) { + out = append(out, e) + } + } + return out, nil +} + +// --- safe path resolution --- + +func (r *Reader) resolveDir(namespace string) (string, error) { + return r.safeJoin(namespace) +} + +func (r *Reader) resolveFile(namespace, name string) (string, error) { + return r.safeJoin(filepath.Join(namespace, name)) +} + +// safeJoin joins rel under the root and guarantees the result stays within it. +func (r *Reader) safeJoin(rel string) (string, error) { + joined := filepath.Clean(filepath.Join(r.root, rel)) + if joined != r.root && !strings.HasPrefix(joined, r.root+string(os.PathSeparator)) { + return "", ErrUnsafePath + } + return joined, nil +} + +func sizeOf(info fs.FileInfo) int64 { + if info == nil { + return 0 + } + return info.Size() +} + +func modTime(info fs.FileInfo) time.Time { + if info == nil { + return time.Time{} + } + return info.ModTime() +} + +// metadata reads just enough of a file to extract title/description. +func metadata(path, name, format string) (title, desc string) { + data, err := os.ReadFile(path) + if err != nil { + return defaultTitle(name), "" + } + return titleFrom(string(data), name, format) +} + +// titleFrom derives a title/description from a file's content. For markdown it +// reads simple `key: value` YAML frontmatter (no YAML dependency) and falls back +// to the first `# heading`; otherwise the filename. +func titleFrom(content, name, format string) (title, desc string) { + title = defaultTitle(name) + if format == "markdown" { + fm := frontmatter(content) + if t := fm["title"]; t != "" { + title = t + } else if n := fm["name"]; n != "" { + title = n + } else if h := firstHeading(content); h != "" { + title = h + } + desc = fm["description"] + } + return title, desc +} + +func defaultTitle(name string) string { + base := filepath.Base(name) + return strings.TrimSuffix(base, filepath.Ext(base)) +} + +// frontmatter parses a leading `---` … `---` block of simple key: value lines. +func frontmatter(content string) map[string]string { + out := map[string]string{} + if !strings.HasPrefix(content, "---") { + return out + } + lines := strings.Split(content, "\n") + if len(lines) == 0 || strings.TrimSpace(lines[0]) != "---" { + return out + } + for _, line := range lines[1:] { + if strings.TrimSpace(line) == "---" { + break + } + key, val, ok := strings.Cut(line, ":") + if !ok { + continue + } + out[strings.TrimSpace(key)] = strings.Trim(strings.TrimSpace(val), `"'`) + } + return out +} + +func firstHeading(content string) string { + for _, line := range strings.Split(content, "\n") { + if strings.HasPrefix(line, "# ") { + return strings.TrimSpace(line[2:]) + } + } + return "" +} diff --git a/internal/memory/reader_test.go b/internal/memory/reader_test.go new file mode 100644 index 0000000..168fa82 --- /dev/null +++ b/internal/memory/reader_test.go @@ -0,0 +1,117 @@ +package memory + +import ( + "errors" + "os" + "path/filepath" + "testing" +) + +func writeFile(t *testing.T, root, rel, content string) { + t.Helper() + p := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestList_TitlesAndFormats(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "developer/style.md", + "---\ntitle: Coding Style\ndescription: house rules\n---\n# ignored\nbody") + writeFile(t, root, "developer/notes.md", "# Heading Title\n\nsome notes") + writeFile(t, root, "developer/config.yaml", "key: value\n") + writeFile(t, root, "developer/ignore.png", "binary") // non-memory ext skipped + + r := NewReader(root) + entries, err := r.List("developer") + if err != nil { + t.Fatalf("List: %v", err) + } + if len(entries) != 3 { + t.Fatalf("got %d entries, want 3: %+v", len(entries), entries) + } + byName := map[string]Entry{} + for _, e := range entries { + byName[e.Name] = e + } + if byName["style.md"].Title != "Coding Style" || byName["style.md"].Description != "house rules" { + t.Errorf("frontmatter title/desc = %+v", byName["style.md"]) + } + if byName["style.md"].Format != "markdown" { + t.Errorf("format = %q", byName["style.md"].Format) + } + if byName["notes.md"].Title != "Heading Title" { + t.Errorf("heading title = %q", byName["notes.md"].Title) + } + if byName["config.yaml"].Format != "yaml" { + t.Errorf("yaml format = %q", byName["config.yaml"].Format) + } +} + +func TestRead(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "ns/doc.md", "# Doc\ncontent here") + r := NewReader(root) + + content, e, err := r.Read("ns", "doc.md") + if err != nil { + t.Fatalf("Read: %v", err) + } + if content != "# Doc\ncontent here" || e.Title != "Doc" { + t.Fatalf("read = %q, entry=%+v", content, e) + } + + if _, _, err := r.Read("ns", "missing.md"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound, got %v", err) + } +} + +func TestSearch(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "ns/a.md", "talks about postgres") + writeFile(t, root, "ns/b.md", "talks about redis") + r := NewReader(root) + + hits, err := r.Search("ns", "postgres") + if err != nil { + t.Fatal(err) + } + if len(hits) != 1 || hits[0].Name != "a.md" { + t.Fatalf("search hits = %+v", hits) + } + // Empty query returns everything. + all, _ := r.Search("ns", "") + if len(all) != 2 { + t.Fatalf("empty query = %d, want 2", len(all)) + } +} + +func TestPathTraversalRejected(t *testing.T) { + root := t.TempDir() + writeFile(t, root, "ns/ok.md", "ok") + // A secret outside the root must be unreachable. + writeFile(t, filepath.Dir(root), "secret.txt", "topsecret") + + r := NewReader(root) + if _, _, err := r.Read("ns", "../../secret.txt"); !errors.Is(err, ErrUnsafePath) { + t.Fatalf("expected ErrUnsafePath, got %v", err) + } + if _, _, err := r.Read("..", "secret.txt"); !errors.Is(err, ErrUnsafePath) { + t.Fatalf("expected ErrUnsafePath for namespace escape, got %v", err) + } +} + +func TestList_MissingNamespaceIsEmpty(t *testing.T) { + r := NewReader(t.TempDir()) + entries, err := r.List("does-not-exist") + if err != nil { + t.Fatalf("List on missing namespace should not error: %v", err) + } + if len(entries) != 0 { + t.Fatalf("expected empty, got %+v", entries) + } +} diff --git a/internal/storage/facts.go b/internal/storage/facts.go new file mode 100644 index 0000000..2cb9505 --- /dev/null +++ b/internal/storage/facts.go @@ -0,0 +1,82 @@ +package storage + +import ( + "context" + "database/sql" + "fmt" + "time" +) + +// Fact is one episodic memory key/value. +type Fact struct { + Namespace string + Key string + Value string + RunID string + UpdatedAt time.Time +} + +// PutFact inserts or updates an episodic fact by (namespace, key). +func (s *SQLite) PutFact(ctx context.Context, f Fact) error { + _, err := s.db.ExecContext(ctx, ` + INSERT INTO memory_facts (namespace, key, value, run_id, updated_at) + VALUES (?,?,?,?,?) + ON CONFLICT(namespace, key) DO UPDATE SET + value=excluded.value, run_id=excluded.run_id, updated_at=excluded.updated_at`, + f.Namespace, f.Key, f.Value, nullStr(f.RunID), fmtTime(f.UpdatedAt)) + if err != nil { + return fmt.Errorf("storage: put fact: %w", err) + } + return nil +} + +// GetFact returns a fact by (namespace, key), or ErrNotFound. +func (s *SQLite) GetFact(ctx context.Context, namespace, key string) (Fact, error) { + row := s.db.QueryRowContext(ctx, + `SELECT namespace, key, value, run_id, updated_at FROM memory_facts WHERE namespace = ? AND key = ?`, + namespace, key) + f, err := scanFact(row) + if err == sql.ErrNoRows { + return Fact{}, ErrNotFound + } + return f, err +} + +// ListFacts returns all facts in a namespace, key order. +func (s *SQLite) ListFacts(ctx context.Context, namespace string) ([]Fact, error) { + rows, err := s.db.QueryContext(ctx, + `SELECT namespace, key, value, run_id, updated_at FROM memory_facts WHERE namespace = ? ORDER BY key`, + namespace) + if err != nil { + return nil, fmt.Errorf("storage: list facts: %w", err) + } + defer rows.Close() + var out []Fact + for rows.Next() { + f, err := scanFact(rows) + if err != nil { + return nil, err + } + out = append(out, f) + } + return out, rows.Err() +} + +func scanFact(row rowScanner) (Fact, error) { + var f Fact + var runID sql.NullString + var updated string + if err := row.Scan(&f.Namespace, &f.Key, &f.Value, &runID, &updated); err != nil { + return Fact{}, err + } + f.RunID = runID.String + f.UpdatedAt = parseTime(updated) + return f, nil +} + +func nullStr(s string) any { + if s == "" { + return nil + } + return s +} diff --git a/internal/storage/migrations/00004_memory_facts.sql b/internal/storage/migrations/00004_memory_facts.sql new file mode 100644 index 0000000..a46bd3a --- /dev/null +++ b/internal/storage/migrations/00004_memory_facts.sql @@ -0,0 +1,19 @@ +-- +goose Up +-- Episodic memory: small, fact-granular key/value state an agent accumulates +-- across a run (distinct from the git-native markdown/YAML the user owns on disk). +-- Keyed by namespace + key; run_id is optional attribution (no FK — facts may be +-- global, and we don't want a fact write to fail on an unknown run). + +CREATE TABLE memory_facts ( + namespace TEXT NOT NULL, + key TEXT NOT NULL, + value TEXT NOT NULL, + run_id TEXT, + updated_at TEXT NOT NULL, + PRIMARY KEY (namespace, key) +); + +CREATE INDEX idx_memory_facts_ns ON memory_facts(namespace); + +-- +goose Down +-- Forward-only migrations (COMPATIBILITY.md). No down migration is provided. diff --git a/internal/storage/sqlite_test.go b/internal/storage/sqlite_test.go index 5d595ab..0c3ac0f 100644 --- a/internal/storage/sqlite_test.go +++ b/internal/storage/sqlite_test.go @@ -293,6 +293,33 @@ func TestApprovals(t *testing.T) { } } +func TestFacts(t *testing.T) { + s := openTemp(t) + ctx := context.Background() + now := time.Now().UTC() + + if _, err := s.GetFact(ctx, "ns", "k"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound, got %v", err) + } + if err := s.PutFact(ctx, Fact{Namespace: "ns", Key: "lang", Value: "go", UpdatedAt: now}); err != nil { + t.Fatal(err) + } + // Upsert overwrites. + if err := s.PutFact(ctx, Fact{Namespace: "ns", Key: "lang", Value: "go+python", RunID: "r1", UpdatedAt: now}); err != nil { + t.Fatal(err) + } + got, err := s.GetFact(ctx, "ns", "lang") + if err != nil || got.Value != "go+python" || got.RunID != "r1" { + t.Fatalf("GetFact = %+v, %v", got, err) + } + _ = s.PutFact(ctx, Fact{Namespace: "ns", Key: "db", Value: "sqlite", UpdatedAt: now}) + _ = s.PutFact(ctx, Fact{Namespace: "other", Key: "x", Value: "y", UpdatedAt: now}) + facts, err := s.ListFacts(ctx, "ns") + if err != nil || len(facts) != 2 { + t.Fatalf("ListFacts(ns) = %v, %v; want 2", facts, err) + } +} + func mustRun(t *testing.T, s *SQLite, id string, now time.Time) { t.Helper() if err := s.UpsertRun(context.Background(), RunRecord{ diff --git a/internal/storage/store.go b/internal/storage/store.go index 45724ba..370514c 100644 --- a/internal/storage/store.go +++ b/internal/storage/store.go @@ -147,6 +147,13 @@ type Store interface { // AppendToolCall records a tool invocation. AppendToolCall(ctx context.Context, t ToolCallRecord) error + // PutFact inserts or updates an episodic memory fact by (namespace, key). + PutFact(ctx context.Context, f Fact) error + // GetFact returns a fact, or ErrNotFound. + GetFact(ctx context.Context, namespace, key string) (Fact, error) + // ListFacts returns all facts in a namespace. + ListFacts(ctx context.Context, namespace string) ([]Fact, error) + // CreateApproval persists a new (pending) approval request. CreateApproval(ctx context.Context, a ApprovalRecord) error // GetApproval returns an approval by id, or ErrNotFound. diff --git a/sdks/python/riskkernel/client.py b/sdks/python/riskkernel/client.py index ed96cd0..0ce5dbe 100644 --- a/sdks/python/riskkernel/client.py +++ b/sdks/python/riskkernel/client.py @@ -12,6 +12,7 @@ import urllib.error import urllib.request from typing import Any, Optional +from urllib.parse import quote as _q from .errors import APIError, BudgetExceeded @@ -122,3 +123,31 @@ def resolve_approval(self, run_id: str, approval_id: str, approve: bool, "decision": "approve" if approve else "deny", "reason": reason, "decidedBy": decided_by, }) + + # --- git-native memory --- + + def list_memory(self, namespace: str = "", query: str = "") -> list: + """List (or keyword-search) the user-owned markdown/YAML memory entries.""" + q = [] + if namespace: + q.append("namespace=" + _q(namespace)) + if query: + q.append("q=" + _q(query)) + path = "/v1/memory" + ("?" + "&".join(q) if q else "") + return self._request("GET", path) + + def read_memory(self, name: str, namespace: str = "") -> dict: + """Read one memory file's content + metadata.""" + path = f"/v1/memory/entry?name={_q(name)}" + if namespace: + path += "&namespace=" + _q(namespace) + return self._request("GET", path) + + def list_facts(self, namespace: str = "") -> list: + path = "/v1/memory/facts" + ("?namespace=" + _q(namespace) if namespace else "") + return self._request("GET", path) + + def put_fact(self, namespace: str, key: str, value: str, run_id: str = "") -> dict: + return self._request("PUT", "/v1/memory/facts", { + "namespace": namespace, "key": key, "value": value, "runId": run_id, + }) diff --git a/sdks/python/tests/test_sdk.py b/sdks/python/tests/test_sdk.py index 6d8c843..e49a8ee 100644 --- a/sdks/python/tests/test_sdk.py +++ b/sdks/python/tests/test_sdk.py @@ -39,6 +39,12 @@ def _read(self): def do_GET(self): p = self.path + if p.startswith("/v1/memory/facts"): + return self._send(200, [{"namespace": "dev", "key": "db", "value": "sqlite"}]) + if p.startswith("/v1/memory/entry"): + return self._send(200, {"name": "style.md", "title": "Style", "content": "use tabs"}) + if p.startswith("/v1/memory"): + return self._send(200, [{"name": "style.md", "title": "Style", "format": "markdown"}]) if p.startswith("/v1/approvals/"): STATE.approval_polls += 1 status = "approved" if STATE.approval_polls >= 2 else "pending" @@ -74,6 +80,12 @@ def do_POST(self): return self._send(200, {"id": "run-1", "status": "running"}) return self._send(404, {"code": "not_found", "message": "no"}) + def do_PUT(self): + if self.path == "/v1/memory/facts": + self._read() + return self._send(201, {"namespace": "dev", "key": "db", "value": "sqlite"}) + return self._send(404, {"code": "not_found", "message": "no"}) + class SDKTest(unittest.TestCase): @classmethod @@ -141,6 +153,17 @@ def test_cancel(self): out = run.cancel("done") self.assertEqual(out["status"], "cancelled") + def test_memory(self): + c = self.rt.client + entries = c.list_memory(namespace="dev") + self.assertEqual(entries[0]["title"], "Style") + entry = c.read_memory("style.md", namespace="dev") + self.assertEqual(entry["content"], "use tabs") + facts = c.list_facts(namespace="dev") + self.assertEqual(facts[0]["key"], "db") + out = c.put_fact("dev", "db", "sqlite") + self.assertEqual(out["value"], "sqlite") + def test_proxy_config(self): with self.rt.governed_run(name="t") as run: cfg = run.proxy_config()