From bdd36847d77ef024a33bbb106feaffa28d101290 Mon Sep 17 00:00:00 2001 From: Alex Godoroja Date: Fri, 3 Jul 2026 19:47:35 -0700 Subject: [PATCH] scaffold: local metadata methods + response capture (host-local recall) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An HTTP app can now persist a method's result to a host-local JSON file and read it back with a backend-free local method — so an agent recalls host-local state without a round-trip (e.g. agentphone: buy_number writes the provisioned number to ~/.pilot/.agentphone; mynumber reads it). - Method.local {store} — a local read (no http/cli route); returns the store's JSON, or {"entries":[]} when absent. Plane "local" in .help. - HTTPRoute.capture_to — after a 2xx, best-effort append the response to the store's "entries" array (deduped by id). A capture failure never fails the call. - Manifest auto-grants fs.read (+ fs.write for capture targets) on the store, with ~ normalized to $HOME (adapter expands ~ at runtime, so grant == path). - Submission (rich) carries local.store + http.capture_to through ToConfig; validation rejects a method with both a local and a backend route. Tests: generate+compile a managed-http app with capture + local read, asserting the dispatcher wiring, the $HOME grants, and validation rules. Full suite green. --- internal/publish/submission.go | 46 ++++++- internal/scaffold/config.go | 88 ++++++++++++- internal/scaffold/templates/main.go.tmpl | 92 +++++++++++++- .../scaffold/templates/manifest.json.tmpl | 6 + internal/scaffold/zz_local_metadata_test.go | 118 ++++++++++++++++++ 5 files changed, 342 insertions(+), 8 deletions(-) create mode 100644 internal/scaffold/zz_local_metadata_test.go diff --git a/internal/publish/submission.go b/internal/publish/submission.go index 33462de..225b881 100644 --- a/internal/publish/submission.go +++ b/internal/publish/submission.go @@ -143,9 +143,20 @@ type SubMethod struct { Timeout string `json:"timeout"` // optional Go duration (e.g. "280s") overriding the latency-class default HTTP SubRoute `json:"http"` // http backend route CLI SubCLIRoute `json:"cli"` // cli backend route + Local *SubLocal `json:"local"` // local metadata route (no backend call) Params []SubParam `json:"params"` } +// SubLocal makes a method read a local JSON metadata file (no backend call) — +// the read side of SubRoute.CaptureTo. e.g. agentphone.mynumber reads the number +// this daemon provisioned. +type SubLocal struct { + Store string `json:"store"` // metadata file path, e.g. ~/.pilot/.agentphone +} + +// HasLocal reports whether this method is a local metadata read. +func (m SubMethod) HasLocal() bool { return m.Local != nil && strings.TrimSpace(m.Local.Store) != "" } + // HasHTTP / HasCLI report which route a hybrid method declares (exactly one). func (m SubMethod) HasHTTP() bool { return strings.TrimSpace(m.HTTP.Path) != "" } func (m SubMethod) HasCLI() bool { @@ -156,6 +167,10 @@ func (m SubMethod) HasCLI() bool { type SubRoute struct { Verb string `json:"verb"` // GET | POST Path string `json:"path"` // e.g. /current + // CaptureTo, when set, persists this method's successful response into a local + // metadata file so a SubLocal method can recall it (e.g. buy_number → + // ~/.pilot/.agentphone). Best-effort; never fails the call. + CaptureTo string `json:"capture_to"` } // SubCLIRoute is the backend CLI mapping for a method. Enumerated methods bake @@ -315,12 +330,29 @@ func (s Submission) Validate() []string { case s.Backend.IsCLI(): e = append(e, validateSubCLIMethod(n, m)...) default: - e = append(e, validateSubHTTPMethod(n, m)...) + if m.HasLocal() { + e = append(e, validateSubLocalMethod(n, m)...) + } else { + e = append(e, validateSubHTTPMethod(n, m)...) + } } } return e } +// validateSubLocalMethod checks one method's local metadata route: a store path +// is required and it must not also declare an http/cli route. +func validateSubLocalMethod(n string, m SubMethod) []string { + var e []string + if m.Local == nil || strings.TrimSpace(m.Local.Store) == "" { + e = append(e, fmt.Sprintf("Method %q: a local method needs local.store (the metadata file path)", n)) + } + if m.HasHTTP() || m.HasCLI() { + e = append(e, fmt.Sprintf("Method %q: a local method must not also declare an http/cli route", n)) + } + return e +} + // hasCLIMethods reports whether any submission method declares a cli route. func (s Submission) hasCLIMethods() bool { for _, m := range s.Methods { @@ -525,16 +557,20 @@ func (s Submission) ToConfig() *scaffold.Config { Params: params, } // A hybrid app routes per method by which route it declares; cli/http apps - // route every method the same way. + // route every method the same way. A local method (metadata read) has no + // backend route at all. methodIsCLI := s.Backend.IsCLI() || (s.Backend.IsHybrid() && m.HasCLI()) - if methodIsCLI { + switch { + case m.HasLocal(): + method.Local = &scaffold.LocalRoute{Store: m.Local.Store} + case methodIsCLI: method.CLI = &scaffold.CLIRoute{ Args: m.CLI.Args, ParamsAsFlags: m.CLI.ParamsAsFlags, Passthrough: m.CLI.Passthrough, } - } else { - route := &scaffold.HTTPRoute{Verb: orDefault(m.HTTP.Verb, "GET"), Path: m.HTTP.Path} + default: + route := &scaffold.HTTPRoute{Verb: orDefault(m.HTTP.Verb, "GET"), Path: m.HTTP.Path, CaptureTo: m.HTTP.CaptureTo} // Carry each param's explicit request location so the generator can // resolve query/path/path_raw/body/header placement. Omitted `in` // keeps the verb/path default (back-compat). diff --git a/internal/scaffold/config.go b/internal/scaffold/config.go index 304f597..067cf8e 100644 --- a/internal/scaffold/config.go +++ b/internal/scaffold/config.go @@ -300,6 +300,54 @@ func (c *Config) HasHTTPMethods() bool { return false } +// HasLocalMethods reports whether any method reads a local metadata store. +func (c *Config) HasLocalMethods() bool { + for _, m := range c.Methods { + if m.Local != nil { + return true + } + } + return false +} + +// LocalStoreGrant is one metadata-file path the adapter touches, with whether it +// writes it (a capture target) in addition to reading it. Grant is the manifest +// grant target (leading ~ normalized to $HOME) — the adapter expands ~ to $HOME +// at runtime, so the grant and the opened path agree. +type LocalStoreGrant struct { + Path string + Grant string + Write bool +} + +// LocalStores returns the distinct local metadata files the adapter reads +// (LocalRoute.Store) or writes (HTTPRoute.CaptureTo), for the manifest fs grants. +// Sorted for deterministic generation; a path that is both read and written is +// emitted once with Write=true. +func (c *Config) LocalStores() []LocalStoreGrant { + write := map[string]bool{} + seen := map[string]bool{} + for _, m := range c.Methods { + if m.Local != nil && m.Local.Store != "" { + seen[m.Local.Store] = true + } + if m.HTTP != nil && m.HTTP.CaptureTo != "" { + seen[m.HTTP.CaptureTo] = true + write[m.HTTP.CaptureTo] = true + } + } + out := make([]LocalStoreGrant, 0, len(seen)) + for p := range seen { + grant := p + if strings.HasPrefix(grant, "~/") { + grant = "$HOME/" + grant[2:] + } + out = append(out, LocalStoreGrant{Path: p, Grant: grant, Write: write[p]}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Path < out[j].Path }) + return out +} + // DefaultProvisionPath is the broker's reserved auto-provision route. A hybrid // app's provision method uses this path; the generated dispatch treats a method // whose http.path equals it as the provisioning call (no request body forwarded). @@ -398,10 +446,20 @@ type Method struct { Timeout string `yaml:"timeout"` // Go duration; default from duration class HTTP *HTTPRoute `yaml:"http"` // http backend route CLI *CLIRoute `yaml:"cli"` // cli backend route + Local *LocalRoute `yaml:"local"` // local metadata route (no backend call) Params map[string]string `yaml:"params"` // name -> human description, for help Roundtrip string `yaml:"roundtrip"` // measured warm roundtrip, for help } +// LocalRoute makes a method run entirely on the host with NO backend call: it +// reads a local JSON metadata file (see HTTPRoute.CaptureTo, which writes it) and +// returns its contents. Used for host-local state an agent should recall without +// a round-trip — e.g. agentphone.mynumber reads the number this daemon +// provisioned. `store` may start with ~ (expanded to $HOME at runtime). +type LocalRoute struct { + Store string `yaml:"store"` // path to the JSON metadata file, e.g. ~/.pilot/.agentphone +} + // HTTPRoute maps a method to one backend HTTP endpoint. The path may contain // {name} placeholders, filled at call time from the payload (REST-style path // params, e.g. /v1/calls/{call_id}). Of the remaining payload fields, a @@ -422,6 +480,14 @@ type HTTPRoute struct { Verb string `yaml:"verb"` // GET (default) | POST | PATCH | PUT | DELETE Path string `yaml:"path"` // e.g. /current or /v1/calls/{call_id} + // CaptureTo, when set, persists this method's successful (2xx) JSON response + // into a local metadata file (a LocalRoute method reads it back). Best-effort: + // a capture failure never fails the call. The response object is appended to + // the file's "entries" array (deduped by "id" when present). Path may start + // with ~ (expanded to $HOME at runtime). e.g. agentphone.buy_number captures + // the provisioned number to ~/.pilot/.agentphone. + CaptureTo string `yaml:"capture_to"` + // ParamIn carries each param's explicit location (one of the five values // above); a param absent from the map takes the verb/path default. Set from // the submission in ToConfig (the YAML form uses per-param `in`). @@ -753,7 +819,14 @@ func (c *Config) Validate() []error { } switch c.Backend.Type { case "http": - errs = append(errs, c.validateHTTPMethod(i, m)...) + switch { + case m.Local != nil && m.HTTP != nil: + errs = append(errs, fmt.Errorf("methods[%d] (%s): a method cannot declare both a local and an http route", i, m.Name)) + case m.Local != nil: + errs = append(errs, c.validateLocalMethod(i, m)...) + default: + errs = append(errs, c.validateHTTPMethod(i, m)...) + } case "cli": errs = append(errs, c.validateCLIMethod(i, m)...) case "hybrid": @@ -775,6 +848,19 @@ func (c *Config) Validate() []error { return errs } +// validateLocalMethod checks one method's local metadata route: it needs a +// store path and must not also declare an http/cli route. +func (c *Config) validateLocalMethod(i int, m Method) []error { + var errs []error + if m.Local == nil || strings.TrimSpace(m.Local.Store) == "" { + errs = append(errs, fmt.Errorf("methods[%d] (%s): a local method needs local.store (the metadata file path)", i, m.Name)) + } + if m.HTTP != nil || m.CLI != nil { + errs = append(errs, fmt.Errorf("methods[%d] (%s): a local method must not also declare an http/cli route", i, m.Name)) + } + return errs +} + // validateHTTPMethod checks one method's http route (verb, path, placeholders, // per-param locations). Shared by http and hybrid apps. func (c *Config) validateHTTPMethod(i int, m Method) []error { diff --git a/internal/scaffold/templates/main.go.tmpl b/internal/scaffold/templates/main.go.tmpl index 6872110..0f8a335 100644 --- a/internal/scaffold/templates/main.go.tmpl +++ b/internal/scaffold/templates/main.go.tmpl @@ -177,18 +177,106 @@ func serve(ctx context.Context, socketPath string, d *ipc.Dispatcher) error { {{- if eq .Backend.Type "http"}} func registerHandlers(d *ipc.Dispatcher, c *backend.Client, version, backendURL string) { {{- range .Methods}} - d.Register("{{.Name}}", forward(c, route{ +{{- if .Local}} + d.Register("{{.Name}}", localRead(expandHome("{{.Local.Store}}"))) // {{.Duration}} (local metadata) +{{- else}} + d.Register("{{.Name}}", {{if .HTTP.CaptureTo}}captureWrap(expandHome("{{.HTTP.CaptureTo}}"), {{end}}forward(c, route{ method: "{{.HTTP.Verb}}", pathTmpl: "{{.HTTP.Path}}", bodyVerb: {{.HTTP.BodyVerb}}, pathParams: []string{ {{- range $p := .HTTP.PathParams}}{{printf "%q" $p}}, {{end -}} }, rawPathParams: []string{ {{- range $p := .HTTP.RawPathParams}}{{printf "%q" $p}}, {{end -}} }, queryParams: []string{ {{- range $p := .HTTP.QueryParams}}{{printf "%q" $p}}, {{end -}} }, bodyParams: []string{ {{- range $p := .HTTP.BodyParams}}{{printf "%q" $p}}, {{end -}} }, headerParams: []string{ {{- range $p := .HTTP.HeaderParams}}{{printf "%q" $p}}, {{end -}} }, - }, dur("{{.TimeoutFor}}"))) // {{.Duration}} + }, dur("{{.TimeoutFor}}")){{if .HTTP.CaptureTo}}){{end}}) // {{.Duration}} +{{- end}} {{- end}} d.Register("{{.Namespace}}.help", helpHandler(version, backendURL)) } +// ── local metadata store (host-local number recall) ───────────────────────── +// +// expandHome resolves a leading ~ so a metadata path like ~/.pilot/.agentphone +// points at this host's home. localRead returns the store's JSON (a LocalRoute +// method); captureWrap persists a method's successful response into it so a +// later localRead can recall it (e.g. buy_number → mynumber). + +func expandHome(p string) string { + if strings.HasPrefix(p, "~/") { + if h, err := os.UserHomeDir(); err == nil { + return filepath.Join(h, p[2:]) + } + } + return p +} + +// localRead reads a JSON metadata store and returns it verbatim. A missing, +// empty, or invalid file yields {"entries":[]} — never an error, so an agent can +// always ask "what's mine?" without a backend round-trip. +func localRead(store string) ipc.Handler { + return func(ctx context.Context, _ *ipc.Envelope) (json.RawMessage, error) { + b, err := os.ReadFile(store) + if err != nil || len(strings.TrimSpace(string(b))) == 0 || !json.Valid(b) { + return json.RawMessage(`{"entries":[]}`), nil + } + return append(json.RawMessage(nil), b...), nil + } +} + +// captureWrap wraps an http handler: on a successful (non-error, non-empty) call +// it best-effort appends the JSON response to the store's "entries" array. A +// capture failure never fails the call. +func captureWrap(store string, h ipc.Handler) ipc.Handler { + return func(ctx context.Context, req *ipc.Envelope) (json.RawMessage, error) { + out, err := h(ctx, req) + if err == nil && len(strings.TrimSpace(string(out))) > 0 { + appendEntry(store, out) + } + return out, err + } +} + +// appendEntry merges one JSON-object response into the store's "entries" array, +// deduped by "id" when present, creating the parent dir + file as needed. It only +// captures JSON objects; anything else is ignored. Best-effort — errors swallowed. +func appendEntry(store string, entry json.RawMessage) { + var obj map[string]json.RawMessage + if json.Unmarshal(entry, &obj) != nil { + return + } + var d struct { + Entries []json.RawMessage `json:"entries"` + } + if b, err := os.ReadFile(store); err == nil { + _ = json.Unmarshal(b, &d) + } + if idRaw, ok := obj["id"]; ok { + var id string + if json.Unmarshal(idRaw, &id) == nil && id != "" { + kept := d.Entries[:0] + for _, e := range d.Entries { + var eo map[string]json.RawMessage + var eid string + if json.Unmarshal(e, &eo) == nil { + _ = json.Unmarshal(eo["id"], &eid) + } + if eid != id { + kept = append(kept, e) + } + } + d.Entries = kept + } + } + d.Entries = append(d.Entries, entry) + out, err := json.MarshalIndent(d, "", " ") + if err != nil { + return + } + if dir := filepath.Dir(store); dir != "" { + _ = os.MkdirAll(dir, 0o755) + } + _ = os.WriteFile(store, append(out, '\n'), 0o600) +} + // route is the resolved per-method request shape baked in at generation time: // the verb, the path template (with {name} placeholders), and the per-location // param buckets. A param named in pathParams/rawPathParams fills a placeholder diff --git a/internal/scaffold/templates/manifest.json.tmpl b/internal/scaffold/templates/manifest.json.tmpl index 70d7e9b..0a627f8 100644 --- a/internal/scaffold/templates/manifest.json.tmpl +++ b/internal/scaffold/templates/manifest.json.tmpl @@ -31,6 +31,12 @@ {{- if .HasCLIMethods}} {"cap": "proc.exec", "target": "{{index .Backend.Command 0}}"}, {{- end}} +{{- range .LocalStores}} + {"cap": "fs.read", "target": "{{.Grant}}"}, +{{- if .Write}} + {"cap": "fs.write", "target": "{{.Grant}}"}, +{{- end}} +{{- end}} {{- if .HasAssets}} {"cap": "fs.read", "target": "$APP/install.json"}, {"cap": "fs.write", "target": "$APP"}, diff --git a/internal/scaffold/zz_local_metadata_test.go b/internal/scaffold/zz_local_metadata_test.go new file mode 100644 index 0000000..8b7f28e --- /dev/null +++ b/internal/scaffold/zz_local_metadata_test.go @@ -0,0 +1,118 @@ +package scaffold + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +// localMetaSpec is a managed HTTP app that captures a method's response to a +// host-local metadata file and exposes a local read method over the same file. +const localMetaSpec = ` +id: io.pilot.locx +app_version: 0.1.0 +description: "Local metadata capture + read." +backend: + base_url: https://api.example.com + auth: managed +methods: + - name: locx.buy + summary: "Buy a thing; captures the result to a local store." + http: { verb: POST, path: /v1/things, capture_to: "~/.pilot/.locx" } + - name: locx.mine + summary: "Recall the things this host bought (local, no backend call)." + local: { store: "~/.pilot/.locx" } +` + +// TestGeneratedLocalMetadataCompiles verifies the local-metadata feature wires +// correctly: a local read method + a capture_to on an http method, the fs grants +// for the store, and that the whole project type-checks. +func TestGeneratedLocalMetadataCompiles(t *testing.T) { + if testing.Short() { + t.Skip("skipping compile test in -short mode") + } + goBin, err := exec.LookPath("go") + if err != nil { + t.Skip("go toolchain not available") + } + cfg := parseSpec(t, localMetaSpec) + if errs := cfg.Validate(); len(errs) != 0 { + t.Fatalf("validate: %v", errs) + } + dir := t.TempDir() + if _, err := Generate(cfg, dir); err != nil { + t.Fatalf("generate: %v", err) + } + if sum, err := os.ReadFile(filepath.Join("..", "..", "go.sum")); err == nil { + _ = os.WriteFile(filepath.Join(dir, "go.sum"), sum, 0o644) + } + + main, err := os.ReadFile(filepath.Join(dir, "cmd", cfg.BinaryName, "main.go")) + if err != nil { + t.Fatalf("read main.go: %v", err) + } + for _, want := range []string{ + `localRead(expandHome("~/.pilot/.locx"))`, // local method reads the store + `captureWrap(expandHome("~/.pilot/.locx"), forward`, // http method captures into it + `func appendEntry(`, `func localRead(`, // helpers generated + } { + if !strings.Contains(string(main), want) { + t.Errorf("generated main.go missing: %s", want) + } + } + + // The manifest must grant fs.read + fs.write on the store ($HOME-normalized). + mf, err := os.ReadFile(filepath.Join(dir, "manifest.json")) + if err != nil { + t.Fatalf("read manifest: %v", err) + } + for _, want := range []string{ + `{"cap": "fs.read", "target": "$HOME/.pilot/.locx"}`, + `{"cap": "fs.write", "target": "$HOME/.pilot/.locx"}`, + } { + if !strings.Contains(string(mf), want) { + t.Errorf("manifest missing grant: %s", want) + } + } + + cmd := exec.Command(goBin, "build", "./...") + cmd.Dir = dir + cmd.Env = append(os.Environ(), "GOFLAGS=-mod=mod") + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("generated local-metadata project failed to compile: %v\n%s", err, out) + } +} + +// TestLocalMethodValidation pins the local-route rules. +func TestLocalMethodValidation(t *testing.T) { + base := ` +id: io.pilot.locx +app_version: 0.1.0 +description: "x" +backend: { base_url: https://api.example.com, auth: managed } +methods: +` + cases := []struct { + name string + method string + wantErr bool + }{ + {"valid local", " - {name: locx.mine, summary: m, local: {store: \"~/.pilot/.locx\"}}", false}, + {"local no store", " - {name: locx.mine, summary: m, local: {store: \"\"}}", true}, + {"local + http", " - {name: locx.mine, summary: m, local: {store: \"~/x\"}, http: {verb: GET, path: /x}}", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + c := parseResolved(t, base+tc.method+"\n") + errs := c.Validate() + if tc.wantErr && len(errs) == 0 { + t.Error("expected a validation error, got none") + } + if !tc.wantErr && len(errs) != 0 { + t.Errorf("unexpected errors: %v", errs) + } + }) + } +}