diff --git a/examples/caddy-plugin/Caddyfile b/examples/caddy-plugin/Caddyfile index e6a0b9c..89afc9e 100644 --- a/examples/caddy-plugin/Caddyfile +++ b/examples/caddy-plugin/Caddyfile @@ -4,6 +4,9 @@ httpsig { # This domain should have /.well-known/http-message-signatures-directory defined directory_base http-message-signatures-example.research.cloudflare.com + # Experimental registry draft support. Add more registry lines to trust more cards. + registry https://http-message-signatures-example.research.cloudflare.com/test-registry.txt + fail_on_load_error false } # Responds if signature is valid diff --git a/examples/caddy-plugin/README.md b/examples/caddy-plugin/README.md index 6aee1dc..21c2463 100644 --- a/examples/caddy-plugin/README.md +++ b/examples/caddy-plugin/README.md @@ -5,7 +5,7 @@ [Caddy plugin](https://caddyserver.com/docs/extending-caddy) extending Caddy configuration to allow for validation of web-bot-auth as defined in [draft-meunier-web-bot-auth-architecture](https://thibmeu.github.io/http-message-signatures-directory/draft-meunier-web-bot-auth-architecture.html). -## Tables of Content +## Table of Contents - [Features](#features) - [Usage](#usage) @@ -17,7 +17,10 @@ This is an example plugin and only supports Ed25519. You can find a test key in [Appendix B.1.4 of RFC 9421](https://datatracker.ietf.org/doc/html/rfc9421#name-example-ed25519-test-key). - `httpsig` configuration hook -- Parse HTTP Message Signatures directory +- Load keys from a direct HTTP Message Signatures directory with `directory_base` +- Load keys from registry signature-agent cards with `registry` +- Load multiple inline keys, or multiple keys from `jwks_uri` +- Load IP allowlists from `ips_uri` as defined in [draft-illyes-webbotauth-jafar-00](https://datatracker.ietf.org/doc/html/draft-illyes-webbotauth-jafar-00) - Block request without a valid signature ## Usage @@ -42,6 +45,8 @@ And finally, you run caddy To generate a signed request, you can use the sibling [browser extension](../browser-extension). +`directory_base` is the direct directory mode. `registry` is experimental registry draft support and can be repeated. `fail_on_load_error` defaults to `false`; set it to `true` if Caddy should fail provisioning when directories, registries, cards, IP lists, or keys cannot be loaded. + ## Security Considerations This software has not been audited. Please use at your sole discretion. diff --git a/examples/caddy-plugin/directory.go b/examples/caddy-plugin/directory.go new file mode 100644 index 0000000..099f20f --- /dev/null +++ b/examples/caddy-plugin/directory.go @@ -0,0 +1,64 @@ +// Copyright 2025 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpsig + +import ( + "encoding/json" + "errors" + "net/url" + "strings" + + "go.uber.org/zap" +) + +const directoryPath = "/.well-known/http-message-signatures-directory" + +type Directory struct { + Keys []json.RawMessage `json:"keys"` +} + +func (m *Middleware) addKeysFromDirectory(keys keySpecs, directoryBase string, record func(string, error, ...zap.Field)) { + directoryURL, err := buildDirectoryURL(directoryBase) + if err != nil { + record("failed to parse directory_base", err, zap.String("directory_base", directoryBase)) + return + } + + var directory Directory + if err := fetchJSON(directoryURL, &directory); err != nil { + record("failed to fetch directory", err, zap.String("url", directoryURL)) + return + } + addKeys(keys, directory.Keys, directoryURL, record) +} + +func buildDirectoryURL(directoryBase string) (string, error) { + base := directoryBase + if !strings.Contains(base, "://") { + base = "https://" + base + } + u, err := url.Parse(base) + if err != nil { + return "", err + } + if u.Scheme == "" || u.Host == "" { + return "", errors.New("missing scheme or host") + } + if u.Scheme != "http" && u.Scheme != "https" { + return "", errors.New("unsupported scheme") + } + u.Path = strings.TrimRight(u.Path, "/") + directoryPath + return u.String(), nil +} diff --git a/examples/caddy-plugin/fetch.go b/examples/caddy-plugin/fetch.go new file mode 100644 index 0000000..dece584 --- /dev/null +++ b/examples/caddy-plugin/fetch.go @@ -0,0 +1,67 @@ +// Copyright 2025 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpsig + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "time" +) + +var provisioningHTTPClient = &http.Client{Timeout: 10 * time.Second} + +func parseURL(rawURL string) (*url.URL, error) { + u, err := url.Parse(rawURL) + if err != nil { + return nil, err + } + if u.Scheme == "" || u.Host == "" { + return nil, errors.New("missing scheme or host") + } + return u, nil +} + +func fetchJSON(target string, value interface{}) error { + if err := validateHTTPURL(target); err != nil { + return err + } + return fetchJSONUnchecked(target, value) +} + +func fetchHTTPSJSON(target string, value interface{}) error { + u, err := parseURL(target) + if err != nil { + return err + } + if u.Scheme != "https" { + return errors.New("must use https") + } + return fetchJSONUnchecked(target, value) +} + +func fetchJSONUnchecked(target string, value interface{}) error { + resp, err := provisioningHTTPClient.Get(target) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("status %s", resp.Status) + } + return json.NewDecoder(resp.Body).Decode(value) +} diff --git a/examples/caddy-plugin/handler.go b/examples/caddy-plugin/handler.go index 7997238..bc02e60 100644 --- a/examples/caddy-plugin/handler.go +++ b/examples/caddy-plugin/handler.go @@ -22,60 +22,114 @@ import ( "net/http" "time" + sfv "github.com/dunglas/httpsfv" "github.com/lestrrat-go/jwx/v3/jwk" "github.com/remitly-oss/httpsig-go" "github.com/remitly-oss/httpsig-go/keyman" ) type SignatureValidator struct { - Verifier *httpsig.Verifier + keys httpsig.KeyFetcher } -func NewValidator(keyData []byte) (*SignatureValidator, error) { +type keySpecs map[string]httpsig.KeySpec + +func (ks keySpecs) addJWK(keyData []byte) (string, error) { + keySpec, err := keySpecFromJWK(keyData) + if err != nil { + return "", err + } + if _, exists := ks[keySpec.KeyID]; exists { + return keySpec.KeyID, nil + } + ks[keySpec.KeyID] = keySpec + return keySpec.KeyID, nil +} + +func keySpecFromJWK(keyData []byte) (httpsig.KeySpec, error) { pubKey, err := jwk.ParseKey(keyData) if err != nil { - return nil, fmt.Errorf("parsing public key: %w", err) + return httpsig.KeySpec{}, fmt.Errorf("parsing public key: %w", err) } thumbprint, err := pubKey.Thumbprint(crypto.SHA256) if err != nil { - return nil, fmt.Errorf("cannot generate key id from key: %w", err) + return httpsig.KeySpec{}, fmt.Errorf("cannot generate key id from key: %w", err) + } + keyID := base64.RawURLEncoding.EncodeToString(thumbprint) + publicKey, err := jwk.PublicRawKeyOf(pubKey) + if err != nil { + return httpsig.KeySpec{}, fmt.Errorf("extracting public key: %w", err) + } + + return httpsig.KeySpec{ + KeyID: keyID, + Algo: httpsig.Algo_ED25519, + PubKey: publicKey, + }, nil +} + +func NewValidator(keys keySpecs) (*SignatureValidator, error) { + if len(keys) == 0 { + return nil, errors.New("no signature keys loaded") } - keyid := base64.RawURLEncoding.EncodeToString(thumbprint) - pk, _ := jwk.PublicRawKeyOf(pubKey) - kf := keyman.NewKeyFetchInMemory(map[string]httpsig.KeySpec{ - keyid: { - KeyID: keyid, - Algo: httpsig.Algo_ED25519, - PubKey: pk, - }, - }) + return &SignatureValidator{keys: keyman.NewKeyFetchInMemory(keys)}, nil +} - verifier, err := httpsig.NewVerifier(kf, httpsig.VerifyProfile{ +func verifyProfile(label string) httpsig.VerifyProfile { + return httpsig.VerifyProfile{ AllowedAlgorithms: []httpsig.Algorithm{httpsig.Algo_ED25519}, RequiredFields: httpsig.Fields("@authority"), RequiredMetadata: httpsig.DefaultVerifyProfile.RequiredMetadata, DisallowedMetadata: []httpsig.Metadata{}, CreatedValidDuration: time.Minute * 5, // Signatures must have been created within the last 5 minutes ExpiredSkew: time.Minute, // If the created parameter is present, the Date header cannot be more than a minute off. - }) - if err != nil { - return nil, fmt.Errorf("creating verifier: %w", err) + SignatureLabel: label, } - - return &SignatureValidator{Verifier: verifier}, nil } -func (v *SignatureValidator) Validate(r *http.Request) error { - result, err := v.Verifier.Verify(r) +func signatureLabels(r *http.Request) ([]string, error) { + signatureInput := r.Header.Get("Signature-Input") + if signatureInput == "" { + return nil, errors.New("missing signature-input header") + } + dictionary, err := sfv.UnmarshalDictionary([]string{signatureInput}) if err != nil { - return err + return nil, err } + return dictionary.Names(), nil +} - if !result.Verified { - return errors.New("request not signed or signature is invalid") +func (v *SignatureValidator) Validate(r *http.Request) (string, error) { + labels, err := signatureLabels(r) + if err != nil { + return "", err } + var lastErr error + for _, label := range labels { + verifier, err := httpsig.NewVerifier(v.keys, verifyProfile(label)) + if err != nil { + return "", fmt.Errorf("creating verifier: %w", err) + } + result, err := verifier.Verify(r) + if err != nil { + lastErr = err + continue + } + if !result.Verified { + lastErr = errors.New("request not signed or signature is invalid") + continue + } - return nil + keyID, err := result.KeyID() + if err != nil { + return "", err + } + return keyID, nil + } + if lastErr != nil { + return "", lastErr + } + return "", errors.New("request not signed or signature is invalid") } diff --git a/examples/caddy-plugin/handler_test.go b/examples/caddy-plugin/handler_test.go new file mode 100644 index 0000000..7833eba --- /dev/null +++ b/examples/caddy-plugin/handler_test.go @@ -0,0 +1,175 @@ +// Copyright 2025 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpsig + +import ( + "crypto/ed25519" + "crypto/rand" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + httpsiggo "github.com/remitly-oss/httpsig-go" +) + +func TestNewValidatorRejectsEmptyKeys(t *testing.T) { + _, err := NewValidator(keySpecs{}) + if err == nil { + t.Fatal("expected empty key set to fail") + } +} + +func TestNewValidatorAcceptsMultipleKeys(t *testing.T) { + keys := keySpecs{} + firstJWK, _, firstKeyID := testJWK(t) + secondJWK, secondPrivateKey, secondKeyID := testJWK(t) + + if firstKeyID == secondKeyID { + t.Fatal("test generated duplicate key IDs") + } + if _, err := keys.addJWK(firstJWK); err != nil { + t.Fatalf("add first key: %v", err) + } + if _, err := keys.addJWK(secondJWK); err != nil { + t.Fatalf("add second key: %v", err) + } + + validator, err := NewValidator(keys) + if err != nil { + t.Fatalf("new validator: %v", err) + } + + req := signedRequest(t, secondPrivateKey, secondKeyID) + keyID, err := validator.Validate(req) + if err != nil { + t.Fatalf("validate request signed with second key: %v", err) + } + if keyID != secondKeyID { + t.Fatalf("keyID = %q, want %q", keyID, secondKeyID) + } +} + +func TestKeySpecsCanSkipInvalidKeys(t *testing.T) { + keys := keySpecs{} + if _, err := keys.addJWK(json.RawMessage(`not json`)); err == nil { + t.Fatal("expected invalid key to fail") + } + + validJWK, privateKey, keyID := testJWK(t) + if _, err := keys.addJWK(validJWK); err != nil { + t.Fatalf("add valid key: %v", err) + } + + validator, err := NewValidator(keys) + if err != nil { + t.Fatalf("new validator: %v", err) + } + + req := signedRequest(t, privateKey, keyID) + if _, err := validator.Validate(req); err != nil { + t.Fatalf("validate request signed with valid key: %v", err) + } +} + +func TestNewValidatorAcceptsAnySignatureLabel(t *testing.T) { + keys := keySpecs{} + jwk, privateKey, keyID := testJWK(t) + if _, err := keys.addJWK(jwk); err != nil { + t.Fatalf("add key: %v", err) + } + validator, err := NewValidator(keys) + if err != nil { + t.Fatalf("new validator: %v", err) + } + + req := signedRequestWithLabel(t, privateKey, keyID, "bot-auth") + gotKeyID, err := validator.Validate(req) + if err != nil { + t.Fatalf("validate request: %v", err) + } + if gotKeyID != keyID { + t.Fatalf("keyID = %q, want %q", gotKeyID, keyID) + } +} + +func TestKeySpecsIgnoresDuplicateKeys(t *testing.T) { + keys := keySpecs{} + jwk, _, keyID := testJWK(t) + + firstKeyID, err := keys.addJWK(jwk) + if err != nil { + t.Fatalf("add key: %v", err) + } + secondKeyID, err := keys.addJWK(jwk) + if err != nil { + t.Fatalf("add duplicate key: %v", err) + } + if firstKeyID != keyID || secondKeyID != keyID { + t.Fatalf("key IDs = %q, %q; want %q", firstKeyID, secondKeyID, keyID) + } + if len(keys) != 1 { + t.Fatalf("len(keys) = %d, want 1", len(keys)) + } +} + +func testJWK(t *testing.T) (json.RawMessage, ed25519.PrivateKey, string) { + t.Helper() + + publicKey, privateKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("generate key: %v", err) + } + + jwk := json.RawMessage(fmt.Sprintf( + `{"kty":"OKP","crv":"Ed25519","x":"%s"}`, + base64.RawURLEncoding.EncodeToString(publicKey), + )) + keySpec, err := keySpecFromJWK(jwk) + if err != nil { + t.Fatalf("parse generated JWK: %v", err) + } + + return jwk, privateKey, keySpec.KeyID +} + +func signedRequest(t *testing.T, privateKey ed25519.PrivateKey, keyID string) *http.Request { + t.Helper() + return signedRequestWithLabel(t, privateKey, keyID, "sig1") +} + +func signedRequestWithLabel(t *testing.T, privateKey ed25519.PrivateKey, keyID string, label string) *http.Request { + t.Helper() + + req := httptest.NewRequest("GET", "https://example.com/data", nil) + signer, err := httpsiggo.NewSigner(httpsiggo.SigningProfile{ + Algorithm: httpsiggo.Algo_ED25519, + Fields: httpsiggo.Fields("@authority"), + Metadata: []httpsiggo.Metadata{httpsiggo.MetaCreated, httpsiggo.MetaKeyID}, + Label: label, + }, httpsiggo.SigningKey{ + Key: privateKey, + MetaKeyID: keyID, + }) + if err != nil { + t.Fatalf("new signer: %v", err) + } + if err := signer.Sign(req); err != nil { + t.Fatalf("sign request: %v", err) + } + return req +} diff --git a/examples/caddy-plugin/httpsig.go b/examples/caddy-plugin/httpsig.go index 7100266..902ecd9 100644 --- a/examples/caddy-plugin/httpsig.go +++ b/examples/caddy-plugin/httpsig.go @@ -15,16 +15,21 @@ package httpsig import ( - "encoding/json" + "errors" "fmt" + "net" "net/http" + "strconv" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "go.uber.org/zap" ) +var nopLogger = zap.NewNop() + func init() { caddy.RegisterModule(Middleware{}) httpcaddyfile.RegisterHandlerDirective("httpsig", func(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) { @@ -35,18 +40,15 @@ func init() { ) } -type Directory struct { - Keys []json.RawMessage `json:"keys"` - Purpose *string `json:"purpose,omitempty"` -} - -// Middleware struct to hold the configuration for the handler type Middleware struct { - DirectoryBase string `json:"directory_base"` - validator *SignatureValidator + DirectoryBase string `json:"directory_base,omitempty"` + RegistryURLs []string `json:"registry,omitempty"` + FailOnLoadError bool `json:"fail_on_load_error,omitempty"` + validator *SignatureValidator + keyNets map[string][]*net.IPNet + logger *zap.Logger } -// CaddyModule function to provide module information to Caddy func (m Middleware) CaddyModule() caddy.ModuleInfo { return caddy.ModuleInfo{ ID: "http.handlers.httpsig", @@ -54,40 +56,61 @@ func (m Middleware) CaddyModule() caddy.ModuleInfo { } } -// Provision method for setting up the validator with the public key func (m *Middleware) Provision(ctx caddy.Context) error { - // consider the case where the directory ios localhost - resp, err := http.Get("https://" + m.DirectoryBase + "/.well-known/http-message-signatures-directory") - if err != nil { - return nil + m.logger = ctx.Logger() + keys := keySpecs{} + m.keyNets = map[string][]*net.IPNet{} + loadErrors := []error{} + record := func(message string, err error, fields ...zap.Field) { + if err == nil { + err = errors.New(message) + } + loadErrors = append(loadErrors, fmt.Errorf("%s: %w", message, err)) + m.log().Warn(message, append(fields, zap.Error(err))...) } - defer resp.Body.Close() - var dir Directory - err = json.NewDecoder(resp.Body).Decode(&dir) - if err != nil { - return err + if m.DirectoryBase != "" { + m.addKeysFromDirectory(keys, m.DirectoryBase, record) + } + for _, registryURL := range m.RegistryURLs { + m.addKeysFromRegistry(keys, registryURL, record) } - validator, err := NewValidator(dir.Keys[0]) + validator, err := NewValidator(keys) if err != nil { - return err + record("failed to create validator", err) + } else { + m.validator = validator + } + + if m.FailOnLoadError && len(loadErrors) > 0 { + return errors.Join(loadErrors...) } - m.validator = validator return nil } -// ServeHTTP method to handle the request and validate the signature func (m *Middleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { - if err := m.validator.Validate(r); err != nil { - fmt.Println(err) + if m.validator == nil { + m.log().Warn("HTTP signature validator is unavailable") + http.Error(w, "HTTP signature validator is unavailable", http.StatusServiceUnavailable) + return nil + } + + keyID, err := m.validator.Validate(r) + if err != nil { + m.log().Info("Invalid HTTP signature", zap.Error(err)) http.Error(w, "Invalid HTTP signature", http.StatusUnauthorized) return nil } + + if len(m.keyNets[keyID]) > 0 && !m.remoteIPAllowed(r.RemoteAddr, m.keyNets[keyID]) { + m.log().Info("IP not in allowlist", zap.String("remote_addr", r.RemoteAddr)) + http.Error(w, "IP not allowed", http.StatusUnauthorized) + return nil + } return next.ServeHTTP(w, r) } -// UnmarshalCaddyfile method to allow configuration via the Caddyfile func (m *Middleware) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { for d.Next() { for d.NextBlock(0) { @@ -97,6 +120,20 @@ func (m *Middleware) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { return d.ArgErr() } m.DirectoryBase = d.Val() + case "registry": + if !d.NextArg() { + return d.ArgErr() + } + m.RegistryURLs = append(m.RegistryURLs, d.Val()) + case "fail_on_load_error": + if !d.NextArg() { + return d.ArgErr() + } + value, err := strconv.ParseBool(d.Val()) + if err != nil { + return d.Errf("invalid fail_on_load_error value %q", d.Val()) + } + m.FailOnLoadError = value default: return d.Errf("unknown option '%s'", d.Val()) } @@ -104,3 +141,27 @@ func (m *Middleware) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } return nil } + +func (m *Middleware) log() *zap.Logger { + if m.logger != nil { + return m.logger + } + return nopLogger +} + +func (m *Middleware) remoteIPAllowed(remoteAddr string, nets []*net.IPNet) bool { + host, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + host = remoteAddr + } + ip := net.ParseIP(host) + if ip == nil { + return false + } + for _, ipNet := range nets { + if ipNet.Contains(ip) { + return true + } + } + return false +} diff --git a/examples/caddy-plugin/httpsig_test.go b/examples/caddy-plugin/httpsig_test.go new file mode 100644 index 0000000..72428b5 --- /dev/null +++ b/examples/caddy-plugin/httpsig_test.go @@ -0,0 +1,89 @@ +// Copyright 2025 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpsig + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" +) + +func TestServeHTTPWithoutValidatorReturnsUnavailable(t *testing.T) { + middleware := Middleware{} + recorder := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodGet, "https://example.com/data", nil) + + err := middleware.ServeHTTP(recorder, req, caddyhttp.HandlerFunc(func(http.ResponseWriter, *http.Request) error { + t.Fatal("next handler should not run") + return nil + })) + if err != nil { + t.Fatalf("serve http: %v", err) + } + if recorder.Code != http.StatusServiceUnavailable { + t.Fatalf("status = %d, want %d", recorder.Code, http.StatusServiceUnavailable) + } +} + +func TestUnmarshalCaddyfileSupportsDirectoryAndRegistry(t *testing.T) { + dispenser := caddyfile.NewTestDispenser(`httpsig { + directory_base example.com + registry https://example.com/registry.txt + fail_on_load_error true + }`) + middleware := Middleware{} + + if err := middleware.UnmarshalCaddyfile(dispenser); err != nil { + t.Fatalf("unmarshal caddyfile: %v", err) + } + if middleware.DirectoryBase != "example.com" { + t.Fatalf("directory_base = %q", middleware.DirectoryBase) + } + if len(middleware.RegistryURLs) != 1 || middleware.RegistryURLs[0] != "https://example.com/registry.txt" { + t.Fatalf("registry URLs = %v", middleware.RegistryURLs) + } + if !middleware.FailOnLoadError { + t.Fatal("fail_on_load_error was not parsed") + } +} + +func TestDirectoryURL(t *testing.T) { + url, err := buildDirectoryURL("example.com") + if err != nil { + t.Fatalf("directory URL: %v", err) + } + if url != "https://example.com/.well-known/http-message-signatures-directory" { + t.Fatalf("url = %q", url) + } + + url, err = buildDirectoryURL("http://localhost:8787/base/") + if err != nil { + t.Fatalf("directory URL with scheme: %v", err) + } + if !strings.HasPrefix(url, "http://localhost:8787/base/.well-known/") { + t.Fatalf("url = %q", url) + } +} + +func TestStripRegistryComment(t *testing.T) { + line := stripRegistryComment("https://example.com/card # comment") + if line != "https://example.com/card" { + t.Fatalf("line = %q", line) + } +} diff --git a/examples/caddy-plugin/registry.go b/examples/caddy-plugin/registry.go new file mode 100644 index 0000000..ccdb48c --- /dev/null +++ b/examples/caddy-plugin/registry.go @@ -0,0 +1,175 @@ +// Copyright 2025 Cloudflare, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httpsig + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + + "go.uber.org/zap" +) + +type JWKSResponse struct { + Keys []json.RawMessage `json:"keys"` +} + +type JAFARPrefix struct { + IPv4Prefix string `json:"ipv4Prefix,omitempty"` + IPv6Prefix string `json:"ipv6Prefix,omitempty"` +} + +type JAFARIPList struct { + Prefixes []JAFARPrefix `json:"prefixes"` +} + +type SignatureAgentCard struct { + JWKSUri *string `json:"jwks_uri"` + IPSUri *string `json:"ips_uri"` + Keys []json.RawMessage `json:"keys"` +} + +func (m *Middleware) addKeysFromRegistry(keys keySpecs, registryURL string, record func(string, error, ...zap.Field)) { + for _, cardURL := range cardURLsFromRegistry(registryURL, record) { + m.addKeysFromCard(keys, cardURL, record) + } +} + +func cardURLsFromRegistry(registryURL string, record func(string, error, ...zap.Field)) []string { + if err := validateHTTPURL(registryURL); err != nil { + record("failed to parse registry URL", err, zap.String("url", registryURL)) + return nil + } + + resp, err := provisioningHTTPClient.Get(registryURL) + if err != nil { + record("failed to fetch registry", err, zap.String("url", registryURL)) + return nil + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + record("failed to fetch registry", fmt.Errorf("status %s", resp.Status), zap.String("url", registryURL)) + return nil + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + record("failed to read registry response", err, zap.String("url", registryURL)) + return nil + } + + cardURLs := []string{} + for _, cardURL := range strings.Split(string(body), "\n") { + cardURL = stripRegistryComment(cardURL) + if cardURL != "" { + cardURLs = append(cardURLs, cardURL) + } + } + return cardURLs +} + +func (m *Middleware) addKeysFromCard(keys keySpecs, cardURL string, record func(string, error, ...zap.Field)) { + if err := validateHTTPURL(cardURL); err != nil { + record("failed to parse signature-agent card URL", err, zap.String("url", cardURL)) + return + } + + var card SignatureAgentCard + if err := fetchJSON(cardURL, &card); err != nil { + record("failed to fetch signature-agent card", err, zap.String("url", cardURL)) + return + } + + cardKeys := card.Keys + if card.JWKSUri != nil { + var jwks JWKSResponse + if err := fetchHTTPSJSON(*card.JWKSUri, &jwks); err != nil { + record("failed to fetch jwks_uri", err, zap.String("url", *card.JWKSUri)) + return + } + cardKeys = jwks.Keys + } + keyIDs := addKeys(keys, cardKeys, cardURL, record) + + if card.IPSUri != nil && len(keyIDs) > 0 { + m.addAllowedNets(*card.IPSUri, keyIDs, record) + } +} + +func addKeys(keys keySpecs, jwks []json.RawMessage, source string, record func(string, error, ...zap.Field)) []string { + if len(jwks) == 0 { + record("no keys found", nil, zap.String("url", source)) + return nil + } + keyIDs := []string{} + for _, key := range jwks { + keyID, err := keys.addJWK(key) + if err != nil { + record("failed to import key", err, zap.String("url", source)) + continue + } + keyIDs = append(keyIDs, keyID) + } + return keyIDs +} + +func (m *Middleware) addAllowedNets(ipsURL string, keyIDs []string, record func(string, error, ...zap.Field)) { + var ipList JAFARIPList + if err := fetchHTTPSJSON(ipsURL, &ipList); err != nil { + record("failed to fetch ips_uri", err, zap.String("url", ipsURL)) + return + } + + nets := []*net.IPNet{} + for _, prefix := range ipList.Prefixes { + cidr := prefix.IPv4Prefix + if cidr == "" { + cidr = prefix.IPv6Prefix + } + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + record("failed to parse CIDR", err, zap.String("cidr", cidr)) + continue + } + nets = append(nets, ipNet) + } + for _, keyID := range keyIDs { + m.keyNets[keyID] = append(m.keyNets[keyID], nets...) + } +} + +func stripRegistryComment(line string) string { + for index := 0; index < len(line); index++ { + if line[index] == '#' && (index == 0 || line[index-1] == ' ' || line[index-1] == '\t') { + return strings.TrimSpace(line[:index]) + } + } + return strings.TrimSpace(line) +} + +func validateHTTPURL(rawURL string) error { + u, err := parseURL(rawURL) + if err != nil { + return err + } + if u.Scheme != "http" && u.Scheme != "https" { + return errors.New("unsupported scheme") + } + return nil +} diff --git a/examples/verification-workers/assets/ips.json b/examples/verification-workers/assets/ips.json new file mode 100644 index 0000000..eff5760 --- /dev/null +++ b/examples/verification-workers/assets/ips.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://raw.githubusercontent.com/thibmeu/jafar/refs/tags/draft-illyes-webbotauth-jafar-00/schema.json", + "creationTime": "2026-04-25T20:00:00Z", + "prefixes": [ + { + "ipv4Prefix": "173.245.48.0/20" + }, + { + "ipv4Prefix": "103.21.244.0/22" + }, + { + "ipv4Prefix": "103.22.200.0/22" + }, + { + "ipv4Prefix": "103.31.4.0/22" + }, + { + "ipv4Prefix": "141.101.64.0/18" + }, + { + "ipv4Prefix": "108.162.192.0/18" + }, + { + "ipv4Prefix": "190.93.240.0/20" + }, + { + "ipv4Prefix": "188.114.96.0/20" + }, + { + "ipv4Prefix": "197.234.240.0/22" + }, + { + "ipv4Prefix": "198.41.128.0/17" + }, + { + "ipv4Prefix": "162.158.0.0/15" + }, + { + "ipv4Prefix": "104.16.0.0/13" + }, + { + "ipv4Prefix": "104.24.0.0/14" + }, + { + "ipv4Prefix": "172.64.0.0/13" + }, + { + "ipv4Prefix": "131.0.72.0/22" + }, + { + "ipv6Prefix": "2400:cb00::/32" + }, + { + "ipv6Prefix": "2606:4700::/32" + }, + { + "ipv6Prefix": "2803:f800::/32" + }, + { + "ipv6Prefix": "2405:b500::/32" + }, + { + "ipv6Prefix": "2405:8100::/32" + }, + { + "ipv6Prefix": "2a06:98c0::/29" + }, + { + "ipv6Prefix": "2c0f:f248::/32" + } + ] +} diff --git a/examples/verification-workers/src/index.ts b/examples/verification-workers/src/index.ts index 37a7793..dee04eb 100644 --- a/examples/verification-workers/src/index.ts +++ b/examples/verification-workers/src/index.ts @@ -17,10 +17,13 @@ import { HTTP_MESSAGE_SIGNATURES_DIRECTORY, MediaType, Signer, + SignatureAgentCard, VerificationParams, directoryResponseHeaders, helpers, jwkToKeyID, + parseRegistry, + parseSignatureAgentCard, signatureHeaders, verify, } from "web-bot-auth"; @@ -52,6 +55,34 @@ async function getExampleDirectory(): Promise { }; } +function signatureAgentOrigin(env: Env): string { + return new URL(env.SIGNATURE_AGENT).origin; +} + +function getSignatureAgentCard(env: Env): SignatureAgentCard { + const origin = signatureAgentOrigin(env); + return parseSignatureAgentCard({ + client_name: "Example Bot", + client_uri: origin, + logo_uri: `${origin}/favicon.png`, + contacts: [], + "expected-user-agent": "Mozilla/5.0 ExampleBot", + "rfc9309-product-token": "ExampleBot", + "rfc9309-compliance": ["User-Agent", "Allow", "Disallow", "Content-Usage"], + trigger: "fetcher", + purpose: "example", + "rate-control": "429", + jwks_uri: `${origin}${HTTP_MESSAGE_SIGNATURES_DIRECTORY}`, + ips_uri: `${origin}/ips.json`, + }); +} + +function registryResponse(env: Env): Response { + const registry = `${signatureAgentOrigin(env)}/signature-agent-card\n`; + parseRegistry(registry); + return new Response(registry, { headers: { "content-type": "text/plain" } }); +} + async function fetchDirectory(signatureAgent: string): Promise { // make "some" validatation of the Signature-Agent header before making a request let parsed: string; @@ -208,6 +239,18 @@ export default { return response; } + if (url.pathname === "/signature-agent-card") { + return Response.json(getSignatureAgentCard(env)); + } + + if (url.pathname === "/test-registry.txt") { + return registryResponse(env); + } + + if (url.pathname === "/ips.json") { + return env.ASSETS.fetch(request); + } + const status = await verifySignature(env, request); switch (status) { case SignatureValidationStatus.NEUTRAL: diff --git a/examples/verification-workers/test/index.spec.ts b/examples/verification-workers/test/index.spec.ts index f4fbb63..553d258 100644 --- a/examples/verification-workers/test/index.spec.ts +++ b/examples/verification-workers/test/index.spec.ts @@ -27,6 +27,7 @@ import worker from "../src/index"; const IncomingRequest = Request; const sampleURL = "https://example.com"; +const signatureAgentOrigin = new URL(env.SIGNATURE_AGENT).origin; const proxyURL = `${sampleURL}/v0/api/proxy-directory`; const directoryURL = `${sampleURL}/.well-known/http-message-signatures-directory`; const turnstileVerifyURL = @@ -192,6 +193,32 @@ describe("/.well-known/http-message-signatures-directory endpoint", () => { }); }); +describe("registry draft endpoints", () => { + it("serves a signature-agent card", async () => { + const response = await SELF.fetch(`${sampleURL}/signature-agent-card`); + + expect(response.status).toEqual(200); + expect(response.headers.get("content-type")).toContain("application/json"); + expect(await response.json()).toMatchObject({ + jwks_uri: `${signatureAgentOrigin}/.well-known/http-message-signatures-directory`, + ips_uri: `${signatureAgentOrigin}/ips.json`, + }); + }); + + it("serves a test registry", async () => { + const response = await SELF.fetch(`${sampleURL}/test-registry.txt`); + + expect(response.status).toEqual(200); + expect(await response.text()).toContain("/signature-agent-card"); + }); + + it("serves a JAFAR IP list", async () => { + const response = await SELF.fetch(`${sampleURL}/ips.json`); + + expect(response.status).toEqual(200); + }); +}); + describe("/v0/api/proxy-directory endpoint", () => { it("rejects non-POST requests", async () => { const request = new IncomingRequest(proxyURL, { diff --git a/examples/verification-workers/worker-configuration.d.ts b/examples/verification-workers/worker-configuration.d.ts index 6ddc697..5eb5f81 100644 --- a/examples/verification-workers/worker-configuration.d.ts +++ b/examples/verification-workers/worker-configuration.d.ts @@ -3,10 +3,11 @@ // Runtime types generated with workerd@1.20250508.0 2025-04-06 declare namespace Cloudflare { interface Env { - SIGNATURE_AGENT: "http-message-signatures-example.research.cloudflare.com"; - TARGET_URL: "https://http-message-signatures-example.research.cloudflare.com/debug"; + SIGNATURE_AGENT: "https://http-message-signatures-example.research.cloudflare.com"; + TARGET_URL: "https://research.cloudflare.com/web-bot-auth-test/0.0.1"; TURNSTILE_SECRET_KEY: string; TURNSTILE_SITE_KEY: string; + ASSETS: Fetcher; } } interface Env extends Cloudflare.Env {} diff --git a/examples/verification-workers/wrangler.jsonc b/examples/verification-workers/wrangler.jsonc index 053e79a..4bbafb9 100644 --- a/examples/verification-workers/wrangler.jsonc +++ b/examples/verification-workers/wrangler.jsonc @@ -20,4 +20,8 @@ "SIGNATURE_AGENT": "https://http-message-signatures-example.research.cloudflare.com", "TARGET_URL": "https://research.cloudflare.com/web-bot-auth-test/0.0.1", }, + "assets": { + "directory": "./assets", + "binding": "ASSETS", + }, } diff --git a/packages/web-bot-auth/src/index.ts b/packages/web-bot-auth/src/index.ts index a83cfda..9ef8e85 100644 --- a/packages/web-bot-auth/src/index.ts +++ b/packages/web-bot-auth/src/index.ts @@ -211,3 +211,9 @@ export function verify( export interface Directory extends httpsig.Directory { purpose: string; } + +export { + parseRegistry, + parseSignatureAgentCard, + type SignatureAgentCard, +} from "./registry"; diff --git a/packages/web-bot-auth/src/registry.ts b/packages/web-bot-auth/src/registry.ts new file mode 100644 index 0000000..e4b81d9 --- /dev/null +++ b/packages/web-bot-auth/src/registry.ts @@ -0,0 +1,198 @@ +export interface SignatureAgentCard { + client_name?: string; + client_uri?: string; + logo_uri?: string; + contacts?: string[]; + "expected-user-agent"?: string; + "rfc9309-product-token"?: string; + "rfc9309-compliance"?: string[]; + trigger?: "fetcher" | "crawler"; + purpose?: string; + "targeted-content"?: string; + "rate-control"?: string; + "rate-expectation"?: string; + "known-urls"?: string[]; + jwks_uri?: string; + ips_uri?: string; + keys?: unknown[]; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function stringArray(value: unknown, field: string): string[] | undefined { + if (value === undefined) { + return undefined; + } + if ( + !Array.isArray(value) || + !value.every((entry) => typeof entry === "string") + ) { + throw new Error(`${field} must be an array of strings`); + } + return value; +} + +function uriArray(value: unknown, field: string): string[] | undefined { + const values = stringArray(value, field); + if (values === undefined) { + return undefined; + } + for (const uri of values) { + new URL(uri); + } + return values; +} + +function stringValue(value: unknown, field: string): string | undefined { + if (value === undefined) { + return undefined; + } + if (typeof value !== "string") { + throw new Error(`${field} must be a string`); + } + return value; +} + +function urlValue( + value: unknown, + field: string, + allowedSchemes: string[] +): string | undefined { + const parsed = stringValue(value, field); + if (parsed === undefined) { + return undefined; + } + const url = new URL(parsed); + if (!allowedSchemes.includes(url.protocol.slice(0, -1))) { + throw new Error(`${field} must use one of: ${allowedSchemes.join(", ")}`); + } + return parsed; +} + +function clientURIValue(value: unknown): string | undefined { + const parsed = stringValue(value, "client_uri"); + if (parsed === undefined) { + return undefined; + } + const url = new URL(parsed); + if (url.protocol === "data:") { + if ( + !parsed.startsWith("data:text/plain,") && + !parsed.startsWith("data:text/plain;") + ) { + throw new Error("client_uri data URL must use text/plain"); + } + return parsed; + } + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error("client_uri must use one of: http, https, data:text/plain"); + } + return parsed; +} + +function stripRegistryComment(line: string): string { + for (let index = 0; index < line.length; index += 1) { + if ( + line.charAt(index) === "#" && + (index === 0 || /\s/.test(line.charAt(index - 1))) + ) { + return line.slice(0, index).trim(); + } + } + return line.trim(); +} + +/** + * Parse a draft-meunier-webbotauth-registry-02 registry. + * + * This implementation intentionally supports only HTTP(S) card URLs. Inline + * data: cards are not supported by this package. + */ +export function parseRegistry(registry: string): URL[] { + const entries: URL[] = []; + for (const [index, rawLine] of registry.split(/\r\n|\n|\r/).entries()) { + const line = stripRegistryComment(rawLine); + if (line === "") { + continue; + } + + let url: URL; + try { + url = new URL(line); + } catch (error) { + throw new Error(`registry line ${index + 1} is not a URL`, { + cause: error, + }); + } + + if (url.protocol !== "http:" && url.protocol !== "https:") { + throw new Error(`registry line ${index + 1} uses unsupported scheme`); + } + entries.push(url); + } + return entries; +} + +export function parseSignatureAgentCard(input: unknown): SignatureAgentCard { + if (!isRecord(input)) { + throw new Error("signature agent card must be an object"); + } + if (Object.keys(input).length === 0) { + throw new Error("signature agent card must contain at least one parameter"); + } + + const card: SignatureAgentCard = {}; + card.client_name = stringValue(input.client_name, "client_name"); + card.client_uri = clientURIValue(input.client_uri); + card.logo_uri = urlValue(input.logo_uri, "logo_uri", [ + "http", + "https", + "data", + ]); + card.contacts = uriArray(input.contacts, "contacts"); + card["expected-user-agent"] = stringValue( + input["expected-user-agent"], + "expected-user-agent" + ); + card["rfc9309-product-token"] = stringValue( + input["rfc9309-product-token"], + "rfc9309-product-token" + ); + card["rfc9309-compliance"] = stringArray( + input["rfc9309-compliance"], + "rfc9309-compliance" + ); + + const trigger = stringValue(input.trigger, "trigger"); + if (trigger !== undefined) { + if (trigger !== "fetcher" && trigger !== "crawler") { + throw new Error("trigger must be fetcher or crawler"); + } + card.trigger = trigger; + } + + card.purpose = stringValue(input.purpose, "purpose"); + card["targeted-content"] = stringValue( + input["targeted-content"], + "targeted-content" + ); + card["rate-control"] = stringValue(input["rate-control"], "rate-control"); + card["rate-expectation"] = stringValue( + input["rate-expectation"], + "rate-expectation" + ); + card["known-urls"] = stringArray(input["known-urls"], "known-urls"); + card.jwks_uri = urlValue(input.jwks_uri, "jwks_uri", ["https"]); + card.ips_uri = urlValue(input.ips_uri, "ips_uri", ["https"]); + + if (input.keys !== undefined) { + if (!Array.isArray(input.keys)) { + throw new Error("keys must be an array"); + } + card.keys = input.keys; + } + + return card; +} diff --git a/packages/web-bot-auth/test/registry.test.ts b/packages/web-bot-auth/test/registry.test.ts new file mode 100644 index 0000000..ad59e1f --- /dev/null +++ b/packages/web-bot-auth/test/registry.test.ts @@ -0,0 +1,82 @@ +import { describe, it, expect } from "vitest"; +import { parseRegistry, parseSignatureAgentCard } from "../src/index"; + +import vectors from "./test_data/web_bot_auth_registry_v1.json"; + +type Vectors = (typeof vectors)[number]; + +describe.each(vectors)("Web-bot-auth-registry-Vector-%#", (v: Vectors) => { + it("should pass IETF draft registry test vectors", () => { + expect(parseRegistry(v.registry_txt).map((entry) => entry.href)).toEqual( + v.signature_agent_cards + ); + }); +}); + +describe("parseRegistry", () => { + it("ignores blank lines and comments", () => { + expect( + parseRegistry(` + # bots + https://bot.example/card # local comment + + http://localhost:8787/card + `).map((entry) => entry.href) + ).toEqual(["https://bot.example/card", "http://localhost:8787/card"]); + }); + + it("rejects unsupported schemes", () => { + expect(() => parseRegistry("ftp://example.com/card")).toThrow( + "unsupported scheme" + ); + }); + + it("does not support inline data cards", () => { + expect(() => + parseRegistry('data:application/json,{"client_name":"Inline Bot"}') + ).toThrow("unsupported scheme"); + }); +}); + +describe("parseSignatureAgentCard", () => { + it("validates known card parameters", () => { + expect( + parseSignatureAgentCard({ + client_name: "Example Bot", + trigger: "fetcher", + jwks_uri: + "https://example.com/.well-known/http-message-signatures-directory", + ips_uri: "https://example.com/ips.json", + }) + ).toMatchObject({ + client_name: "Example Bot", + trigger: "fetcher", + }); + }); + + it("rejects invalid card parameters", () => { + expect(() => + parseSignatureAgentCard({ jwks_uri: "http://example.com/jwks" }) + ).toThrow("jwks_uri must use one of: https"); + }); + + it("ignores unknown parameters", () => { + expect(parseSignatureAgentCard({ unknown: true })).toEqual({}); + }); + + it("rejects empty cards", () => { + expect(() => parseSignatureAgentCard({})).toThrow("at least one parameter"); + }); + + it("requires client_uri data URLs to be text/plain", () => { + expect(() => + parseSignatureAgentCard({ client_uri: "data:image/png,abc" }) + ).toThrow("text/plain"); + }); + + it("requires contacts to be URIs", () => { + expect(() => + parseSignatureAgentCard({ contacts: ["not a uri"] }) + ).toThrow(); + }); +}); diff --git a/packages/web-bot-auth/test/test_data/web_bot_auth_registry_v1.json b/packages/web-bot-auth/test/test_data/web_bot_auth_registry_v1.json new file mode 100644 index 0000000..6c9db27 --- /dev/null +++ b/packages/web-bot-auth/test/test_data/web_bot_auth_registry_v1.json @@ -0,0 +1,15 @@ +[ + { + "registry_txt": "https://example.com/.well-known/signature-agent-card", + "signature_agent_cards": [ + "https://example.com/.well-known/signature-agent-card" + ] + }, + { + "registry_txt": "https://bot1.example.com/.well-known/signature-agent-card\nhttps://bot2.example.com/.well-known/signature-agent-card", + "signature_agent_cards": [ + "https://bot1.example.com/.well-known/signature-agent-card", + "https://bot2.example.com/.well-known/signature-agent-card" + ] + } +]