From 424ce16df5bb5fe72afb0af7cde256b3b58fc172 Mon Sep 17 00:00:00 2001 From: Cai_Tang Date: Sat, 28 Mar 2026 16:23:44 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=8F=90=E5=8D=87=20WebFetch=20?= =?UTF-8?q?=E7=BD=91=E9=A1=B5=E6=AD=A3=E6=96=87=E6=8F=90=E5=8F=96=E4=B8=8E?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E5=8F=AF=E6=B6=88=E8=B4=B9=E6=80=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- go.mod | 7 +- go.sum | 14 +- internal/app/bootstrap.go | 25 +- internal/app/bootstrap_test.go | 45 ++++ internal/config/config_test.go | 54 +++++ internal/config/model.go | 137 +++++++++++ internal/tools/webfetch/html.go | 112 +++++++++ internal/tools/webfetch/tool.go | 331 ++++++++++++++++++++++++--- internal/tools/webfetch/tool_test.go | 297 +++++++++++++++++++++--- 9 files changed, 950 insertions(+), 72 deletions(-) create mode 100644 internal/tools/webfetch/html.go diff --git a/go.mod b/go.mod index 0db3a58d..563ad14d 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,8 @@ require ( github.com/charmbracelet/bubbles v1.0.0 github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 + github.com/joho/godotenv v1.5.1 + golang.org/x/net v0.50.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -20,7 +22,6 @@ require ( github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.5.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect - github.com/joho/godotenv v1.5.1 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-localereader v0.0.1 // indirect @@ -31,6 +32,6 @@ require ( github.com/rivo/uniseg v0.4.7 // indirect github.com/sahilm/fuzzy v0.1.1 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/sys v0.38.0 // indirect - golang.org/x/text v0.3.8 // indirect + golang.org/x/sys v0.41.0 // indirect + golang.org/x/text v0.34.0 // indirect ) diff --git a/go.sum b/go.sum index d5645656..8adf72f8 100644 --- a/go.sum +++ b/go.sum @@ -56,12 +56,18 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= +golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= -golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= +golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= +golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8= +golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/app/bootstrap.go b/internal/app/bootstrap.go index 2639538f..d7999b7b 100644 --- a/internal/app/bootstrap.go +++ b/internal/app/bootstrap.go @@ -23,14 +23,7 @@ func NewProgram(ctx context.Context) (*tea.Program, error) { return nil, err } - toolRegistry := tools.NewRegistry() - toolRegistry.Register(filesystem.New(cfg.Workdir)) - toolRegistry.Register(filesystem.NewWrite(cfg.Workdir)) - toolRegistry.Register(filesystem.NewGrep(cfg.Workdir)) - toolRegistry.Register(filesystem.NewGlob(cfg.Workdir)) - toolRegistry.Register(filesystem.NewEdit(cfg.Workdir)) - toolRegistry.Register(bash.New(cfg.Workdir, cfg.Shell, time.Duration(cfg.ToolTimeoutSec)*time.Second)) - toolRegistry.Register(webfetch.New(time.Duration(cfg.ToolTimeoutSec) * time.Second)) + toolRegistry := buildToolRegistry(cfg) sessionStore := agentruntime.NewSessionStore(loader.BaseDir()) runtimeSvc := agentruntime.New(manager, toolRegistry, sessionStore) @@ -45,3 +38,19 @@ func NewProgram(ctx context.Context) (*tea.Program, error) { tea.WithMouseCellMotion(), ), nil } + +func buildToolRegistry(cfg config.Config) *tools.Registry { + toolRegistry := tools.NewRegistry() + toolRegistry.Register(filesystem.New(cfg.Workdir)) + toolRegistry.Register(filesystem.NewWrite(cfg.Workdir)) + toolRegistry.Register(filesystem.NewGrep(cfg.Workdir)) + toolRegistry.Register(filesystem.NewGlob(cfg.Workdir)) + toolRegistry.Register(filesystem.NewEdit(cfg.Workdir)) + toolRegistry.Register(bash.New(cfg.Workdir, cfg.Shell, time.Duration(cfg.ToolTimeoutSec)*time.Second)) + toolRegistry.Register(webfetch.New(webfetch.Config{ + Timeout: time.Duration(cfg.ToolTimeoutSec) * time.Second, + MaxResponseBytes: cfg.Tools.WebFetch.MaxResponseBytes, + SupportedContentTypes: cfg.Tools.WebFetch.SupportedContentTypes, + })) + return toolRegistry +} diff --git a/internal/app/bootstrap_test.go b/internal/app/bootstrap_test.go index a25e7a4d..5974d9ac 100644 --- a/internal/app/bootstrap_test.go +++ b/internal/app/bootstrap_test.go @@ -2,9 +2,15 @@ package app import ( "context" + "encoding/json" + "net/http" + "net/http/httptest" "os" "path/filepath" "testing" + + "github.com/dust/neo-code/internal/config" + "github.com/dust/neo-code/internal/tools" ) func TestNewProgram(t *testing.T) { @@ -25,3 +31,42 @@ func TestNewProgram(t *testing.T) { t.Fatalf("expected config file to be created at %q: %v", configPath, err) } } + +func TestBuildToolRegistryUsesWebFetchConfig(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("1234567890")) + })) + defer server.Close() + + cfg := config.Default().Clone() + cfg.Workdir = t.TempDir() + cfg.Tools.WebFetch.MaxResponseBytes = 4 + + registry := buildToolRegistry(cfg) + tool, err := registry.Get("webfetch") + if err != nil { + t.Fatalf("registry.Get(webfetch) error = %v", err) + } + + args, err := json.Marshal(map[string]string{"url": server.URL}) + if err != nil { + t.Fatalf("marshal args: %v", err) + } + + result, execErr := tool.Execute(context.Background(), tools.ToolCallInput{ + Name: "webfetch", + Arguments: args, + }) + if execErr != nil { + t.Fatalf("webfetch execute error = %v", execErr) + } + if truncated, ok := result.Metadata["truncated"].(bool); !ok || !truncated { + t.Fatalf("expected truncated metadata, got %+v", result.Metadata) + } + if result.Content == "" { + t.Fatalf("expected formatted webfetch content") + } +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 108317d1..578d824f 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -24,6 +24,12 @@ selected_provider: openai current_model: gpt-5.4 workdir: . shell: powershell +tools: + webfetch: + max_response_bytes: 4096 + supported_content_types: + - text/html + - text/plain providers: - name: openai type: openai @@ -43,6 +49,12 @@ providers: if provider.BaseURL != "https://example.com/v1" { t.Fatalf("expected custom base url, got %q", provider.BaseURL) } + if cfg.Tools.WebFetch.MaxResponseBytes != 4096 { + t.Fatalf("expected custom max_response_bytes 4096, got %d", cfg.Tools.WebFetch.MaxResponseBytes) + } + if len(cfg.Tools.WebFetch.SupportedContentTypes) != 2 { + t.Fatalf("expected 2 supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes) + } }, }, { @@ -73,6 +85,12 @@ providers: if provider.Model != "gpt-4o" { t.Fatalf("expected provider model gpt-4o, got %q", provider.Model) } + if cfg.Tools.WebFetch.MaxResponseBytes != DefaultWebFetchMaxResponseBytes { + t.Fatalf("expected default max_response_bytes %d, got %d", DefaultWebFetchMaxResponseBytes, cfg.Tools.WebFetch.MaxResponseBytes) + } + if len(cfg.Tools.WebFetch.SupportedContentTypes) != len(DefaultWebFetchSupportedContentTypes()) { + t.Fatalf("expected default supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes) + } }, }, } @@ -362,6 +380,12 @@ func TestConfigApplyDefaultsFillsMissingFields(t *testing.T) { if !filepath.IsAbs(cfg.Workdir) { t.Fatalf("expected absolute workdir, got %q", cfg.Workdir) } + if cfg.Tools.WebFetch.MaxResponseBytes != DefaultWebFetchMaxResponseBytes { + t.Fatalf("expected default webfetch max_response_bytes %d, got %d", DefaultWebFetchMaxResponseBytes, cfg.Tools.WebFetch.MaxResponseBytes) + } + if len(cfg.Tools.WebFetch.SupportedContentTypes) != len(DefaultWebFetchSupportedContentTypes()) { + t.Fatalf("expected default supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes) + } } func TestConfigValidateFailures(t *testing.T) { @@ -423,6 +447,24 @@ func TestConfigValidateFailures(t *testing.T) { }(), expectErr: "model is empty", }, + { + name: "invalid webfetch max response bytes", + config: func() *Config { + cfg := validConfig.Clone() + cfg.Tools.WebFetch.MaxResponseBytes = 0 + return &cfg + }(), + expectErr: "max_response_bytes must be greater than 0", + }, + { + name: "invalid webfetch supported content types", + config: func() *Config { + cfg := validConfig.Clone() + cfg.Tools.WebFetch.SupportedContentTypes = []string{""} + return &cfg + }(), + expectErr: "supported_content_types[0] is empty", + }, } for _, tt := range tests { @@ -542,6 +584,8 @@ func TestLoaderLoadAndSaveRoundTrip(t *testing.T) { cfg.Providers[i].Model = "gpt-5.4" } } + cfg.Tools.WebFetch.MaxResponseBytes = 1024 + cfg.Tools.WebFetch.SupportedContentTypes = []string{"text/html", "application/json"} if err := loader.Save(context.Background(), cfg); err != nil { t.Fatalf("Save() error = %v", err) } @@ -553,6 +597,12 @@ func TestLoaderLoadAndSaveRoundTrip(t *testing.T) { if reloaded.CurrentModel != "gpt-5.4" { t.Fatalf("expected current model %q, got %q", "gpt-5.4", reloaded.CurrentModel) } + if reloaded.Tools.WebFetch.MaxResponseBytes != 1024 { + t.Fatalf("expected max_response_bytes %d, got %d", 1024, reloaded.Tools.WebFetch.MaxResponseBytes) + } + if len(reloaded.Tools.WebFetch.SupportedContentTypes) != 2 { + t.Fatalf("expected persisted supported content types, got %+v", reloaded.Tools.WebFetch.SupportedContentTypes) + } } func TestNormalizeWorkdirAndClone(t *testing.T) { @@ -621,9 +671,13 @@ func TestNormalizeWorkdirAndClone(t *testing.T) { cfg := Default() cloned := cfg.Clone() cloned.CurrentModel = "modified" + cloned.Tools.WebFetch.SupportedContentTypes[0] = "application/json" if cfg.CurrentModel == cloned.CurrentModel { t.Fatalf("expected clone to be independent from source") } + if cfg.Tools.WebFetch.SupportedContentTypes[0] == cloned.Tools.WebFetch.SupportedContentTypes[0] { + t.Fatalf("expected webfetch supported content types to be cloned") + } } func TestManagerHelperMethodsAndReloads(t *testing.T) { diff --git a/internal/config/model.go b/internal/config/model.go index 63ecb697..d05e398f 100644 --- a/internal/config/model.go +++ b/internal/config/model.go @@ -3,6 +3,7 @@ package config import ( "errors" "fmt" + "mime" "os" "path/filepath" goruntime "runtime" @@ -30,8 +31,19 @@ const ( DefaultWorkdir = "." DefaultMaxLoops = 8 DefaultToolTimeoutSec = 20 + // DefaultWebFetchMaxResponseBytes bounds the amount of web content returned to the model. + DefaultWebFetchMaxResponseBytes int64 = 256 * 1024 ) +var defaultWebFetchSupportedContentTypes = []string{ + "text/html", + "application/xhtml+xml", + "text/plain", + "application/json", + "application/xml", + "text/xml", +} + type Config struct { Providers []ProviderConfig `yaml:"providers"` SelectedProvider string `yaml:"selected_provider"` @@ -40,6 +52,7 @@ type Config struct { Shell string `yaml:"shell"` MaxLoops int `yaml:"max_loops,omitempty"` ToolTimeoutSec int `yaml:"tool_timeout_sec,omitempty"` + Tools ToolsConfig `yaml:"tools,omitempty"` } type ProviderConfig struct { @@ -60,6 +73,22 @@ type ModelOption struct { Description string } +// ToolsConfig stores tool-specific configuration values. +type ToolsConfig struct { + WebFetch WebFetchConfig `yaml:"webfetch,omitempty"` +} + +// WebFetchConfig controls response filtering and limits for the webfetch tool. +type WebFetchConfig struct { + MaxResponseBytes int64 `yaml:"max_response_bytes,omitempty"` + SupportedContentTypes []string `yaml:"supported_content_types,omitempty"` +} + +// DefaultWebFetchSupportedContentTypes returns the default media types accepted by webfetch. +func DefaultWebFetchSupportedContentTypes() []string { + return append([]string(nil), defaultWebFetchSupportedContentTypes...) +} + func Default() *Config { return &Config{ Providers: []ProviderConfig{ @@ -91,6 +120,9 @@ func Default() *Config { Shell: defaultShell(), MaxLoops: DefaultMaxLoops, ToolTimeoutSec: DefaultToolTimeoutSec, + Tools: ToolsConfig{ + WebFetch: defaultWebFetchConfig(), + }, } } @@ -112,6 +144,7 @@ func (c *Config) Clone() Config { clone := *c clone.Providers = append([]ProviderConfig(nil), c.Providers...) + clone.Tools = c.Tools.Clone() return clone } @@ -148,6 +181,7 @@ func (c *Config) ApplyDefaults() { if c.ToolTimeoutSec <= 0 { c.ToolTimeoutSec = def.ToolTimeoutSec } + c.Tools.ApplyDefaults(def.Tools) c.Workdir = normalizeWorkdir(c.Workdir) } @@ -192,6 +226,9 @@ func (c *Config) Validate() error { if strings.TrimSpace(selected.Model) == "" { return fmt.Errorf("config: selected provider %q has empty model", selected.Name) } + if err := c.Tools.Validate(); err != nil { + return fmt.Errorf("config: tools: %w", err) + } return nil } @@ -344,3 +381,103 @@ func defaultShell() string { } return "bash" } + +func defaultWebFetchConfig() WebFetchConfig { + return WebFetchConfig{ + MaxResponseBytes: DefaultWebFetchMaxResponseBytes, + SupportedContentTypes: DefaultWebFetchSupportedContentTypes(), + } +} + +func (c ToolsConfig) Clone() ToolsConfig { + return ToolsConfig{ + WebFetch: c.WebFetch.Clone(), + } +} + +func (c *ToolsConfig) ApplyDefaults(def ToolsConfig) { + if c == nil { + return + } + + c.WebFetch.ApplyDefaults(def.WebFetch) +} + +func (c ToolsConfig) Validate() error { + if err := c.WebFetch.Validate(); err != nil { + return fmt.Errorf("webfetch: %w", err) + } + return nil +} + +func (c WebFetchConfig) Clone() WebFetchConfig { + clone := c + clone.SupportedContentTypes = append([]string(nil), c.SupportedContentTypes...) + return clone +} + +func (c *WebFetchConfig) ApplyDefaults(def WebFetchConfig) { + if c == nil { + return + } + + if c.MaxResponseBytes <= 0 { + c.MaxResponseBytes = def.MaxResponseBytes + } + c.SupportedContentTypes = normalizeContentTypes(c.SupportedContentTypes, def.SupportedContentTypes) +} + +func (c WebFetchConfig) Validate() error { + if c.MaxResponseBytes <= 0 { + return errors.New("max_response_bytes must be greater than 0") + } + if len(c.SupportedContentTypes) == 0 { + return errors.New("supported_content_types is empty") + } + + for i, contentType := range c.SupportedContentTypes { + if normalizeContentType(contentType) == "" { + return fmt.Errorf("supported_content_types[%d] is empty", i) + } + } + return nil +} + +func normalizeContentTypes(values []string, defaults []string) []string { + source := values + if len(source) == 0 { + source = defaults + } + + normalized := make([]string, 0, len(source)) + seen := make(map[string]struct{}, len(source)) + for _, value := range source { + contentType := normalizeContentType(value) + if contentType == "" { + continue + } + if _, exists := seen[contentType]; exists { + continue + } + seen[contentType] = struct{}{} + normalized = append(normalized, contentType) + } + return normalized +} + +func normalizeContentType(value string) string { + trimmed := strings.ToLower(strings.TrimSpace(value)) + if trimmed == "" { + return "" + } + + mediaType, _, err := mime.ParseMediaType(trimmed) + if err == nil { + return mediaType + } + + if index := strings.Index(trimmed, ";"); index >= 0 { + return strings.TrimSpace(trimmed[:index]) + } + return trimmed +} diff --git a/internal/tools/webfetch/html.go b/internal/tools/webfetch/html.go new file mode 100644 index 00000000..c7ac6f25 --- /dev/null +++ b/internal/tools/webfetch/html.go @@ -0,0 +1,112 @@ +package webfetch + +import ( + "bytes" + "fmt" + "strings" + + "golang.org/x/net/html" +) + +var skippedHTMLTags = map[string]struct{}{ + "canvas": {}, + "head": {}, + "noscript": {}, + "script": {}, + "style": {}, + "svg": {}, + "template": {}, +} + +func extractHTMLContent(body []byte) (string, string, error) { + doc, err := html.Parse(bytes.NewReader(body)) + if err != nil { + return "", "", fmt.Errorf("parse document: %w", err) + } + + root := findElement(doc, "body") + if root == nil { + root = doc + } + + text := normalizeWhitespace(collectVisibleText(root)) + title := normalizeWhitespace(textContent(findElement(doc, "title"))) + return text, title, nil +} + +func findElement(node *html.Node, tag string) *html.Node { + if node == nil { + return nil + } + if node.Type == html.ElementNode && strings.EqualFold(node.Data, tag) { + return node + } + for child := node.FirstChild; child != nil; child = child.NextSibling { + if found := findElement(child, tag); found != nil { + return found + } + } + return nil +} + +func collectVisibleText(node *html.Node) string { + var builder strings.Builder + appendVisibleText(&builder, node) + return builder.String() +} + +func appendVisibleText(builder *strings.Builder, node *html.Node) { + if node == nil { + return + } + if shouldSkipNode(node) { + return + } + + if node.Type == html.TextNode { + text := strings.TrimSpace(node.Data) + if text != "" { + builder.WriteString(text) + builder.WriteByte('\n') + } + } + + for child := node.FirstChild; child != nil; child = child.NextSibling { + appendVisibleText(builder, child) + } +} + +func shouldSkipNode(node *html.Node) bool { + if node == nil || node.Type != html.ElementNode { + return false + } + _, skip := skippedHTMLTags[strings.ToLower(node.Data)] + return skip +} + +func textContent(node *html.Node) string { + if node == nil { + return "" + } + + var builder strings.Builder + var visit func(*html.Node) + visit = func(current *html.Node) { + if current == nil { + return + } + if current.Type == html.TextNode { + builder.WriteString(current.Data) + builder.WriteByte(' ') + } + for child := current.FirstChild; child != nil; child = child.NextSibling { + visit(child) + } + } + visit(node) + return builder.String() +} + +func normalizeWhitespace(value string) string { + return strings.Join(strings.Fields(value), " ") +} diff --git a/internal/tools/webfetch/tool.go b/internal/tools/webfetch/tool.go index b26734fa..c87b97f2 100644 --- a/internal/tools/webfetch/tool.go +++ b/internal/tools/webfetch/tool.go @@ -3,39 +3,75 @@ package webfetch import ( "context" "encoding/json" - "errors" + "fmt" "io" + "mime" "net/http" + "net/url" "strings" "time" + "github.com/dust/neo-code/internal/config" "github.com/dust/neo-code/internal/tools" ) +const ( + toolName = "webfetch" + htmlContentType = "text/html" + xhtmlContentType = "application/xhtml+xml" + reasonInvalidArguments = "invalid arguments" + reasonInvalidURL = "invalid URL" + reasonRequestFailed = "request failed" + reasonReadResponseFailed = "read response failed" + reasonUnsupportedType = "unsupported content type" + reasonEmptyContent = "content is empty after extraction" + errorMessageUnexpectedHTTP = "unexpected HTTP status %s" +) + +// Config controls how webfetch reads and filters remote content. +type Config struct { + Timeout time.Duration + MaxResponseBytes int64 + SupportedContentTypes []string +} + type Tool struct { - client *http.Client - bodyLimit int64 + client *http.Client + cfg Config + supportedText map[string]struct{} } type input struct { URL string `json:"url"` } -func New(timeout time.Duration) *Tool { +type responseData struct { + URL string + Status string + ContentType string + Title string + Content string + Truncated bool +} + +// New creates a webfetch tool with bounded responses and content-type filtering. +func New(cfg Config) *Tool { + normalized := normalizeConfig(cfg) return &Tool{ client: &http.Client{ - Timeout: timeout, + Timeout: normalized.Timeout, }, - bodyLimit: 256 * 1024, + cfg: normalized, + supportedText: newContentTypeSet(normalized.SupportedContentTypes), } } func (t *Tool) Name() string { - return "webfetch" + return toolName } func (t *Tool) Description() string { - return "Fetch text content from a web page using GET with bounded response size." + return "Fetch readable web content with content-type filtering and bounded response size." } func (t *Tool) Schema() map[string]any { @@ -52,40 +88,277 @@ func (t *Tool) Schema() map[string]any { } func (t *Tool) Execute(ctx context.Context, call tools.ToolCallInput) (tools.ToolResult, error) { + in, err := decodeInput(call.Arguments) + if err != nil { + return t.newErrorResult(responseData{}, reasonInvalidArguments, err.Error()), fmt.Errorf("webfetch: parse input: %w", err) + } + + targetURL, err := validateURL(in.URL) + if err != nil { + result := t.newErrorResult(responseData{URL: strings.TrimSpace(in.URL)}, reasonInvalidURL, err.Error()) + return result, fmt.Errorf("webfetch: validate url: %w", err) + } + + resp, err := t.fetch(ctx, targetURL) + if err != nil { + result := t.newErrorResult(responseData{URL: targetURL}, reasonRequestFailed, err.Error()) + return result, fmt.Errorf("webfetch: fetch %s: %w", targetURL, err) + } + defer resp.Body.Close() + + return t.handleResponse(targetURL, resp) +} + +func decodeInput(raw []byte) (input, error) { var in input - if err := json.Unmarshal(call.Arguments, &in); err != nil { - return tools.ToolResult{Name: t.Name()}, err + if err := json.Unmarshal(raw, &in); err != nil { + return input{}, err } - if !strings.HasPrefix(in.URL, "http://") && !strings.HasPrefix(in.URL, "https://") { - return tools.ToolResult{Name: t.Name()}, errors.New("webfetch: url must start with http:// or https://") + return in, nil +} + +func validateURL(raw string) (string, error) { + target := strings.TrimSpace(raw) + if target == "" { + return "", fmt.Errorf("%s: url is required", toolName) } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, in.URL, nil) + parsed, err := url.Parse(target) if err != nil { - return tools.ToolResult{Name: t.Name()}, err + return "", err + } + if parsed.Scheme != "http" && parsed.Scheme != "https" { + return "", fmt.Errorf("%s: url must start with http:// or https://", toolName) + } + if strings.TrimSpace(parsed.Host) == "" { + return "", fmt.Errorf("%s: url host is empty", toolName) } + return parsed.String(), nil +} + +func (t *Tool) fetch(ctx context.Context, targetURL string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) + if err != nil { + return nil, fmt.Errorf("build request: %w", err) + } + req.Header.Set("Accept", strings.Join(t.cfg.SupportedContentTypes, ", ")) + resp, err := t.client.Do(req) if err != nil { - return tools.ToolResult{Name: t.Name()}, err + return nil, err } - defer resp.Body.Close() + return resp, nil +} - body, err := io.ReadAll(io.LimitReader(resp.Body, t.bodyLimit)) +func (t *Tool) handleResponse(targetURL string, resp *http.Response) (tools.ToolResult, error) { + body, truncated, err := readBounded(resp.Body, t.cfg.MaxResponseBytes) if err != nil { - return tools.ToolResult{Name: t.Name()}, err + data := responseData{ + URL: targetURL, + Status: resp.Status, + Truncated: truncated, + } + return t.newErrorResult(data, reasonReadResponseFailed, err.Error()), fmt.Errorf("webfetch: read response body: %w", err) } - result := tools.ToolResult{ - Name: t.Name(), - Content: string(body), - IsError: resp.StatusCode >= 400, - Metadata: map[string]any{ - "url": in.URL, - "status": resp.Status, - }, + data := responseData{ + URL: targetURL, + Status: resp.Status, + ContentType: detectContentType(resp.Header.Get("Content-Type"), body), + Truncated: truncated, + } + + content, title, err := t.extractContent(data.ContentType, body) + if err != nil { + return t.newErrorResult(data, reasonUnsupportedType, ""), fmt.Errorf("webfetch: extract content: %w", err) + } + + data.Title = title + data.Content = content + + if resp.StatusCode >= http.StatusBadRequest { + reason := fmt.Sprintf(errorMessageUnexpectedHTTP, resp.Status) + return t.newErrorResult(data, reason, content), fmt.Errorf("webfetch: "+errorMessageUnexpectedHTTP, resp.Status) + } + if strings.TrimSpace(content) == "" { + return t.newErrorResult(data, reasonEmptyContent, ""), fmt.Errorf("webfetch: %s", reasonEmptyContent) } - if result.IsError { - return result, errors.New(resp.Status) + + return t.newSuccessResult(data), nil +} + +func readBounded(body io.Reader, limit int64) ([]byte, bool, error) { + limited, err := io.ReadAll(io.LimitReader(body, limit+1)) + if err != nil { + return nil, false, err + } + if int64(len(limited)) > limit { + return limited[:limit], true, nil + } + return limited, false, nil +} + +func detectContentType(header string, body []byte) string { + mediaType := normalizeContentType(header) + if mediaType != "" { + return mediaType + } + if len(body) == 0 { + return "" + } + return normalizeContentType(http.DetectContentType(body)) +} + +func (t *Tool) extractContent(contentType string, body []byte) (string, string, error) { + if !t.supports(contentType) { + return "", "", fmt.Errorf("%s: %s", toolName, reasonUnsupportedType) } - return result, nil + if isHTMLContentType(contentType) { + text, title, err := extractHTMLContent(body) + if err != nil { + return "", "", fmt.Errorf("parse html: %w", err) + } + return text, title, nil + } + return normalizePlainText(body), "", nil +} + +func (t *Tool) supports(contentType string) bool { + _, ok := t.supportedText[contentType] + return ok +} + +func (t *Tool) newSuccessResult(data responseData) tools.ToolResult { + return tools.ToolResult{ + Name: t.Name(), + Content: formatSuccess(data), + IsError: false, + Metadata: metadataFromResponse(data), + } +} + +func (t *Tool) newErrorResult(data responseData, reason string, details string) tools.ToolResult { + return tools.ToolResult{ + Name: t.Name(), + Content: formatError(data, reason, details), + IsError: true, + Metadata: metadataFromResponse(data), + } +} + +func formatSuccess(data responseData) string { + lines := formatCommonLines(data) + if data.Title != "" { + lines = append(lines, "title: "+data.Title) + } + return joinMessage(lines, data.Content) +} + +func formatError(data responseData, reason string, details string) string { + lines := append([]string{"webfetch error"}, formatCommonLines(data)...) + lines = append(lines, "reason: "+reason) + return joinMessage(lines, details) +} + +func formatCommonLines(data responseData) []string { + lines := make([]string, 0, 5) + if data.URL != "" { + lines = append(lines, "url: "+data.URL) + } + if data.Status != "" { + lines = append(lines, "status: "+data.Status) + } + if data.ContentType != "" { + lines = append(lines, "content_type: "+data.ContentType) + } + if data.Truncated { + lines = append(lines, "truncated: true") + } + return lines +} + +func joinMessage(lines []string, body string) string { + header := strings.Join(lines, "\n") + content := strings.TrimSpace(body) + if content == "" { + return header + } + if header == "" { + return content + } + return header + "\n\n" + content +} + +func metadataFromResponse(data responseData) map[string]any { + metadata := map[string]any{ + "url": data.URL, + "status": data.Status, + "content_type": data.ContentType, + "truncated": data.Truncated, + } + if data.Title != "" { + metadata["title"] = data.Title + } + return metadata +} + +func normalizeConfig(cfg Config) Config { + if cfg.Timeout <= 0 { + cfg.Timeout = time.Duration(config.DefaultToolTimeoutSec) * time.Second + } + if cfg.MaxResponseBytes <= 0 { + cfg.MaxResponseBytes = config.DefaultWebFetchMaxResponseBytes + } + if len(cfg.SupportedContentTypes) == 0 { + cfg.SupportedContentTypes = config.DefaultWebFetchSupportedContentTypes() + } + + normalized := make([]string, 0, len(cfg.SupportedContentTypes)) + seen := make(map[string]struct{}, len(cfg.SupportedContentTypes)) + for _, contentType := range cfg.SupportedContentTypes { + mediaType := normalizeContentType(contentType) + if mediaType == "" { + continue + } + if _, exists := seen[mediaType]; exists { + continue + } + seen[mediaType] = struct{}{} + normalized = append(normalized, mediaType) + } + cfg.SupportedContentTypes = normalized + return cfg +} + +func newContentTypeSet(contentTypes []string) map[string]struct{} { + set := make(map[string]struct{}, len(contentTypes)) + for _, contentType := range contentTypes { + set[contentType] = struct{}{} + } + return set +} + +func normalizeContentType(value string) string { + trimmed := strings.ToLower(strings.TrimSpace(value)) + if trimmed == "" { + return "" + } + + mediaType, _, err := mime.ParseMediaType(trimmed) + if err == nil { + return mediaType + } + if index := strings.Index(trimmed, ";"); index >= 0 { + return strings.TrimSpace(trimmed[:index]) + } + return trimmed +} + +func isHTMLContentType(contentType string) bool { + return contentType == htmlContentType || contentType == xhtmlContentType +} + +func normalizePlainText(body []byte) string { + text := strings.ReplaceAll(string(body), "\r\n", "\n") + return strings.TrimSpace(text) } diff --git a/internal/tools/webfetch/tool_test.go b/internal/tools/webfetch/tool_test.go index e43bb001..a655d592 100644 --- a/internal/tools/webfetch/tool_test.go +++ b/internal/tools/webfetch/tool_test.go @@ -9,7 +9,9 @@ import ( "testing" "time" + "github.com/dust/neo-code/internal/config" "github.com/dust/neo-code/internal/tools" + "golang.org/x/net/html" ) func TestToolExecute(t *testing.T) { @@ -17,10 +19,44 @@ func TestToolExecute(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { - case "/ok": - w.WriteHeader(http.StatusOK) + case "/html": + w.Header().Set("Content-Type", "text/html; charset=utf-8") + _, _ = w.Write([]byte(` + + + Example Domain + + + + +
+

Example Domain

+

This domain is for use in illustrative examples.

+ +
+ + + `)) + case "/html-empty": + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte(`Blank`)) + case "/plain": + w.Header().Set("Content-Type", "text/plain") _, _ = w.Write([]byte("hello webfetch")) + case "/json": + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"ok":true}`)) + case "/image": + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write([]byte("\x89PNG\x0d\x0a\x1a\x0a")) + case "/octet": + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write([]byte("binary payload")) + case "/large": + w.Header().Set("Content-Type", "text/plain") + _, _ = w.Write([]byte("0123456789abcdef")) case "/fail": + w.Header().Set("Content-Type", "text/plain") w.WriteHeader(http.StatusBadGateway) _, _ = w.Write([]byte("upstream failed")) default: @@ -29,54 +65,186 @@ func TestToolExecute(t *testing.T) { })) defer server.Close() - tool := New(2 * time.Second) - if tool.Name() != "webfetch" { - t.Fatalf("unexpected tool name %q", tool.Name()) + closedServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + closedURL := closedServer.URL + closedServer.Close() + + defaultConfig := Config{ + Timeout: 2 * time.Second, + MaxResponseBytes: config.DefaultWebFetchMaxResponseBytes, + SupportedContentTypes: config.DefaultWebFetchSupportedContentTypes(), + } + if New(defaultConfig).Name() != "webfetch" { + t.Fatalf("unexpected tool name %q", New(defaultConfig).Name()) } - if tool.Description() == "" { + if New(defaultConfig).Description() == "" { t.Fatalf("expected non-empty description") } - if tool.Schema()["type"] != "object" { + if New(defaultConfig).Schema()["type"] != "object" { t.Fatalf("expected schema object") } tests := []struct { - name string - args any - expectErr string - expectContent string - expectStatus string - expectIsError bool + name string + args any + toolConfig Config + expectErr string + expectContent []string + rejectContent []string + expectStatus string + expectType string + expectIsError bool + expectTruncated bool }{ { - name: "fetch success", - args: map[string]string{"url": server.URL + "/ok"}, - expectContent: "hello webfetch", + name: "extracts html body text", + args: map[string]string{"url": server.URL + "/html"}, + toolConfig: defaultConfig, + expectContent: []string{ + "url: " + server.URL + "/html", + "status: 200 OK", + "content_type: text/html", + "title: Example Domain", + "Example Domain This domain is for use in illustrative examples.", + }, + rejectContent: []string{"ignored()", "display:none", "skip me"}, + expectStatus: "200 OK", + expectType: "text/html", + }, + { + name: "rejects empty html content", + args: map[string]string{"url": server.URL + "/html-empty"}, + toolConfig: defaultConfig, + expectErr: reasonEmptyContent, + expectContent: []string{ + "webfetch error", + "content_type: text/html", + "reason: " + reasonEmptyContent, + }, expectStatus: "200 OK", + expectType: "text/html", + expectIsError: true, + }, + { + name: "returns plain text response", + args: map[string]string{"url": server.URL + "/plain"}, + toolConfig: defaultConfig, + expectContent: []string{ + "content_type: text/plain", + "hello webfetch", + }, + expectStatus: "200 OK", + expectType: "text/plain", }, { - name: "fetch http error", - args: map[string]string{"url": server.URL + "/fail"}, - expectErr: "502 Bad Gateway", - expectContent: "upstream failed", + name: "returns json as text", + args: map[string]string{"url": server.URL + "/json"}, + toolConfig: defaultConfig, + expectContent: []string{ + "content_type: application/json", + `{"ok":true}`, + }, + expectStatus: "200 OK", + expectType: "application/json", + }, + { + name: "rejects unsupported image content type", + args: map[string]string{"url": server.URL + "/image"}, + toolConfig: defaultConfig, + expectErr: reasonUnsupportedType, + expectContent: []string{ + "webfetch error", + "content_type: image/png", + "reason: " + reasonUnsupportedType, + }, + expectStatus: "200 OK", + expectType: "image/png", + expectIsError: true, + }, + { + name: "rejects unsupported octet stream", + args: map[string]string{"url": server.URL + "/octet"}, + toolConfig: defaultConfig, + expectErr: reasonUnsupportedType, + expectContent: []string{ + "reason: " + reasonUnsupportedType, + }, + expectStatus: "200 OK", + expectType: "application/octet-stream", + expectIsError: true, + }, + { + name: "marks truncated text responses", + args: map[string]string{"url": server.URL + "/large"}, + toolConfig: Config{ + Timeout: 2 * time.Second, + MaxResponseBytes: 8, + SupportedContentTypes: config.DefaultWebFetchSupportedContentTypes(), + }, + expectContent: []string{ + "truncated: true", + "01234567", + }, + expectStatus: "200 OK", + expectType: "text/plain", + expectTruncated: true, + }, + { + name: "formats http errors consistently", + args: map[string]string{"url": server.URL + "/fail"}, + toolConfig: defaultConfig, + expectErr: "unexpected HTTP status 502 Bad Gateway", + expectContent: []string{ + "webfetch error", + "status: 502 Bad Gateway", + "content_type: text/plain", + "reason: unexpected HTTP status 502 Bad Gateway", + "upstream failed", + }, expectStatus: "502 Bad Gateway", + expectType: "text/plain", + expectIsError: true, + }, + { + name: "rejects invalid scheme", + args: map[string]string{"url": "ftp://example.com"}, + toolConfig: defaultConfig, + expectErr: "url must start with http:// or https://", + expectContent: []string{ + "webfetch error", + "reason: " + reasonInvalidURL, + }, expectIsError: true, }, { - name: "rejects invalid scheme", - args: map[string]string{"url": "ftp://example.com"}, - expectErr: "url must start with http:// or https://", + name: "rejects invalid json", + args: "{", + toolConfig: defaultConfig, + expectErr: "JSON input", + expectContent: []string{ + "webfetch error", + "reason: " + reasonInvalidArguments, + }, + expectIsError: true, }, { - name: "rejects invalid json", - args: "{", - expectErr: "JSON input", + name: "returns request failure when server is unavailable", + args: map[string]string{"url": closedURL}, + toolConfig: defaultConfig, + expectErr: "webfetch: fetch", + expectContent: []string{ + "webfetch error", + "reason: " + reasonRequestFailed, + }, + expectIsError: true, }, } for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { + tool := New(tt.toolConfig) + var raw []byte switch value := tt.args.(type) { case string: @@ -102,15 +270,88 @@ func TestToolExecute(t *testing.T) { t.Fatalf("unexpected error: %v", execErr) } - if tt.expectContent != "" && !strings.Contains(result.Content, tt.expectContent) { - t.Fatalf("expected content containing %q, got %q", tt.expectContent, result.Content) + for _, want := range tt.expectContent { + if !strings.Contains(result.Content, want) { + t.Fatalf("expected content containing %q, got %q", want, result.Content) + } + } + for _, reject := range tt.rejectContent { + if strings.Contains(result.Content, reject) { + t.Fatalf("expected content not to contain %q, got %q", reject, result.Content) + } } if tt.expectStatus != "" && result.Metadata["status"] != tt.expectStatus { t.Fatalf("expected status %q, got %+v", tt.expectStatus, result.Metadata) } + if tt.expectType != "" && result.Metadata["content_type"] != tt.expectType { + t.Fatalf("expected content_type %q, got %+v", tt.expectType, result.Metadata) + } + if truncated, ok := result.Metadata["truncated"].(bool); !ok || truncated != tt.expectTruncated { + t.Fatalf("expected truncated=%v, got %+v", tt.expectTruncated, result.Metadata) + } if result.IsError != tt.expectIsError { t.Fatalf("expected IsError=%v, got %v", tt.expectIsError, result.IsError) } }) } } + +func TestToolDefaultsAndContentTypeFallback(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("fallback text body")) + })) + defer server.Close() + + tool := New(Config{}) + args, err := json.Marshal(map[string]string{"url": server.URL}) + if err != nil { + t.Fatalf("marshal args: %v", err) + } + + result, execErr := tool.Execute(context.Background(), tools.ToolCallInput{ + Name: tool.Name(), + Arguments: args, + }) + if execErr != nil { + t.Fatalf("unexpected error: %v", execErr) + } + if result.Metadata["content_type"] != "text/plain" { + t.Fatalf("expected detected text/plain content type, got %+v", result.Metadata) + } + if !strings.Contains(result.Content, "fallback text body") { + t.Fatalf("expected fallback body content, got %q", result.Content) + } +} + +func TestHTMLHelpers(t *testing.T) { + t.Parallel() + + text, title, err := extractHTMLContent([]byte(`
hello world
`)) + if err != nil { + t.Fatalf("extractHTMLContent() error = %v", err) + } + if text != "hello world" { + t.Fatalf("expected extracted text %q, got %q", "hello world", text) + } + if title != "" { + t.Fatalf("expected empty title, got %q", title) + } + + if findElement(nil, "body") != nil { + t.Fatalf("expected nil findElement result for nil root") + } + if shouldSkipNode(nil) { + t.Fatalf("expected shouldSkipNode(nil) to be false") + } + if normalizeWhitespace(" hello \n world ") != "hello world" { + t.Fatalf("expected normalizeWhitespace to collapse whitespace") + } + + titleNode := &html.Node{Type: html.ElementNode, Data: "title"} + titleNode.AppendChild(&html.Node{Type: html.TextNode, Data: "Example"}) + if textContent(titleNode) != "Example " { + t.Fatalf("expected title text content, got %q", textContent(titleNode)) + } +}