diff --git a/go.mod b/go.mod
index 0db3a58d..563ad14d 100644
--- a/go.mod
+++ b/go.mod
@@ -6,6 +6,8 @@ require (
github.com/charmbracelet/bubbles v1.0.0
github.com/charmbracelet/bubbletea v1.3.10
github.com/charmbracelet/lipgloss v1.1.0
+ github.com/joho/godotenv v1.5.1
+ golang.org/x/net v0.50.0
gopkg.in/yaml.v3 v3.0.1
)
@@ -20,7 +22,6 @@ require (
github.com/clipperhouse/stringish v0.1.1 // indirect
github.com/clipperhouse/uax29/v2 v2.5.0 // indirect
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
- github.com/joho/godotenv v1.5.1 // indirect
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
@@ -31,6 +32,6 @@ require (
github.com/rivo/uniseg v0.4.7 // indirect
github.com/sahilm/fuzzy v0.1.1 // indirect
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
- golang.org/x/sys v0.38.0 // indirect
- golang.org/x/text v0.3.8 // indirect
+ golang.org/x/sys v0.41.0 // indirect
+ golang.org/x/text v0.34.0 // indirect
)
diff --git a/go.sum b/go.sum
index d5645656..8adf72f8 100644
--- a/go.sum
+++ b/go.sum
@@ -56,12 +56,18 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
+golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
diff --git a/internal/app/bootstrap.go b/internal/app/bootstrap.go
index 2639538f..d7999b7b 100644
--- a/internal/app/bootstrap.go
+++ b/internal/app/bootstrap.go
@@ -23,14 +23,7 @@ func NewProgram(ctx context.Context) (*tea.Program, error) {
return nil, err
}
- toolRegistry := tools.NewRegistry()
- toolRegistry.Register(filesystem.New(cfg.Workdir))
- toolRegistry.Register(filesystem.NewWrite(cfg.Workdir))
- toolRegistry.Register(filesystem.NewGrep(cfg.Workdir))
- toolRegistry.Register(filesystem.NewGlob(cfg.Workdir))
- toolRegistry.Register(filesystem.NewEdit(cfg.Workdir))
- toolRegistry.Register(bash.New(cfg.Workdir, cfg.Shell, time.Duration(cfg.ToolTimeoutSec)*time.Second))
- toolRegistry.Register(webfetch.New(time.Duration(cfg.ToolTimeoutSec) * time.Second))
+ toolRegistry := buildToolRegistry(cfg)
sessionStore := agentruntime.NewSessionStore(loader.BaseDir())
runtimeSvc := agentruntime.New(manager, toolRegistry, sessionStore)
@@ -45,3 +38,19 @@ func NewProgram(ctx context.Context) (*tea.Program, error) {
tea.WithMouseCellMotion(),
), nil
}
+
+func buildToolRegistry(cfg config.Config) *tools.Registry {
+ toolRegistry := tools.NewRegistry()
+ toolRegistry.Register(filesystem.New(cfg.Workdir))
+ toolRegistry.Register(filesystem.NewWrite(cfg.Workdir))
+ toolRegistry.Register(filesystem.NewGrep(cfg.Workdir))
+ toolRegistry.Register(filesystem.NewGlob(cfg.Workdir))
+ toolRegistry.Register(filesystem.NewEdit(cfg.Workdir))
+ toolRegistry.Register(bash.New(cfg.Workdir, cfg.Shell, time.Duration(cfg.ToolTimeoutSec)*time.Second))
+ toolRegistry.Register(webfetch.New(webfetch.Config{
+ Timeout: time.Duration(cfg.ToolTimeoutSec) * time.Second,
+ MaxResponseBytes: cfg.Tools.WebFetch.MaxResponseBytes,
+ SupportedContentTypes: cfg.Tools.WebFetch.SupportedContentTypes,
+ }))
+ return toolRegistry
+}
diff --git a/internal/app/bootstrap_test.go b/internal/app/bootstrap_test.go
index a25e7a4d..5974d9ac 100644
--- a/internal/app/bootstrap_test.go
+++ b/internal/app/bootstrap_test.go
@@ -2,9 +2,15 @@ package app
import (
"context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
"os"
"path/filepath"
"testing"
+
+ "github.com/dust/neo-code/internal/config"
+ "github.com/dust/neo-code/internal/tools"
)
func TestNewProgram(t *testing.T) {
@@ -25,3 +31,42 @@ func TestNewProgram(t *testing.T) {
t.Fatalf("expected config file to be created at %q: %v", configPath, err)
}
}
+
+func TestBuildToolRegistryUsesWebFetchConfig(t *testing.T) {
+ t.Parallel()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "text/plain")
+ _, _ = w.Write([]byte("1234567890"))
+ }))
+ defer server.Close()
+
+ cfg := config.Default().Clone()
+ cfg.Workdir = t.TempDir()
+ cfg.Tools.WebFetch.MaxResponseBytes = 4
+
+ registry := buildToolRegistry(cfg)
+ tool, err := registry.Get("webfetch")
+ if err != nil {
+ t.Fatalf("registry.Get(webfetch) error = %v", err)
+ }
+
+ args, err := json.Marshal(map[string]string{"url": server.URL})
+ if err != nil {
+ t.Fatalf("marshal args: %v", err)
+ }
+
+ result, execErr := tool.Execute(context.Background(), tools.ToolCallInput{
+ Name: "webfetch",
+ Arguments: args,
+ })
+ if execErr != nil {
+ t.Fatalf("webfetch execute error = %v", execErr)
+ }
+ if truncated, ok := result.Metadata["truncated"].(bool); !ok || !truncated {
+ t.Fatalf("expected truncated metadata, got %+v", result.Metadata)
+ }
+ if result.Content == "" {
+ t.Fatalf("expected formatted webfetch content")
+ }
+}
diff --git a/internal/config/config_test.go b/internal/config/config_test.go
index 108317d1..578d824f 100644
--- a/internal/config/config_test.go
+++ b/internal/config/config_test.go
@@ -24,6 +24,12 @@ selected_provider: openai
current_model: gpt-5.4
workdir: .
shell: powershell
+tools:
+ webfetch:
+ max_response_bytes: 4096
+ supported_content_types:
+ - text/html
+ - text/plain
providers:
- name: openai
type: openai
@@ -43,6 +49,12 @@ providers:
if provider.BaseURL != "https://example.com/v1" {
t.Fatalf("expected custom base url, got %q", provider.BaseURL)
}
+ if cfg.Tools.WebFetch.MaxResponseBytes != 4096 {
+ t.Fatalf("expected custom max_response_bytes 4096, got %d", cfg.Tools.WebFetch.MaxResponseBytes)
+ }
+ if len(cfg.Tools.WebFetch.SupportedContentTypes) != 2 {
+ t.Fatalf("expected 2 supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes)
+ }
},
},
{
@@ -73,6 +85,12 @@ providers:
if provider.Model != "gpt-4o" {
t.Fatalf("expected provider model gpt-4o, got %q", provider.Model)
}
+ if cfg.Tools.WebFetch.MaxResponseBytes != DefaultWebFetchMaxResponseBytes {
+ t.Fatalf("expected default max_response_bytes %d, got %d", DefaultWebFetchMaxResponseBytes, cfg.Tools.WebFetch.MaxResponseBytes)
+ }
+ if len(cfg.Tools.WebFetch.SupportedContentTypes) != len(DefaultWebFetchSupportedContentTypes()) {
+ t.Fatalf("expected default supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes)
+ }
},
},
}
@@ -362,6 +380,12 @@ func TestConfigApplyDefaultsFillsMissingFields(t *testing.T) {
if !filepath.IsAbs(cfg.Workdir) {
t.Fatalf("expected absolute workdir, got %q", cfg.Workdir)
}
+ if cfg.Tools.WebFetch.MaxResponseBytes != DefaultWebFetchMaxResponseBytes {
+ t.Fatalf("expected default webfetch max_response_bytes %d, got %d", DefaultWebFetchMaxResponseBytes, cfg.Tools.WebFetch.MaxResponseBytes)
+ }
+ if len(cfg.Tools.WebFetch.SupportedContentTypes) != len(DefaultWebFetchSupportedContentTypes()) {
+ t.Fatalf("expected default supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes)
+ }
}
func TestConfigValidateFailures(t *testing.T) {
@@ -423,6 +447,24 @@ func TestConfigValidateFailures(t *testing.T) {
}(),
expectErr: "model is empty",
},
+ {
+ name: "invalid webfetch max response bytes",
+ config: func() *Config {
+ cfg := validConfig.Clone()
+ cfg.Tools.WebFetch.MaxResponseBytes = 0
+ return &cfg
+ }(),
+ expectErr: "max_response_bytes must be greater than 0",
+ },
+ {
+ name: "invalid webfetch supported content types",
+ config: func() *Config {
+ cfg := validConfig.Clone()
+ cfg.Tools.WebFetch.SupportedContentTypes = []string{""}
+ return &cfg
+ }(),
+ expectErr: "supported_content_types[0] is empty",
+ },
}
for _, tt := range tests {
@@ -542,6 +584,8 @@ func TestLoaderLoadAndSaveRoundTrip(t *testing.T) {
cfg.Providers[i].Model = "gpt-5.4"
}
}
+ cfg.Tools.WebFetch.MaxResponseBytes = 1024
+ cfg.Tools.WebFetch.SupportedContentTypes = []string{"text/html", "application/json"}
if err := loader.Save(context.Background(), cfg); err != nil {
t.Fatalf("Save() error = %v", err)
}
@@ -553,6 +597,12 @@ func TestLoaderLoadAndSaveRoundTrip(t *testing.T) {
if reloaded.CurrentModel != "gpt-5.4" {
t.Fatalf("expected current model %q, got %q", "gpt-5.4", reloaded.CurrentModel)
}
+ if reloaded.Tools.WebFetch.MaxResponseBytes != 1024 {
+ t.Fatalf("expected max_response_bytes %d, got %d", 1024, reloaded.Tools.WebFetch.MaxResponseBytes)
+ }
+ if len(reloaded.Tools.WebFetch.SupportedContentTypes) != 2 {
+ t.Fatalf("expected persisted supported content types, got %+v", reloaded.Tools.WebFetch.SupportedContentTypes)
+ }
}
func TestNormalizeWorkdirAndClone(t *testing.T) {
@@ -621,9 +671,13 @@ func TestNormalizeWorkdirAndClone(t *testing.T) {
cfg := Default()
cloned := cfg.Clone()
cloned.CurrentModel = "modified"
+ cloned.Tools.WebFetch.SupportedContentTypes[0] = "application/json"
if cfg.CurrentModel == cloned.CurrentModel {
t.Fatalf("expected clone to be independent from source")
}
+ if cfg.Tools.WebFetch.SupportedContentTypes[0] == cloned.Tools.WebFetch.SupportedContentTypes[0] {
+ t.Fatalf("expected webfetch supported content types to be cloned")
+ }
}
func TestManagerHelperMethodsAndReloads(t *testing.T) {
diff --git a/internal/config/model.go b/internal/config/model.go
index 63ecb697..d05e398f 100644
--- a/internal/config/model.go
+++ b/internal/config/model.go
@@ -3,6 +3,7 @@ package config
import (
"errors"
"fmt"
+ "mime"
"os"
"path/filepath"
goruntime "runtime"
@@ -30,8 +31,19 @@ const (
DefaultWorkdir = "."
DefaultMaxLoops = 8
DefaultToolTimeoutSec = 20
+ // DefaultWebFetchMaxResponseBytes bounds the amount of web content returned to the model.
+ DefaultWebFetchMaxResponseBytes int64 = 256 * 1024
)
+var defaultWebFetchSupportedContentTypes = []string{
+ "text/html",
+ "application/xhtml+xml",
+ "text/plain",
+ "application/json",
+ "application/xml",
+ "text/xml",
+}
+
type Config struct {
Providers []ProviderConfig `yaml:"providers"`
SelectedProvider string `yaml:"selected_provider"`
@@ -40,6 +52,7 @@ type Config struct {
Shell string `yaml:"shell"`
MaxLoops int `yaml:"max_loops,omitempty"`
ToolTimeoutSec int `yaml:"tool_timeout_sec,omitempty"`
+ Tools ToolsConfig `yaml:"tools,omitempty"`
}
type ProviderConfig struct {
@@ -60,6 +73,22 @@ type ModelOption struct {
Description string
}
+// ToolsConfig stores tool-specific configuration values.
+type ToolsConfig struct {
+ WebFetch WebFetchConfig `yaml:"webfetch,omitempty"`
+}
+
+// WebFetchConfig controls response filtering and limits for the webfetch tool.
+type WebFetchConfig struct {
+ MaxResponseBytes int64 `yaml:"max_response_bytes,omitempty"`
+ SupportedContentTypes []string `yaml:"supported_content_types,omitempty"`
+}
+
+// DefaultWebFetchSupportedContentTypes returns the default media types accepted by webfetch.
+func DefaultWebFetchSupportedContentTypes() []string {
+ return append([]string(nil), defaultWebFetchSupportedContentTypes...)
+}
+
func Default() *Config {
return &Config{
Providers: []ProviderConfig{
@@ -91,6 +120,9 @@ func Default() *Config {
Shell: defaultShell(),
MaxLoops: DefaultMaxLoops,
ToolTimeoutSec: DefaultToolTimeoutSec,
+ Tools: ToolsConfig{
+ WebFetch: defaultWebFetchConfig(),
+ },
}
}
@@ -112,6 +144,7 @@ func (c *Config) Clone() Config {
clone := *c
clone.Providers = append([]ProviderConfig(nil), c.Providers...)
+ clone.Tools = c.Tools.Clone()
return clone
}
@@ -148,6 +181,7 @@ func (c *Config) ApplyDefaults() {
if c.ToolTimeoutSec <= 0 {
c.ToolTimeoutSec = def.ToolTimeoutSec
}
+ c.Tools.ApplyDefaults(def.Tools)
c.Workdir = normalizeWorkdir(c.Workdir)
}
@@ -192,6 +226,9 @@ func (c *Config) Validate() error {
if strings.TrimSpace(selected.Model) == "" {
return fmt.Errorf("config: selected provider %q has empty model", selected.Name)
}
+ if err := c.Tools.Validate(); err != nil {
+ return fmt.Errorf("config: tools: %w", err)
+ }
return nil
}
@@ -344,3 +381,103 @@ func defaultShell() string {
}
return "bash"
}
+
+func defaultWebFetchConfig() WebFetchConfig {
+ return WebFetchConfig{
+ MaxResponseBytes: DefaultWebFetchMaxResponseBytes,
+ SupportedContentTypes: DefaultWebFetchSupportedContentTypes(),
+ }
+}
+
+func (c ToolsConfig) Clone() ToolsConfig {
+ return ToolsConfig{
+ WebFetch: c.WebFetch.Clone(),
+ }
+}
+
+func (c *ToolsConfig) ApplyDefaults(def ToolsConfig) {
+ if c == nil {
+ return
+ }
+
+ c.WebFetch.ApplyDefaults(def.WebFetch)
+}
+
+func (c ToolsConfig) Validate() error {
+ if err := c.WebFetch.Validate(); err != nil {
+ return fmt.Errorf("webfetch: %w", err)
+ }
+ return nil
+}
+
+func (c WebFetchConfig) Clone() WebFetchConfig {
+ clone := c
+ clone.SupportedContentTypes = append([]string(nil), c.SupportedContentTypes...)
+ return clone
+}
+
+func (c *WebFetchConfig) ApplyDefaults(def WebFetchConfig) {
+ if c == nil {
+ return
+ }
+
+ if c.MaxResponseBytes <= 0 {
+ c.MaxResponseBytes = def.MaxResponseBytes
+ }
+ c.SupportedContentTypes = normalizeContentTypes(c.SupportedContentTypes, def.SupportedContentTypes)
+}
+
+func (c WebFetchConfig) Validate() error {
+ if c.MaxResponseBytes <= 0 {
+ return errors.New("max_response_bytes must be greater than 0")
+ }
+ if len(c.SupportedContentTypes) == 0 {
+ return errors.New("supported_content_types is empty")
+ }
+
+ for i, contentType := range c.SupportedContentTypes {
+ if normalizeContentType(contentType) == "" {
+ return fmt.Errorf("supported_content_types[%d] is empty", i)
+ }
+ }
+ return nil
+}
+
+func normalizeContentTypes(values []string, defaults []string) []string {
+ source := values
+ if len(source) == 0 {
+ source = defaults
+ }
+
+ normalized := make([]string, 0, len(source))
+ seen := make(map[string]struct{}, len(source))
+ for _, value := range source {
+ contentType := normalizeContentType(value)
+ if contentType == "" {
+ continue
+ }
+ if _, exists := seen[contentType]; exists {
+ continue
+ }
+ seen[contentType] = struct{}{}
+ normalized = append(normalized, contentType)
+ }
+ return normalized
+}
+
+func normalizeContentType(value string) string {
+ trimmed := strings.ToLower(strings.TrimSpace(value))
+ if trimmed == "" {
+ return ""
+ }
+
+ mediaType, _, err := mime.ParseMediaType(trimmed)
+ if err == nil {
+ return mediaType
+ }
+
+ if index := strings.Index(trimmed, ";"); index >= 0 {
+ return strings.TrimSpace(trimmed[:index])
+ }
+ return trimmed
+}
diff --git a/internal/tools/webfetch/html.go b/internal/tools/webfetch/html.go
new file mode 100644
index 00000000..c7ac6f25
--- /dev/null
+++ b/internal/tools/webfetch/html.go
@@ -0,0 +1,112 @@
+package webfetch
+
+import (
+ "bytes"
+ "fmt"
+ "strings"
+
+ "golang.org/x/net/html"
+)
+
+var skippedHTMLTags = map[string]struct{}{
+ "canvas": {},
+ "head": {},
+ "noscript": {},
+ "script": {},
+ "style": {},
+ "svg": {},
+ "template": {},
+}
+
+func extractHTMLContent(body []byte) (string, string, error) {
+ doc, err := html.Parse(bytes.NewReader(body))
+ if err != nil {
+ return "", "", fmt.Errorf("parse document: %w", err)
+ }
+
+ root := findElement(doc, "body")
+ if root == nil {
+ root = doc
+ }
+
+ text := normalizeWhitespace(collectVisibleText(root))
+ title := normalizeWhitespace(textContent(findElement(doc, "title")))
+ return text, title, nil
+}
+
+func findElement(node *html.Node, tag string) *html.Node {
+ if node == nil {
+ return nil
+ }
+ if node.Type == html.ElementNode && strings.EqualFold(node.Data, tag) {
+ return node
+ }
+ for child := node.FirstChild; child != nil; child = child.NextSibling {
+ if found := findElement(child, tag); found != nil {
+ return found
+ }
+ }
+ return nil
+}
+
+func collectVisibleText(node *html.Node) string {
+ var builder strings.Builder
+ appendVisibleText(&builder, node)
+ return builder.String()
+}
+
+func appendVisibleText(builder *strings.Builder, node *html.Node) {
+ if node == nil {
+ return
+ }
+ if shouldSkipNode(node) {
+ return
+ }
+
+ if node.Type == html.TextNode {
+ text := strings.TrimSpace(node.Data)
+ if text != "" {
+ builder.WriteString(text)
+ builder.WriteByte('\n')
+ }
+ }
+
+ for child := node.FirstChild; child != nil; child = child.NextSibling {
+ appendVisibleText(builder, child)
+ }
+}
+
+func shouldSkipNode(node *html.Node) bool {
+ if node == nil || node.Type != html.ElementNode {
+ return false
+ }
+ _, skip := skippedHTMLTags[strings.ToLower(node.Data)]
+ return skip
+}
+
+func textContent(node *html.Node) string {
+ if node == nil {
+ return ""
+ }
+
+ var builder strings.Builder
+ var visit func(*html.Node)
+ visit = func(current *html.Node) {
+ if current == nil {
+ return
+ }
+ if current.Type == html.TextNode {
+ builder.WriteString(current.Data)
+ builder.WriteByte(' ')
+ }
+ for child := current.FirstChild; child != nil; child = child.NextSibling {
+ visit(child)
+ }
+ }
+ visit(node)
+ return builder.String()
+}
+
+func normalizeWhitespace(value string) string {
+ return strings.Join(strings.Fields(value), " ")
+}
diff --git a/internal/tools/webfetch/tool.go b/internal/tools/webfetch/tool.go
index b26734fa..c87b97f2 100644
--- a/internal/tools/webfetch/tool.go
+++ b/internal/tools/webfetch/tool.go
@@ -3,39 +3,75 @@ package webfetch
import (
"context"
"encoding/json"
- "errors"
+ "fmt"
"io"
+ "mime"
"net/http"
+ "net/url"
"strings"
"time"
+ "github.com/dust/neo-code/internal/config"
"github.com/dust/neo-code/internal/tools"
)
+const (
+ toolName = "webfetch"
+ htmlContentType = "text/html"
+ xhtmlContentType = "application/xhtml+xml"
+ reasonInvalidArguments = "invalid arguments"
+ reasonInvalidURL = "invalid URL"
+ reasonRequestFailed = "request failed"
+ reasonReadResponseFailed = "read response failed"
+ reasonUnsupportedType = "unsupported content type"
+ reasonEmptyContent = "content is empty after extraction"
+ errorMessageUnexpectedHTTP = "unexpected HTTP status %s"
+)
+
+// Config controls how webfetch reads and filters remote content.
+type Config struct {
+ Timeout time.Duration
+ MaxResponseBytes int64
+ SupportedContentTypes []string
+}
+
type Tool struct {
- client *http.Client
- bodyLimit int64
+ client *http.Client
+ cfg Config
+ supportedText map[string]struct{}
}
type input struct {
URL string `json:"url"`
}
-func New(timeout time.Duration) *Tool {
+type responseData struct {
+ URL string
+ Status string
+ ContentType string
+ Title string
+ Content string
+ Truncated bool
+}
+
+// New creates a webfetch tool with bounded responses and content-type filtering.
+func New(cfg Config) *Tool {
+ normalized := normalizeConfig(cfg)
return &Tool{
client: &http.Client{
- Timeout: timeout,
+ Timeout: normalized.Timeout,
},
- bodyLimit: 256 * 1024,
+ cfg: normalized,
+ supportedText: newContentTypeSet(normalized.SupportedContentTypes),
}
}
func (t *Tool) Name() string {
- return "webfetch"
+ return toolName
}
func (t *Tool) Description() string {
- return "Fetch text content from a web page using GET with bounded response size."
+ return "Fetch readable web content with content-type filtering and bounded response size."
}
func (t *Tool) Schema() map[string]any {
@@ -52,40 +88,277 @@ func (t *Tool) Schema() map[string]any {
}
func (t *Tool) Execute(ctx context.Context, call tools.ToolCallInput) (tools.ToolResult, error) {
+ in, err := decodeInput(call.Arguments)
+ if err != nil {
+ return t.newErrorResult(responseData{}, reasonInvalidArguments, err.Error()), fmt.Errorf("webfetch: parse input: %w", err)
+ }
+
+ targetURL, err := validateURL(in.URL)
+ if err != nil {
+ result := t.newErrorResult(responseData{URL: strings.TrimSpace(in.URL)}, reasonInvalidURL, err.Error())
+ return result, fmt.Errorf("webfetch: validate url: %w", err)
+ }
+
+ resp, err := t.fetch(ctx, targetURL)
+ if err != nil {
+ result := t.newErrorResult(responseData{URL: targetURL}, reasonRequestFailed, err.Error())
+ return result, fmt.Errorf("webfetch: fetch %s: %w", targetURL, err)
+ }
+ defer resp.Body.Close()
+
+ return t.handleResponse(targetURL, resp)
+}
+
+func decodeInput(raw []byte) (input, error) {
var in input
- if err := json.Unmarshal(call.Arguments, &in); err != nil {
- return tools.ToolResult{Name: t.Name()}, err
+ if err := json.Unmarshal(raw, &in); err != nil {
+ return input{}, err
}
- if !strings.HasPrefix(in.URL, "http://") && !strings.HasPrefix(in.URL, "https://") {
- return tools.ToolResult{Name: t.Name()}, errors.New("webfetch: url must start with http:// or https://")
+ return in, nil
+}
+
+func validateURL(raw string) (string, error) {
+ target := strings.TrimSpace(raw)
+ if target == "" {
+ return "", fmt.Errorf("%s: url is required", toolName)
}
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, in.URL, nil)
+ parsed, err := url.Parse(target)
if err != nil {
- return tools.ToolResult{Name: t.Name()}, err
+ return "", err
+ }
+ if parsed.Scheme != "http" && parsed.Scheme != "https" {
+ return "", fmt.Errorf("%s: url must start with http:// or https://", toolName)
+ }
+ if strings.TrimSpace(parsed.Host) == "" {
+ return "", fmt.Errorf("%s: url host is empty", toolName)
}
+ return parsed.String(), nil
+}
+
+func (t *Tool) fetch(ctx context.Context, targetURL string) (*http.Response, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil)
+ if err != nil {
+ return nil, fmt.Errorf("build request: %w", err)
+ }
+ req.Header.Set("Accept", strings.Join(t.cfg.SupportedContentTypes, ", "))
+
resp, err := t.client.Do(req)
if err != nil {
- return tools.ToolResult{Name: t.Name()}, err
+ return nil, err
}
- defer resp.Body.Close()
+ return resp, nil
+}
- body, err := io.ReadAll(io.LimitReader(resp.Body, t.bodyLimit))
+func (t *Tool) handleResponse(targetURL string, resp *http.Response) (tools.ToolResult, error) {
+ body, truncated, err := readBounded(resp.Body, t.cfg.MaxResponseBytes)
if err != nil {
- return tools.ToolResult{Name: t.Name()}, err
+ data := responseData{
+ URL: targetURL,
+ Status: resp.Status,
+ Truncated: truncated,
+ }
+ return t.newErrorResult(data, reasonReadResponseFailed, err.Error()), fmt.Errorf("webfetch: read response body: %w", err)
}
- result := tools.ToolResult{
- Name: t.Name(),
- Content: string(body),
- IsError: resp.StatusCode >= 400,
- Metadata: map[string]any{
- "url": in.URL,
- "status": resp.Status,
- },
+ data := responseData{
+ URL: targetURL,
+ Status: resp.Status,
+ ContentType: detectContentType(resp.Header.Get("Content-Type"), body),
+ Truncated: truncated,
+ }
+
+ content, title, err := t.extractContent(data.ContentType, body)
+ if err != nil {
+ return t.newErrorResult(data, reasonUnsupportedType, ""), fmt.Errorf("webfetch: extract content: %w", err)
+ }
+
+ data.Title = title
+ data.Content = content
+
+ if resp.StatusCode >= http.StatusBadRequest {
+ reason := fmt.Sprintf(errorMessageUnexpectedHTTP, resp.Status)
+ return t.newErrorResult(data, reason, content), fmt.Errorf("webfetch: "+errorMessageUnexpectedHTTP, resp.Status)
+ }
+ if strings.TrimSpace(content) == "" {
+ return t.newErrorResult(data, reasonEmptyContent, ""), fmt.Errorf("webfetch: %s", reasonEmptyContent)
}
- if result.IsError {
- return result, errors.New(resp.Status)
+
+ return t.newSuccessResult(data), nil
+}
+
+func readBounded(body io.Reader, limit int64) ([]byte, bool, error) {
+ limited, err := io.ReadAll(io.LimitReader(body, limit+1))
+ if err != nil {
+ return nil, false, err
+ }
+ if int64(len(limited)) > limit {
+ return limited[:limit], true, nil
+ }
+ return limited, false, nil
+}
+
+func detectContentType(header string, body []byte) string {
+ mediaType := normalizeContentType(header)
+ if mediaType != "" {
+ return mediaType
+ }
+ if len(body) == 0 {
+ return ""
+ }
+ return normalizeContentType(http.DetectContentType(body))
+}
+
+func (t *Tool) extractContent(contentType string, body []byte) (string, string, error) {
+ if !t.supports(contentType) {
+ return "", "", fmt.Errorf("%s: %s", toolName, reasonUnsupportedType)
}
- return result, nil
+ if isHTMLContentType(contentType) {
+ text, title, err := extractHTMLContent(body)
+ if err != nil {
+ return "", "", fmt.Errorf("parse html: %w", err)
+ }
+ return text, title, nil
+ }
+ return normalizePlainText(body), "", nil
+}
+
+func (t *Tool) supports(contentType string) bool {
+ _, ok := t.supportedText[contentType]
+ return ok
+}
+
+func (t *Tool) newSuccessResult(data responseData) tools.ToolResult {
+ return tools.ToolResult{
+ Name: t.Name(),
+ Content: formatSuccess(data),
+ IsError: false,
+ Metadata: metadataFromResponse(data),
+ }
+}
+
+func (t *Tool) newErrorResult(data responseData, reason string, details string) tools.ToolResult {
+ return tools.ToolResult{
+ Name: t.Name(),
+ Content: formatError(data, reason, details),
+ IsError: true,
+ Metadata: metadataFromResponse(data),
+ }
+}
+
+func formatSuccess(data responseData) string {
+ lines := formatCommonLines(data)
+ if data.Title != "" {
+ lines = append(lines, "title: "+data.Title)
+ }
+ return joinMessage(lines, data.Content)
+}
+
+func formatError(data responseData, reason string, details string) string {
+ lines := append([]string{"webfetch error"}, formatCommonLines(data)...)
+ lines = append(lines, "reason: "+reason)
+ return joinMessage(lines, details)
+}
+
+func formatCommonLines(data responseData) []string {
+ lines := make([]string, 0, 5)
+ if data.URL != "" {
+ lines = append(lines, "url: "+data.URL)
+ }
+ if data.Status != "" {
+ lines = append(lines, "status: "+data.Status)
+ }
+ if data.ContentType != "" {
+ lines = append(lines, "content_type: "+data.ContentType)
+ }
+ if data.Truncated {
+ lines = append(lines, "truncated: true")
+ }
+ return lines
+}
+
+func joinMessage(lines []string, body string) string {
+ header := strings.Join(lines, "\n")
+ content := strings.TrimSpace(body)
+ if content == "" {
+ return header
+ }
+ if header == "" {
+ return content
+ }
+ return header + "\n\n" + content
+}
+
+func metadataFromResponse(data responseData) map[string]any {
+ metadata := map[string]any{
+ "url": data.URL,
+ "status": data.Status,
+ "content_type": data.ContentType,
+ "truncated": data.Truncated,
+ }
+ if data.Title != "" {
+ metadata["title"] = data.Title
+ }
+ return metadata
+}
+
+func normalizeConfig(cfg Config) Config {
+ if cfg.Timeout <= 0 {
+ cfg.Timeout = time.Duration(config.DefaultToolTimeoutSec) * time.Second
+ }
+ if cfg.MaxResponseBytes <= 0 {
+ cfg.MaxResponseBytes = config.DefaultWebFetchMaxResponseBytes
+ }
+ if len(cfg.SupportedContentTypes) == 0 {
+ cfg.SupportedContentTypes = config.DefaultWebFetchSupportedContentTypes()
+ }
+
+ normalized := make([]string, 0, len(cfg.SupportedContentTypes))
+ seen := make(map[string]struct{}, len(cfg.SupportedContentTypes))
+ for _, contentType := range cfg.SupportedContentTypes {
+ mediaType := normalizeContentType(contentType)
+ if mediaType == "" {
+ continue
+ }
+ if _, exists := seen[mediaType]; exists {
+ continue
+ }
+ seen[mediaType] = struct{}{}
+ normalized = append(normalized, mediaType)
+ }
+ cfg.SupportedContentTypes = normalized
+ return cfg
+}
+
+func newContentTypeSet(contentTypes []string) map[string]struct{} {
+ set := make(map[string]struct{}, len(contentTypes))
+ for _, contentType := range contentTypes {
+ set[contentType] = struct{}{}
+ }
+ return set
+}
+
+func normalizeContentType(value string) string {
+ trimmed := strings.ToLower(strings.TrimSpace(value))
+ if trimmed == "" {
+ return ""
+ }
+
+ mediaType, _, err := mime.ParseMediaType(trimmed)
+ if err == nil {
+ return mediaType
+ }
+ if index := strings.Index(trimmed, ";"); index >= 0 {
+ return strings.TrimSpace(trimmed[:index])
+ }
+ return trimmed
+}
+
+func isHTMLContentType(contentType string) bool {
+ return contentType == htmlContentType || contentType == xhtmlContentType
+}
+
+func normalizePlainText(body []byte) string {
+ text := strings.ReplaceAll(string(body), "\r\n", "\n")
+ return strings.TrimSpace(text)
}
diff --git a/internal/tools/webfetch/tool_test.go b/internal/tools/webfetch/tool_test.go
index e43bb001..a655d592 100644
--- a/internal/tools/webfetch/tool_test.go
+++ b/internal/tools/webfetch/tool_test.go
@@ -9,7 +9,9 @@ import (
"testing"
"time"
+ "github.com/dust/neo-code/internal/config"
"github.com/dust/neo-code/internal/tools"
+ "golang.org/x/net/html"
)
func TestToolExecute(t *testing.T) {
@@ -17,10 +19,44 @@ func TestToolExecute(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
- case "/ok":
- w.WriteHeader(http.StatusOK)
+ case "/html":
+ w.Header().Set("Content-Type", "text/html; charset=utf-8")
+ _, _ = w.Write([]byte(`
+
+
+ Example Domain
+
+
+
+
+
+ Example Domain
+ This domain is for use in illustrative examples.
+
+
+
+
+ `))
+ case "/html-empty":
+ w.Header().Set("Content-Type", "text/html")
+ _, _ = w.Write([]byte(`Blank`))
+ case "/plain":
+ w.Header().Set("Content-Type", "text/plain")
_, _ = w.Write([]byte("hello webfetch"))
+ case "/json":
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(`{"ok":true}`))
+ case "/image":
+ w.Header().Set("Content-Type", "image/png")
+ _, _ = w.Write([]byte("\x89PNG\x0d\x0a\x1a\x0a"))
+ case "/octet":
+ w.Header().Set("Content-Type", "application/octet-stream")
+ _, _ = w.Write([]byte("binary payload"))
+ case "/large":
+ w.Header().Set("Content-Type", "text/plain")
+ _, _ = w.Write([]byte("0123456789abcdef"))
case "/fail":
+ w.Header().Set("Content-Type", "text/plain")
w.WriteHeader(http.StatusBadGateway)
_, _ = w.Write([]byte("upstream failed"))
default:
@@ -29,54 +65,186 @@ func TestToolExecute(t *testing.T) {
}))
defer server.Close()
- tool := New(2 * time.Second)
- if tool.Name() != "webfetch" {
- t.Fatalf("unexpected tool name %q", tool.Name())
+ closedServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}))
+ closedURL := closedServer.URL
+ closedServer.Close()
+
+ defaultConfig := Config{
+ Timeout: 2 * time.Second,
+ MaxResponseBytes: config.DefaultWebFetchMaxResponseBytes,
+ SupportedContentTypes: config.DefaultWebFetchSupportedContentTypes(),
+ }
+ if New(defaultConfig).Name() != "webfetch" {
+ t.Fatalf("unexpected tool name %q", New(defaultConfig).Name())
}
- if tool.Description() == "" {
+ if New(defaultConfig).Description() == "" {
t.Fatalf("expected non-empty description")
}
- if tool.Schema()["type"] != "object" {
+ if New(defaultConfig).Schema()["type"] != "object" {
t.Fatalf("expected schema object")
}
tests := []struct {
- name string
- args any
- expectErr string
- expectContent string
- expectStatus string
- expectIsError bool
+ name string
+ args any
+ toolConfig Config
+ expectErr string
+ expectContent []string
+ rejectContent []string
+ expectStatus string
+ expectType string
+ expectIsError bool
+ expectTruncated bool
}{
{
- name: "fetch success",
- args: map[string]string{"url": server.URL + "/ok"},
- expectContent: "hello webfetch",
+ name: "extracts html body text",
+ args: map[string]string{"url": server.URL + "/html"},
+ toolConfig: defaultConfig,
+ expectContent: []string{
+ "url: " + server.URL + "/html",
+ "status: 200 OK",
+ "content_type: text/html",
+ "title: Example Domain",
+ "Example Domain This domain is for use in illustrative examples.",
+ },
+ rejectContent: []string{"ignored()", "display:none", "skip me"},
+ expectStatus: "200 OK",
+ expectType: "text/html",
+ },
+ {
+ name: "rejects empty html content",
+ args: map[string]string{"url": server.URL + "/html-empty"},
+ toolConfig: defaultConfig,
+ expectErr: reasonEmptyContent,
+ expectContent: []string{
+ "webfetch error",
+ "content_type: text/html",
+ "reason: " + reasonEmptyContent,
+ },
expectStatus: "200 OK",
+ expectType: "text/html",
+ expectIsError: true,
+ },
+ {
+ name: "returns plain text response",
+ args: map[string]string{"url": server.URL + "/plain"},
+ toolConfig: defaultConfig,
+ expectContent: []string{
+ "content_type: text/plain",
+ "hello webfetch",
+ },
+ expectStatus: "200 OK",
+ expectType: "text/plain",
},
{
- name: "fetch http error",
- args: map[string]string{"url": server.URL + "/fail"},
- expectErr: "502 Bad Gateway",
- expectContent: "upstream failed",
+ name: "returns json as text",
+ args: map[string]string{"url": server.URL + "/json"},
+ toolConfig: defaultConfig,
+ expectContent: []string{
+ "content_type: application/json",
+ `{"ok":true}`,
+ },
+ expectStatus: "200 OK",
+ expectType: "application/json",
+ },
+ {
+ name: "rejects unsupported image content type",
+ args: map[string]string{"url": server.URL + "/image"},
+ toolConfig: defaultConfig,
+ expectErr: reasonUnsupportedType,
+ expectContent: []string{
+ "webfetch error",
+ "content_type: image/png",
+ "reason: " + reasonUnsupportedType,
+ },
+ expectStatus: "200 OK",
+ expectType: "image/png",
+ expectIsError: true,
+ },
+ {
+ name: "rejects unsupported octet stream",
+ args: map[string]string{"url": server.URL + "/octet"},
+ toolConfig: defaultConfig,
+ expectErr: reasonUnsupportedType,
+ expectContent: []string{
+ "reason: " + reasonUnsupportedType,
+ },
+ expectStatus: "200 OK",
+ expectType: "application/octet-stream",
+ expectIsError: true,
+ },
+ {
+ name: "marks truncated text responses",
+ args: map[string]string{"url": server.URL + "/large"},
+ toolConfig: Config{
+ Timeout: 2 * time.Second,
+ MaxResponseBytes: 8,
+ SupportedContentTypes: config.DefaultWebFetchSupportedContentTypes(),
+ },
+ expectContent: []string{
+ "truncated: true",
+ "01234567",
+ },
+ expectStatus: "200 OK",
+ expectType: "text/plain",
+ expectTruncated: true,
+ },
+ {
+ name: "formats http errors consistently",
+ args: map[string]string{"url": server.URL + "/fail"},
+ toolConfig: defaultConfig,
+ expectErr: "unexpected HTTP status 502 Bad Gateway",
+ expectContent: []string{
+ "webfetch error",
+ "status: 502 Bad Gateway",
+ "content_type: text/plain",
+ "reason: unexpected HTTP status 502 Bad Gateway",
+ "upstream failed",
+ },
expectStatus: "502 Bad Gateway",
+ expectType: "text/plain",
+ expectIsError: true,
+ },
+ {
+ name: "rejects invalid scheme",
+ args: map[string]string{"url": "ftp://example.com"},
+ toolConfig: defaultConfig,
+ expectErr: "url must start with http:// or https://",
+ expectContent: []string{
+ "webfetch error",
+ "reason: " + reasonInvalidURL,
+ },
expectIsError: true,
},
{
- name: "rejects invalid scheme",
- args: map[string]string{"url": "ftp://example.com"},
- expectErr: "url must start with http:// or https://",
+ name: "rejects invalid json",
+ args: "{",
+ toolConfig: defaultConfig,
+ expectErr: "JSON input",
+ expectContent: []string{
+ "webfetch error",
+ "reason: " + reasonInvalidArguments,
+ },
+ expectIsError: true,
},
{
- name: "rejects invalid json",
- args: "{",
- expectErr: "JSON input",
+ name: "returns request failure when server is unavailable",
+ args: map[string]string{"url": closedURL},
+ toolConfig: defaultConfig,
+ expectErr: "webfetch: fetch",
+ expectContent: []string{
+ "webfetch error",
+ "reason: " + reasonRequestFailed,
+ },
+ expectIsError: true,
},
}
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
+ tool := New(tt.toolConfig)
+
var raw []byte
switch value := tt.args.(type) {
case string:
@@ -102,15 +270,88 @@ func TestToolExecute(t *testing.T) {
t.Fatalf("unexpected error: %v", execErr)
}
- if tt.expectContent != "" && !strings.Contains(result.Content, tt.expectContent) {
- t.Fatalf("expected content containing %q, got %q", tt.expectContent, result.Content)
+ for _, want := range tt.expectContent {
+ if !strings.Contains(result.Content, want) {
+ t.Fatalf("expected content containing %q, got %q", want, result.Content)
+ }
+ }
+ for _, reject := range tt.rejectContent {
+ if strings.Contains(result.Content, reject) {
+ t.Fatalf("expected content not to contain %q, got %q", reject, result.Content)
+ }
}
if tt.expectStatus != "" && result.Metadata["status"] != tt.expectStatus {
t.Fatalf("expected status %q, got %+v", tt.expectStatus, result.Metadata)
}
+ if tt.expectType != "" && result.Metadata["content_type"] != tt.expectType {
+ t.Fatalf("expected content_type %q, got %+v", tt.expectType, result.Metadata)
+ }
+ if truncated, ok := result.Metadata["truncated"].(bool); !ok || truncated != tt.expectTruncated {
+ t.Fatalf("expected truncated=%v, got %+v", tt.expectTruncated, result.Metadata)
+ }
if result.IsError != tt.expectIsError {
t.Fatalf("expected IsError=%v, got %v", tt.expectIsError, result.IsError)
}
})
}
}
+
+func TestToolDefaultsAndContentTypeFallback(t *testing.T) {
+ t.Parallel()
+
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write([]byte("fallback text body"))
+ }))
+ defer server.Close()
+
+ tool := New(Config{})
+ args, err := json.Marshal(map[string]string{"url": server.URL})
+ if err != nil {
+ t.Fatalf("marshal args: %v", err)
+ }
+
+ result, execErr := tool.Execute(context.Background(), tools.ToolCallInput{
+ Name: tool.Name(),
+ Arguments: args,
+ })
+ if execErr != nil {
+ t.Fatalf("unexpected error: %v", execErr)
+ }
+ if result.Metadata["content_type"] != "text/plain" {
+ t.Fatalf("expected detected text/plain content type, got %+v", result.Metadata)
+ }
+ if !strings.Contains(result.Content, "fallback text body") {
+ t.Fatalf("expected fallback body content, got %q", result.Content)
+ }
+}
+
+func TestHTMLHelpers(t *testing.T) {
+ t.Parallel()
+
+ text, title, err := extractHTMLContent([]byte(`hello world
`))
+ if err != nil {
+ t.Fatalf("extractHTMLContent() error = %v", err)
+ }
+ if text != "hello world" {
+ t.Fatalf("expected extracted text %q, got %q", "hello world", text)
+ }
+ if title != "" {
+ t.Fatalf("expected empty title, got %q", title)
+ }
+
+ if findElement(nil, "body") != nil {
+ t.Fatalf("expected nil findElement result for nil root")
+ }
+ if shouldSkipNode(nil) {
+ t.Fatalf("expected shouldSkipNode(nil) to be false")
+ }
+ if normalizeWhitespace(" hello \n world ") != "hello world" {
+ t.Fatalf("expected normalizeWhitespace to collapse whitespace")
+ }
+
+ titleNode := &html.Node{Type: html.ElementNode, Data: "title"}
+ titleNode.AppendChild(&html.Node{Type: html.TextNode, Data: "Example"})
+ if textContent(titleNode) != "Example " {
+ t.Fatalf("expected title text content, got %q", textContent(titleNode))
+ }
+}