Merge remote-tracking branch 'origin/main'

# Conflicts:
#	internal/config/model.go

Author: phantom5099

go.mod

@@ -6,6 +6,8 @@ require (
 	github.com/charmbracelet/bubbles v1.0.0
 	github.com/charmbracelet/bubbletea v1.3.10
 	github.com/charmbracelet/lipgloss v1.1.0
+	github.com/joho/godotenv v1.5.1
+	golang.org/x/net v0.50.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -20,7 +22,6 @@ require (
 	github.com/clipperhouse/stringish v0.1.1 // indirect
 	github.com/clipperhouse/uax29/v2 v2.5.0 // indirect
 	github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
-	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/mattn/go-localereader v0.0.1 // indirect
@@ -31,6 +32,6 @@ require (
 	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/sahilm/fuzzy v0.1.1 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
-	golang.org/x/sys v0.38.0 // indirect
-	golang.org/x/text v0.3.8 // indirect
+	golang.org/x/sys v0.41.0 // indirect
+	golang.org/x/text v0.34.0 // indirect
 )

go.sum

@@ -56,12 +56,18 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
+golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
+golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
+golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

internal/app/bootstrap.go

@@ -25,14 +25,7 @@ func NewProgram(ctx context.Context) (*tea.Program, error) {
 		return nil, err
 	}
 
-	toolRegistry := tools.NewRegistry()
-	toolRegistry.Register(filesystem.New(cfg.Workdir))
-	toolRegistry.Register(filesystem.NewWrite(cfg.Workdir))
-	toolRegistry.Register(filesystem.NewGrep(cfg.Workdir))
-	toolRegistry.Register(filesystem.NewGlob(cfg.Workdir))
-	toolRegistry.Register(filesystem.NewEdit(cfg.Workdir))
-	toolRegistry.Register(bash.New(cfg.Workdir, cfg.Shell, time.Duration(cfg.ToolTimeoutSec)*time.Second))
-	toolRegistry.Register(webfetch.New(time.Duration(cfg.ToolTimeoutSec) * time.Second))
+	toolRegistry := buildToolRegistry(cfg)
 
 	providerRegistry, err := builtin.NewRegistry()
 	if err != nil {
@@ -53,3 +46,19 @@ func NewProgram(ctx context.Context) (*tea.Program, error) {
 		tea.WithMouseCellMotion(),
 	), nil
 }
+
+func buildToolRegistry(cfg config.Config) *tools.Registry {
+	toolRegistry := tools.NewRegistry()
+	toolRegistry.Register(filesystem.New(cfg.Workdir))
+	toolRegistry.Register(filesystem.NewWrite(cfg.Workdir))
+	toolRegistry.Register(filesystem.NewGrep(cfg.Workdir))
+	toolRegistry.Register(filesystem.NewGlob(cfg.Workdir))
+	toolRegistry.Register(filesystem.NewEdit(cfg.Workdir))
+	toolRegistry.Register(bash.New(cfg.Workdir, cfg.Shell, time.Duration(cfg.ToolTimeoutSec)*time.Second))
+	toolRegistry.Register(webfetch.New(webfetch.Config{
+		Timeout:               time.Duration(cfg.ToolTimeoutSec) * time.Second,
+		MaxResponseBytes:      cfg.Tools.WebFetch.MaxResponseBytes,
+		SupportedContentTypes: cfg.Tools.WebFetch.SupportedContentTypes,
+	}))
+	return toolRegistry
+}

internal/app/bootstrap_test.go

@@ -2,9 +2,15 @@ package app
 
 import (
 	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"testing"
+
+	"github.com/dust/neo-code/internal/config"
+	"github.com/dust/neo-code/internal/tools"
 )
 
 func TestNewProgram(t *testing.T) {
@@ -25,3 +31,42 @@ func TestNewProgram(t *testing.T) {
 		t.Fatalf("expected config file to be created at %q: %v", configPath, err)
 	}
 }
+
+func TestBuildToolRegistryUsesWebFetchConfig(t *testing.T) {
+	t.Parallel()
+
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain")
+		_, _ = w.Write([]byte("1234567890"))
+	}))
+	defer server.Close()
+
+	cfg := config.Default().Clone()
+	cfg.Workdir = t.TempDir()
+	cfg.Tools.WebFetch.MaxResponseBytes = 4
+
+	registry := buildToolRegistry(cfg)
+	tool, err := registry.Get("webfetch")
+	if err != nil {
+		t.Fatalf("registry.Get(webfetch) error = %v", err)
+	}
+
+	args, err := json.Marshal(map[string]string{"url": server.URL})
+	if err != nil {
+		t.Fatalf("marshal args: %v", err)
+	}
+
+	result, execErr := tool.Execute(context.Background(), tools.ToolCallInput{
+		Name:      "webfetch",
+		Arguments: args,
+	})
+	if execErr != nil {
+		t.Fatalf("webfetch execute error = %v", execErr)
+	}
+	if truncated, ok := result.Metadata["truncated"].(bool); !ok || !truncated {
+		t.Fatalf("expected truncated metadata, got %+v", result.Metadata)
+	}
+	if result.Content == "" {
+		t.Fatalf("expected formatted webfetch content")
+	}
+}

internal/config/config_test.go

@@ -50,7 +50,15 @@ selected_provider: openai
 current_model: gpt-5.4
 workdir: .
 shell: powershell
+
 provider_overrides:
+tools:
+  webfetch:
+    max_response_bytes: 4096
+    supported_content_types:
+      - text/html
+      - text/plain
+providers:
   - name: openai
     base_url: https://example.com/v1
     model: gpt-5.4
@@ -68,6 +76,12 @@ provider_overrides:
 				if provider.BaseURL != "https://example.com/v1" {
 					t.Fatalf("expected custom base url, got %q", provider.BaseURL)
 				}
+				if cfg.Tools.WebFetch.MaxResponseBytes != 4096 {
+					t.Fatalf("expected custom max_response_bytes 4096, got %d", cfg.Tools.WebFetch.MaxResponseBytes)
+				}
+				if len(cfg.Tools.WebFetch.SupportedContentTypes) != 2 {
+					t.Fatalf("expected 2 supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes)
+				}
 			},
 		},
 		{
@@ -126,6 +140,12 @@ providers:
 				if provider.Model != "gpt-4o" {
 					t.Fatalf("expected provider model gpt-4o, got %q", provider.Model)
 				}
+				if cfg.Tools.WebFetch.MaxResponseBytes != DefaultWebFetchMaxResponseBytes {
+					t.Fatalf("expected default max_response_bytes %d, got %d", DefaultWebFetchMaxResponseBytes, cfg.Tools.WebFetch.MaxResponseBytes)
+				}
+				if len(cfg.Tools.WebFetch.SupportedContentTypes) != len(DefaultWebFetchSupportedContentTypes()) {
+					t.Fatalf("expected default supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes)
+				}
 			},
 		},
 	}
@@ -388,6 +408,12 @@ func TestConfigApplyDefaultsFillsMissingFields(t *testing.T) {
 	if !filepath.IsAbs(cfg.Workdir) {
 		t.Fatalf("expected absolute workdir, got %q", cfg.Workdir)
 	}
+	if cfg.Tools.WebFetch.MaxResponseBytes != DefaultWebFetchMaxResponseBytes {
+		t.Fatalf("expected default webfetch max_response_bytes %d, got %d", DefaultWebFetchMaxResponseBytes, cfg.Tools.WebFetch.MaxResponseBytes)
+	}
+	if len(cfg.Tools.WebFetch.SupportedContentTypes) != len(DefaultWebFetchSupportedContentTypes()) {
+		t.Fatalf("expected default supported content types, got %+v", cfg.Tools.WebFetch.SupportedContentTypes)
+	}
 }
 
 func TestConfigValidateFailures(t *testing.T) {
@@ -449,6 +475,24 @@ func TestConfigValidateFailures(t *testing.T) {
 			}(),
 			expectErr: "model is empty",
 		},
+		{
+			name: "invalid webfetch max response bytes",
+			config: func() *Config {
+				cfg := validConfig.Clone()
+				cfg.Tools.WebFetch.MaxResponseBytes = 0
+				return &cfg
+			}(),
+			expectErr: "max_response_bytes must be greater than 0",
+		},
+		{
+			name: "invalid webfetch supported content types",
+			config: func() *Config {
+				cfg := validConfig.Clone()
+				cfg.Tools.WebFetch.SupportedContentTypes = []string{""}
+				return &cfg
+			}(),
+			expectErr: "supported_content_types[0] is empty",
+		},
 	}
 
 	for _, tt := range tests {
@@ -568,6 +612,8 @@ func TestLoaderLoadAndSaveRoundTrip(t *testing.T) {
 			cfg.Providers[i].Model = "gpt-5.4"
 		}
 	}
+	cfg.Tools.WebFetch.MaxResponseBytes = 1024
+	cfg.Tools.WebFetch.SupportedContentTypes = []string{"text/html", "application/json"}
 	if err := loader.Save(context.Background(), cfg); err != nil {
 		t.Fatalf("Save() error = %v", err)
 	}
@@ -591,6 +637,12 @@ func TestLoaderLoadAndSaveRoundTrip(t *testing.T) {
 	if reloaded.CurrentModel != "gpt-5.4" {
 		t.Fatalf("expected current model %q, got %q", "gpt-5.4", reloaded.CurrentModel)
 	}
+	if reloaded.Tools.WebFetch.MaxResponseBytes != 1024 {
+		t.Fatalf("expected max_response_bytes %d, got %d", 1024, reloaded.Tools.WebFetch.MaxResponseBytes)
+	}
+	if len(reloaded.Tools.WebFetch.SupportedContentTypes) != 2 {
+		t.Fatalf("expected persisted supported content types, got %+v", reloaded.Tools.WebFetch.SupportedContentTypes)
+	}
 }
 
 func TestLoaderUsesUpdatedBuiltinProviderWhenUserHasNoOverride(t *testing.T) {
@@ -732,9 +784,13 @@ func TestNormalizeWorkdirAndClone(t *testing.T) {
 	cfg := testDefaultConfig()
 	cloned := cfg.Clone()
 	cloned.CurrentModel = "modified"
+	cloned.Tools.WebFetch.SupportedContentTypes[0] = "application/json"
 	if cfg.CurrentModel == cloned.CurrentModel {
 		t.Fatalf("expected clone to be independent from source")
 	}
+	if cfg.Tools.WebFetch.SupportedContentTypes[0] == cloned.Tools.WebFetch.SupportedContentTypes[0] {
+		t.Fatalf("expected webfetch supported content types to be cloned")
+	}
 }
 
 func TestManagerHelperMethodsAndReloads(t *testing.T) {

internal/tools/webfetch/html.go

@@ -0,0 +1,112 @@
+package webfetch
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+var skippedHTMLTags = map[string]struct{}{
+	"canvas":   {},
+	"head":     {},
+	"noscript": {},
+	"script":   {},
+	"style":    {},
+	"svg":      {},
+	"template": {},
+}
+
+func extractHTMLContent(body []byte) (string, string, error) {
+	doc, err := html.Parse(bytes.NewReader(body))
+	if err != nil {
+		return "", "", fmt.Errorf("parse document: %w", err)
+	}
+
+	root := findElement(doc, "body")
+	if root == nil {
+		root = doc
+	}
+
+	text := normalizeWhitespace(collectVisibleText(root))
+	title := normalizeWhitespace(textContent(findElement(doc, "title")))
+	return text, title, nil
+}
+
+func findElement(node *html.Node, tag string) *html.Node {
+	if node == nil {
+		return nil
+	}
+	if node.Type == html.ElementNode && strings.EqualFold(node.Data, tag) {
+		return node
+	}
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		if found := findElement(child, tag); found != nil {
+			return found
+		}
+	}
+	return nil
+}
+
+func collectVisibleText(node *html.Node) string {
+	var builder strings.Builder
+	appendVisibleText(&builder, node)
+	return builder.String()
+}
+
+func appendVisibleText(builder *strings.Builder, node *html.Node) {
+	if node == nil {
+		return
+	}
+	if shouldSkipNode(node) {
+		return
+	}
+
+	if node.Type == html.TextNode {
+		text := strings.TrimSpace(node.Data)
+		if text != "" {
+			builder.WriteString(text)
+			builder.WriteByte('\n')
+		}
+	}
+
+	for child := node.FirstChild; child != nil; child = child.NextSibling {
+		appendVisibleText(builder, child)
+	}
+}
+
+func shouldSkipNode(node *html.Node) bool {
+	if node == nil || node.Type != html.ElementNode {
+		return false
+	}
+	_, skip := skippedHTMLTags[strings.ToLower(node.Data)]
+	return skip
+}
+
+func textContent(node *html.Node) string {
+	if node == nil {
+		return ""
+	}
+
+	var builder strings.Builder
+	var visit func(*html.Node)
+	visit = func(current *html.Node) {
+		if current == nil {
+			return
+		}
+		if current.Type == html.TextNode {
+			builder.WriteString(current.Data)
+			builder.WriteByte(' ')
+		}
+		for child := current.FirstChild; child != nil; child = child.NextSibling {
+			visit(child)
+		}
+	}
+	visit(node)
+	return builder.String()
+}
+
+func normalizeWhitespace(value string) string {
+	return strings.Join(strings.Fields(value), " ")
+}